ZODB/ 0000751 0001750 0000144 00000000000 10424632375 011170 5 ustar ctheune users ZODB/src/ 0000751 0001750 0000144 00000000000 10424632415 011752 5 ustar ctheune users ZODB/src/persistent/ 0000751 0001750 0000144 00000000000 10424632372 014154 5 ustar ctheune users ZODB/src/persistent/SETUP.cfg 0000640 0001750 0000144 00000001210 10050174114 015515 0 ustar ctheune users # Extension information for zpkg.
# Mark an "exported" header for use from other packages.
# This is not needed for headers only used within the package.
#
header cPersistence.h
# This is included by cPersistence.h, so all users of cPersistence.h
# have to be able to include this indirectly.
#
header ring.h
source cPersistence.c
source ring.c
depends-on cPersistence.h
depends-on ring.h
source cPickleCache.c
source ring.c
depends-on cPersistence.h
depends-on ring.h
source TimeStamp.c
ZODB/src/persistent/wref.py 0000640 0001750 0000144 00000020102 10214427545 015465 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""ZODB-based persistent weakrefs
$Id: wref.py 29450 2005-03-11 23:53:09Z tim_one $
"""
from persistent import Persistent
WeakRefMarker = object()
class WeakRef(object):
"""Persistent weak references
Persistent weak references are used much like Python weak
references. The major difference is that you can't specify an
object to be called when the object is removed from the database.
Here's an example. We'll start by creating a persistent object and
a refernce to it:
>>> import persistent.list
>>> import ZODB.tests.util
>>> ob = persistent.list.PersistentList()
>>> ref = WeakRef(ob)
>>> ref() is ob
True
The hash of the ref if the same as the hash of the referenced object:
>>> hash(ref) == hash(ob)
True
Two refs to the same object are equal:
>>> WeakRef(ob) == ref
True
>>> ob2 = persistent.list.PersistentList([1])
>>> WeakRef(ob2) == ref
False
Lets save the reference and the referenced object in a database:
>>> db = ZODB.tests.util.DB()
>>> conn1 = db.open()
>>> conn1.root()['ob'] = ob
>>> conn1.root()['ref'] = ref
>>> ZODB.tests.util.commit()
If we open a new connection, we can use the reference:
>>> conn2 = db.open()
>>> conn2.root()['ref']() is conn2.root()['ob']
True
>>> hash(conn2.root()['ref']) == hash(conn2.root()['ob'])
True
But if we delete the referenced object and pack:
>>> del conn2.root()['ob']
>>> ZODB.tests.util.commit()
>>> ZODB.tests.util.pack(db)
And then look in a new connection:
>>> conn3 = db.open()
>>> conn3.root()['ob']
Traceback (most recent call last):
...
KeyError: 'ob'
Trying to dereference the reference returns None:
>>> conn3.root()['ref']()
Trying to get a hash, raises a type error:
>>> hash(conn3.root()['ref'])
Traceback (most recent call last):
...
TypeError: Weakly-referenced object has gone away
Always explicitly close databases: :)
>>> db.close()
"""
# We set _p_oid to a marker so that the serialization system can
# provide special handling of weakrefs.
_p_oid = WeakRefMarker
def __init__(self, ob):
self._v_ob = ob
self.oid = ob._p_oid
self.dm = ob._p_jar
def __call__(self):
try:
return self._v_ob
except AttributeError:
try:
self._v_ob = self.dm[self.oid]
except KeyError:
return None
return self._v_ob
def __hash__(self):
self = self()
if self is None:
raise TypeError('Weakly-referenced object has gone away')
return hash(self)
def __eq__(self, other):
self = self()
if self is None:
raise TypeError('Weakly-referenced object has gone away')
other = other()
if other is None:
raise TypeError('Weakly-referenced object has gone away')
return self == other
class PersistentWeakKeyDictionary(Persistent):
"""Persistent weak key dictionary
This is akin to WeakKeyDictionaries. Note, however, that removal
of items is extremely lazy. See below.
We'll start by creating a PersistentWeakKeyDictionary and adding
some persistent objects to it.
>>> d = PersistentWeakKeyDictionary()
>>> import ZODB.tests.util
>>> p1 = ZODB.tests.util.P('p1')
>>> p2 = ZODB.tests.util.P('p2')
>>> p3 = ZODB.tests.util.P('p3')
>>> d[p1] = 1
>>> d[p2] = 2
>>> d[p3] = 3
We'll create an extra persistent object that's not in the dict:
>>> p4 = ZODB.tests.util.P('p4')
Now we'll excercise iteration and item access:
>>> l = [(str(k), d[k], d.get(k)) for k in d]
>>> l.sort()
>>> l
[('P(p1)', 1, 1), ('P(p2)', 2, 2), ('P(p3)', 3, 3)]
And the containment operator:
>>> [p in d for p in [p1, p2, p3, p4]]
[True, True, True, False]
We can add the dict and the referenced objects to a database:
>>> db = ZODB.tests.util.DB()
>>> conn1 = db.open()
>>> conn1.root()['p1'] = p1
>>> conn1.root()['d'] = d
>>> conn1.root()['p2'] = p2
>>> conn1.root()['p3'] = p3
>>> ZODB.tests.util.commit()
And things still work, as before:
>>> l = [(str(k), d[k], d.get(k)) for k in d]
>>> l.sort()
>>> l
[('P(p1)', 1, 1), ('P(p2)', 2, 2), ('P(p3)', 3, 3)]
>>> [p in d for p in [p1, p2, p3, p4]]
[True, True, True, False]
Likewise, we can read the objects from another connection and
things still work.
>>> conn2 = db.open()
>>> d = conn2.root()['d']
>>> p1 = conn2.root()['p1']
>>> p2 = conn2.root()['p2']
>>> p3 = conn2.root()['p3']
>>> l = [(str(k), d[k], d.get(k)) for k in d]
>>> l.sort()
>>> l
[('P(p1)', 1, 1), ('P(p2)', 2, 2), ('P(p3)', 3, 3)]
>>> [p in d for p in [p1, p2, p3, p4]]
[True, True, True, False]
Now, we'll delete one of the objects from the database, but *not*
from the dictionary:
>>> del conn2.root()['p2']
>>> ZODB.tests.util.commit()
And pack the database, so that the no-longer referenced p2 is
actually removed from the database.
>>> ZODB.tests.util.pack(db)
Now if we access the dictionary in a new connection, it no longer
has p2:
>>> conn3 = db.open()
>>> d = conn3.root()['d']
>>> l = [(str(k), d[k], d.get(k)) for k in d]
>>> l.sort()
>>> l
[('P(p1)', 1, 1), ('P(p3)', 3, 3)]
It's worth nothing that that the versions of the dictionary in
conn1 and conn2 still have p2, because p2 is still in the caches
for those connections.
Always explicitly close databases: :)
>>> db.close()
"""
# TODO: It's expensive trying to load dead objects from the database.
# It would be helpful if the data manager/connection cached these.
def __init__(self, adict=None, **kwargs):
self.data = {}
if adict is not None:
keys = getattr(adict, "keys", None)
if keys is None:
adict = dict(adict)
self.update(adict)
if kwargs:
self.update(kwargs)
def __getstate__(self):
state = Persistent.__getstate__(self)
state['data'] = state['data'].items()
return state
def __setstate__(self, state):
state['data'] = dict([
(k, v) for (k, v) in state['data']
if k() is not None
])
Persistent.__setstate__(self, state)
def __setitem__(self, key, value):
self.data[WeakRef(key)] = value
def __getitem__(self, key):
return self.data[WeakRef(key)]
def __delitem__(self, key):
del self.data[WeakRef(key)]
def get(self, key, default=None):
"""D.get(k[, d]) -> D[k] if k in D, else d.
>>> import ZODB.tests.util
>>> key = ZODB.tests.util.P("key")
>>> missing = ZODB.tests.util.P("missing")
>>> d = PersistentWeakKeyDictionary([(key, 1)])
>>> d.get(key)
1
>>> d.get(missing)
>>> d.get(missing, 12)
12
"""
return self.data.get(WeakRef(key), default)
def __contains__(self, key):
return WeakRef(key) in self.data
def __iter__(self):
for k in self.data:
yield k()
def update(self, adict):
if isinstance(adict, PersistentWeakKeyDictionary):
self.data.update(adict.update)
else:
for k, v in adict.items():
self.data[WeakRef(k)] = v
# TODO: May need more methods, and tests.
ZODB/src/persistent/ring.c 0000640 0001750 0000144 00000003464 10057366465 015277 0 ustar ctheune users /*****************************************************************************
Copyright (c) 2003 Zope Corporation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
#define RING_C "$Id: ring.c 25186 2004-06-02 15:07:33Z jim $\n"
/* Support routines for the doubly-linked list of cached objects.
The cache stores a doubly-linked list of persistent objects, with
space for the pointers allocated in the objects themselves. The cache
stores the distinguished head of the list, which is not a valid
persistent object.
The next pointers traverse the ring in order starting with the least
recently used object. The prev pointers traverse the ring in order
starting with the most recently used object.
*/
#include "Python.h"
#include "ring.h"
void
ring_add(CPersistentRing *ring, CPersistentRing *elt)
{
assert(!elt->r_next);
elt->r_next = ring;
elt->r_prev = ring->r_prev;
ring->r_prev->r_next = elt;
ring->r_prev = elt;
}
void
ring_del(CPersistentRing *elt)
{
elt->r_next->r_prev = elt->r_prev;
elt->r_prev->r_next = elt->r_next;
elt->r_next = NULL;
elt->r_prev = NULL;
}
void
ring_move_to_head(CPersistentRing *ring, CPersistentRing *elt)
{
elt->r_prev->r_next = elt->r_next;
elt->r_next->r_prev = elt->r_prev;
elt->r_next = ring;
elt->r_prev = ring->r_prev;
ring->r_prev->r_next = elt;
ring->r_prev = elt;
}
ZODB/src/persistent/cPickleCache.c 0000640 0001750 0000144 00000077275 10225135606 016635 0 ustar ctheune users /*****************************************************************************
Copyright (c) 2001, 2002 Zope Corporation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
/*
Objects are stored under three different regimes:
Regime 1: Persistent Classes
Persistent Classes are part of ZClasses. They are stored in the
self->data dictionary, and are never garbage collected.
The klass_items() method returns a sequence of (oid,object) tuples for
every Persistent Class, which should make it possible to implement
garbage collection in Python if necessary.
Regime 2: Ghost Objects
There is no benefit to keeping a ghost object which has no external
references, therefore a weak reference scheme is used to ensure that
ghost objects are removed from memory as soon as possible, when the
last external reference is lost.
Ghost objects are stored in the self->data dictionary. Normally a
dictionary keeps a strong reference on its values, however this
reference count is 'stolen'.
This weak reference scheme leaves a dangling reference, in the
dictionary, when the last external reference is lost. To clean up this
dangling reference the persistent object dealloc function calls
self->cache->_oid_unreferenced(self->oid). The cache looks up the oid
in the dictionary, ensures it points to an object whose reference
count is zero, then removes it from the dictionary. Before removing
the object from the dictionary it must temporarily resurrect the
object in much the same way that class instances are resurrected
before their __del__ is called.
Since ghost objects are stored under a different regime to non-ghost
objects, an extra ghostify function in cPersistenceAPI replaces
self->state=GHOST_STATE assignments that were common in other
persistent classes (such as BTrees).
Regime 3: Non-Ghost Objects
Non-ghost objects are stored in two data structures: the dictionary
mapping oids to objects and a doubly-linked list that encodes the
order in which the objects were accessed. The dictionary reference is
borrowed, as it is for ghosts. The list reference is a new reference;
the list stores recently used objects, even if they are otherwise
unreferenced, to avoid loading the object from the database again.
The doubly-link-list nodes contain next and previous pointers linking
together the cache and all non-ghost persistent objects.
The node embedded in the cache is the home position. On every
attribute access a non-ghost object will relink itself just behind the
home position in the ring. Objects accessed least recently will
eventually find themselves positioned after the home position.
Occasionally other nodes are temporarily inserted in the ring as
position markers. The cache contains a ring_lock flag which must be
set and unset before and after doing so. Only if the flag is unset can
the cache assume that all nodes are either his own home node, or nodes
from persistent objects. This assumption is useful during the garbage
collection process.
The number of non-ghost objects is counted in self->non_ghost_count.
The garbage collection process consists of traversing the ring, and
deactivating (that is, turning into a ghost) every object until
self->non_ghost_count is down to the target size, or until it
reaches the home position again.
Note that objects in the sticky or changed states are still kept in
the ring, however they can not be deactivated. The garbage collection
process must skip such objects, rather than deactivating them.
*/
static char cPickleCache_doc_string[] =
"Defines the PickleCache used by ZODB Connection objects.\n"
"\n"
"$Id: cPickleCache.c 29896 2005-04-07 04:48:06Z tim_one $\n";
#define DONT_USE_CPERSISTENCECAPI
#include "cPersistence.h"
#include "structmember.h"
#include
#include
#undef Py_FindMethod
/* Python string objects to speed lookups; set by module init. */
static PyObject *py__p_changed;
static PyObject *py__p_deactivate;
static PyObject *py__p_jar;
static PyObject *py__p_oid;
static cPersistenceCAPIstruct *capi;
/* This object is the pickle cache. The CACHE_HEAD macro guarantees
that layout of this struct is the same as the start of
ccobject_head in cPersistence.c */
typedef struct {
CACHE_HEAD
int klass_count; /* count of persistent classes */
PyObject *data; /* oid -> object dict */
PyObject *jar; /* Connection object */
int cache_size; /* target number of items in cache */
/* Most of the time the ring contains only:
* many nodes corresponding to persistent objects
* one 'home' node from the cache.
In some cases it is handy to temporarily add other types
of node into the ring as placeholders. 'ring_lock' is a boolean
indicating that someone has already done this. Currently this
is only used by the garbage collection code. */
int ring_lock;
/* 'cache_drain_resistance' controls how quickly the cache size will drop
when it is smaller than the configured size. A value of zero means it will
not drop below the configured size (suitable for most caches). Otherwise,
it will remove cache_non_ghost_count/cache_drain_resistance items from
the cache every time (suitable for rarely used caches, such as those
associated with Zope versions. */
int cache_drain_resistance;
} ccobject;
static int cc_ass_sub(ccobject *self, PyObject *key, PyObject *v);
/* ---------------------------------------------------------------- */
#define OBJECT_FROM_RING(SELF, HERE) \
((cPersistentObject *)(((char *)here) - offsetof(cPersistentObject, ring)))
/* Insert self into the ring, following after. */
static void
insert_after(CPersistentRing *self, CPersistentRing *after)
{
assert(self != NULL);
assert(after != NULL);
self->r_prev = after;
self->r_next = after->r_next;
after->r_next->r_prev = self;
after->r_next = self;
}
/* Remove self from the ring. */
static void
unlink_from_ring(CPersistentRing *self)
{
assert(self != NULL);
self->r_prev->r_next = self->r_next;
self->r_next->r_prev = self->r_prev;
}
static int
scan_gc_items(ccobject *self, int target)
{
/* This function must only be called with the ring lock held,
because it places non-object placeholders in the ring.
*/
cPersistentObject *object;
CPersistentRing *here;
CPersistentRing before_original_home;
int result = -1; /* guilty until proved innocent */
/* Scan the ring, from least to most recently used, deactivating
* up-to-date objects, until we either find the ring_home again or
* or we've ghosted enough objects to reach the target size.
* Tricky: __getattr__ and __del__ methods can do anything, and in
* particular if we ghostify an object with a __del__ method, that method
* can load the object again, putting it back into the MRU part of the
* ring. Waiting to find ring_home again can thus cause an infinite
* loop (Collector #1208). So before_original_home records the MRU
* position we start with, and we stop the scan when we reach that.
*/
insert_after(&before_original_home, self->ring_home.r_prev);
here = self->ring_home.r_next; /* least recently used object */
while (here != &before_original_home && self->non_ghost_count > target) {
assert(self->ring_lock);
assert(here != &self->ring_home);
/* At this point we know that the ring only contains nodes
from persistent objects, plus our own home node. We know
this because the ring lock is held. We can safely assume
the current ring node is a persistent object now we know it
is not the home */
object = OBJECT_FROM_RING(self, here);
if (object->state == cPersistent_UPTODATE_STATE) {
CPersistentRing placeholder;
PyObject *method;
PyObject *temp;
int error_occurred = 0;
/* deactivate it. This is the main memory saver. */
/* Add a placeholder, a dummy node in the ring. We need
to do this to mark our position in the ring. It is
possible that the PyObject_GetAttr() call below will
invoke a __getattr__() hook in Python. Also possible
that deactivation will lead to a __del__ method call.
So another thread might run, and mutate the ring as a side
effect of object accesses. There's no predicting then where
in the ring here->next will point after that. The
placeholder won't move as a side effect of calling Python
code.
*/
insert_after(&placeholder, here);
method = PyObject_GetAttr((PyObject *)object, py__p_deactivate);
if (method == NULL)
error_occurred = 1;
else {
temp = PyObject_CallObject(method, NULL);
Py_DECREF(method);
if (temp == NULL)
error_occurred = 1;
}
here = placeholder.r_next;
unlink_from_ring(&placeholder);
if (error_occurred)
goto Done;
}
else
here = here->r_next;
}
result = 0;
Done:
unlink_from_ring(&before_original_home);
return result;
}
static PyObject *
lockgc(ccobject *self, int target_size)
{
/* This is thread-safe because of the GIL, and there's nothing
* in between checking the ring_lock and acquiring it that calls back
* into Python.
*/
if (self->ring_lock) {
Py_INCREF(Py_None);
return Py_None;
}
self->ring_lock = 1;
if (scan_gc_items(self, target_size) < 0) {
self->ring_lock = 0;
return NULL;
}
self->ring_lock = 0;
Py_INCREF(Py_None);
return Py_None;
}
static PyObject *
cc_incrgc(ccobject *self, PyObject *args)
{
int obsolete_arg = -999;
int starting_size = self->non_ghost_count;
int target_size = self->cache_size;
if (self->cache_drain_resistance >= 1) {
/* This cache will gradually drain down to a small size. Check
a (small) number of objects proportional to the current size */
int target_size_2 = (starting_size - 1
- starting_size / self->cache_drain_resistance);
if (target_size_2 < target_size)
target_size = target_size_2;
}
if (!PyArg_ParseTuple(args, "|i:incrgc", &obsolete_arg))
return NULL;
if (obsolete_arg != -999
&&
(PyErr_Warn(PyExc_DeprecationWarning,
"No argument expected")
< 0))
return NULL;
return lockgc(self, target_size);
}
static PyObject *
cc_full_sweep(ccobject *self, PyObject *args)
{
int dt = -999;
/* TODO: This should be deprecated; */
if (!PyArg_ParseTuple(args, "|i:full_sweep", &dt))
return NULL;
if (dt == -999)
return lockgc(self, 0);
else
return cc_incrgc(self, args);
}
static PyObject *
cc_minimize(ccobject *self, PyObject *args)
{
int ignored = -999;
if (!PyArg_ParseTuple(args, "|i:minimize", &ignored))
return NULL;
if (ignored != -999
&&
(PyErr_Warn(PyExc_DeprecationWarning,
"No argument expected")
< 0))
return NULL;
return lockgc(self, 0);
}
static int
_invalidate(ccobject *self, PyObject *key)
{
static PyObject *_p_invalidate = NULL;
PyObject *meth, *v;
v = PyDict_GetItem(self->data, key);
if (v == NULL)
return 0;
if (_p_invalidate == NULL)
{
_p_invalidate = PyString_InternFromString("_p_invalidate");
if (_p_invalidate == NULL)
{
/* It doesn't make any sense to ignore this error, but
the caller ignores all errors.
TODO: and why does it do that? This should be fixed
*/
return -1;
}
}
if (v->ob_refcnt <= 1 && PyType_Check(v)) {
/* This looks wrong, but it isn't. We use strong references to types
because they don't have the ring members.
The result is that we *never* remove classes unless
they are modified. We can fix this by using wekrefs uniformly.
*/
self->klass_count--;
return PyDict_DelItem(self->data, key);
}
meth = PyObject_GetAttr(v, _p_invalidate);
if (meth == NULL)
return -1;
v = PyObject_CallObject(meth, NULL);
Py_DECREF(meth);
return v == NULL ? -1 : 0;
}
static PyObject *
cc_invalidate(ccobject *self, PyObject *inv)
{
PyObject *key, *v;
int i = 0;
if (PyDict_Check(inv))
{
while (PyDict_Next(inv, &i, &key, &v))
{
if (_invalidate(self, key) < 0)
return NULL;
}
PyDict_Clear(inv);
}
else {
if (PyString_Check(inv))
{
if (_invalidate(self, inv) < 0)
return NULL;
}
else {
int l, r;
l = PyObject_Length(inv);
if (l < 0)
return NULL;
for (i=l; --i >= 0; ) {
key = PySequence_GetItem(inv, i);
if (!key)
return NULL;
r = _invalidate(self, key);
Py_DECREF(key);
if (r < 0)
return NULL;
}
/* Dubious: modifying the input may be an unexpected side effect. */
PySequence_DelSlice(inv, 0, l);
}
}
Py_INCREF(Py_None);
return Py_None;
}
static PyObject *
cc_get(ccobject *self, PyObject *args)
{
PyObject *r, *key, *d = NULL;
if (!PyArg_ParseTuple(args, "O|O:get", &key, &d))
return NULL;
r = PyDict_GetItem(self->data, key);
if (!r) {
if (d)
r = d;
else
r = Py_None;
}
Py_INCREF(r);
return r;
}
static PyObject *
cc_items(ccobject *self)
{
return PyObject_CallMethod(self->data, "items", "");
}
static PyObject *
cc_klass_items(ccobject *self)
{
PyObject *l,*k,*v;
int p = 0;
l = PyList_New(0);
if (l == NULL)
return NULL;
while (PyDict_Next(self->data, &p, &k, &v)) {
if(PyType_Check(v)) {
v = Py_BuildValue("OO", k, v);
if (v == NULL) {
Py_DECREF(l);
return NULL;
}
if (PyList_Append(l, v) < 0) {
Py_DECREF(v);
Py_DECREF(l);
return NULL;
}
Py_DECREF(v);
}
}
return l;
}
static PyObject *
cc_debug_info(ccobject *self)
{
PyObject *l,*k,*v;
int p = 0;
l = PyList_New(0);
if (l == NULL)
return NULL;
while (PyDict_Next(self->data, &p, &k, &v))
{
if (v->ob_refcnt <= 0)
v = Py_BuildValue("Oi", k, v->ob_refcnt);
else if (! PyType_Check(v) &&
(v->ob_type->tp_basicsize >= sizeof(cPersistentObject))
)
v = Py_BuildValue("Oisi",
k, v->ob_refcnt, v->ob_type->tp_name,
((cPersistentObject*)v)->state);
else
v = Py_BuildValue("Ois", k, v->ob_refcnt, v->ob_type->tp_name);
if (v == NULL)
goto err;
if (PyList_Append(l, v) < 0)
goto err;
}
return l;
err:
Py_DECREF(l);
return NULL;
}
static PyObject *
cc_lru_items(ccobject *self)
{
PyObject *l;
CPersistentRing *here;
if (self->ring_lock) {
/* When the ring lock is held, we have no way of know which
ring nodes belong to persistent objects, and which a
placeholders. */
PyErr_SetString(PyExc_ValueError,
".lru_items() is unavailable during garbage collection");
return NULL;
}
l = PyList_New(0);
if (l == NULL)
return NULL;
here = self->ring_home.r_next;
while (here != &self->ring_home) {
PyObject *v;
cPersistentObject *object = OBJECT_FROM_RING(self, here);
if (object == NULL) {
Py_DECREF(l);
return NULL;
}
v = Py_BuildValue("OO", object->oid, object);
if (v == NULL) {
Py_DECREF(l);
return NULL;
}
if (PyList_Append(l, v) < 0) {
Py_DECREF(v);
Py_DECREF(l);
return NULL;
}
Py_DECREF(v);
here = here->r_next;
}
return l;
}
static void
cc_oid_unreferenced(ccobject *self, PyObject *oid)
{
/* This is called by the persistent object deallocation function
when the reference count on a persistent object reaches
zero. We need to fix up our dictionary; its reference is now
dangling because we stole its reference count. Be careful to
not release the global interpreter lock until this is
complete. */
PyObject *v;
/* If the cache has been cleared by GC, data will be NULL. */
if (!self->data)
return;
v = PyDict_GetItem(self->data, oid);
assert(v);
assert(v->ob_refcnt == 0);
/* Need to be very hairy here because a dictionary is about
to decref an already deleted object.
*/
#ifdef Py_TRACE_REFS
/* This is called from the deallocation function after the
interpreter has untracked the reference. Track it again.
*/
_Py_NewReference(v);
/* Don't increment total refcount as a result of the
shenanigans played in this function. The _Py_NewReference()
call above creates artificial references to v.
*/
_Py_RefTotal--;
assert(v->ob_type);
#else
Py_INCREF(v);
#endif
assert(v->ob_refcnt == 1);
/* Incremement the refcount again, because delitem is going to
DECREF it. If it's refcount reached zero again, we'd call back to
the dealloc function that called us.
*/
Py_INCREF(v);
/* TODO: Should we call _Py_ForgetReference() on error exit? */
if (PyDict_DelItem(self->data, oid) < 0)
return;
Py_DECREF((ccobject *)((cPersistentObject *)v)->cache);
((cPersistentObject *)v)->cache = NULL;
assert(v->ob_refcnt == 1);
/* Undo the temporary resurrection.
Don't DECREF the object, because this function is called from
the object's dealloc function. If the refcnt reaches zero, it
will all be invoked recursively.
*/
_Py_ForgetReference(v);
}
static PyObject *
cc_ringlen(ccobject *self)
{
CPersistentRing *here;
int c = 0;
for (here = self->ring_home.r_next; here != &self->ring_home;
here = here->r_next)
c++;
return PyInt_FromLong(c);
}
static struct PyMethodDef cc_methods[] = {
{"items", (PyCFunction)cc_items, METH_NOARGS,
"Return list of oid, object pairs for all items in cache."},
{"lru_items", (PyCFunction)cc_lru_items, METH_NOARGS,
"List (oid, object) pairs from the lru list, as 2-tuples."},
{"klass_items", (PyCFunction)cc_klass_items, METH_NOARGS,
"List (oid, object) pairs of cached persistent classes."},
{"full_sweep", (PyCFunction)cc_full_sweep, METH_VARARGS,
"full_sweep() -- Perform a full sweep of the cache."},
{"minimize", (PyCFunction)cc_minimize, METH_VARARGS,
"minimize([ignored]) -- Remove as many objects as possible\n\n"
"Ghostify all objects that are not modified. Takes an optional\n"
"argument, but ignores it."},
{"incrgc", (PyCFunction)cc_incrgc, METH_VARARGS,
"incrgc() -- Perform incremental garbage collection\n\n"
"This method had been depricated!"
"Some other implementations support an optional parameter 'n' which\n"
"indicates a repetition count; this value is ignored."},
{"invalidate", (PyCFunction)cc_invalidate, METH_O,
"invalidate(oids) -- invalidate one, many, or all ids"},
{"get", (PyCFunction)cc_get, METH_VARARGS,
"get(key [, default]) -- get an item, or a default"},
{"ringlen", (PyCFunction)cc_ringlen, METH_NOARGS,
"ringlen() -- Returns number of non-ghost items in cache."},
{"debug_info", (PyCFunction)cc_debug_info, METH_NOARGS,
"debug_info() -- Returns debugging data about objects in the cache."},
{NULL, NULL} /* sentinel */
};
static int
cc_init(ccobject *self, PyObject *args, PyObject *kwds)
{
int cache_size = 100;
PyObject *jar;
if (!PyArg_ParseTuple(args, "O|i", &jar, &cache_size))
return -1;
self->jar = NULL;
self->data = PyDict_New();
if (self->data == NULL) {
Py_DECREF(self);
return -1;
}
/* Untrack the dict mapping oids to objects.
The dict contains uncounted references to ghost objects, so it
isn't safe for GC to visit it. If GC finds an object with more
referents that refcounts, it will die with an assertion failure.
When the cache participates in GC, it will need to traverse the
objects in the doubly-linked list, which will account for all the
non-ghost objects.
*/
PyObject_GC_UnTrack((void *)self->data);
self->jar = jar;
Py_INCREF(jar);
self->cache_size = cache_size;
self->non_ghost_count = 0;
self->klass_count = 0;
self->cache_drain_resistance = 0;
self->ring_lock = 0;
self->ring_home.r_next = &self->ring_home;
self->ring_home.r_prev = &self->ring_home;
return 0;
}
static void
cc_dealloc(ccobject *self)
{
Py_XDECREF(self->data);
Py_XDECREF(self->jar);
PyObject_GC_Del(self);
}
static int
cc_clear(ccobject *self)
{
int pos = 0;
PyObject *k, *v;
/* Clearing the cache is delicate.
A non-ghost object will show up in the ring and in the dict. If
we deallocating the dict before clearing the ring, the GC will
decref each object in the dict. Since the dict references are
uncounted, this will lead to objects having negative refcounts.
Freeing the non-ghost objects should eliminate many objects from
the cache, but there may still be ghost objects left. It's
not safe to decref the dict until it's empty, so we need to manually
clear those out of the dict, too. We accomplish that by replacing
all the ghost objects with None.
*/
/* We don't need to lock the ring, because the cache is unreachable.
It should be impossible for anyone to be modifying the cache.
*/
assert(! self->ring_lock);
while (self->ring_home.r_next != &self->ring_home) {
CPersistentRing *here = self->ring_home.r_next;
cPersistentObject *o = OBJECT_FROM_RING(self, here);
if (o->cache) {
Py_INCREF(o); /* account for uncounted reference */
if (PyDict_DelItem(self->data, o->oid) < 0)
return -1;
}
o->cache = NULL;
Py_DECREF(self);
self->ring_home.r_next = here->r_next;
o->ring.r_prev = NULL;
o->ring.r_next = NULL;
Py_DECREF(o);
here = here->r_next;
}
Py_XDECREF(self->jar);
while (PyDict_Next(self->data, &pos, &k, &v)) {
Py_INCREF(v);
if (PyDict_SetItem(self->data, k, Py_None) < 0)
return -1;
}
Py_XDECREF(self->data);
self->data = NULL;
self->jar = NULL;
return 0;
}
static int
cc_traverse(ccobject *self, visitproc visit, void *arg)
{
int err;
CPersistentRing *here;
/* If we're in the midst of cleaning up old objects, the ring contains
* assorted junk we must not pass on to the visit() callback. This
* should be rare (our cleanup code would need to have called back
* into Python, which in turn triggered Python's gc). When it happens,
* simply don't chase any pointers. The cache will appear to be a
* source of external references then, and at worst we miss cleaning
* up a dead cycle until the next time Python's gc runs.
*/
if (self->ring_lock)
return 0;
#define VISIT(SLOT) \
if (SLOT) { \
err = visit((PyObject *)(SLOT), arg); \
if (err) \
return err; \
}
VISIT(self->jar);
here = self->ring_home.r_next;
/* It is possible that an object is traversed after it is cleared.
In that case, there is no ring.
*/
if (!here)
return 0;
while (here != &self->ring_home) {
cPersistentObject *o = OBJECT_FROM_RING(self, here);
VISIT(o);
here = here->r_next;
}
#undef VISIT
return 0;
}
static int
cc_length(ccobject *self)
{
return PyObject_Length(self->data);
}
static PyObject *
cc_subscript(ccobject *self, PyObject *key)
{
PyObject *r;
r = PyDict_GetItem(self->data, key);
if (r == NULL) {
PyErr_SetObject(PyExc_KeyError, key);
return NULL;
}
Py_INCREF(r);
return r;
}
static int
cc_add_item(ccobject *self, PyObject *key, PyObject *v)
{
int result;
PyObject *oid, *object_again, *jar;
cPersistentObject *p;
/* Sanity check the value given to make sure it is allowed in the cache */
if (PyType_Check(v)) {
/* Its a persistent class, such as a ZClass. Thats ok. */
}
else if (v->ob_type->tp_basicsize < sizeof(cPersistentObject)) {
/* If it's not an instance of a persistent class, (ie Python
classes that derive from persistent.Persistent, BTrees,
etc), report an error.
TODO: checking sizeof() seems a poor test.
*/
PyErr_SetString(PyExc_TypeError,
"Cache values must be persistent objects.");
return -1;
}
/* Can't access v->oid directly because the object might be a
* persistent class.
*/
oid = PyObject_GetAttr(v, py__p_oid);
if (oid == NULL)
return -1;
if (! PyString_Check(oid)) {
PyErr_Format(PyExc_TypeError,
"Cached object oid must be a string, not a %s",
oid->ob_type->tp_name);
return -1;
}
/* we know they are both strings.
* now check if they are the same string.
*/
result = PyObject_Compare(key, oid);
if (PyErr_Occurred()) {
Py_DECREF(oid);
return -1;
}
Py_DECREF(oid);
if (result) {
PyErr_SetString(PyExc_ValueError, "Cache key does not match oid");
return -1;
}
/* useful sanity check, but not strictly an invariant of this class */
jar = PyObject_GetAttr(v, py__p_jar);
if (jar == NULL)
return -1;
if (jar==Py_None) {
Py_DECREF(jar);
PyErr_SetString(PyExc_ValueError,
"Cached object jar missing");
return -1;
}
Py_DECREF(jar);
object_again = PyDict_GetItem(self->data, key);
if (object_again) {
if (object_again != v) {
PyErr_SetString(PyExc_ValueError,
"A different object already has the same oid");
return -1;
} else {
/* re-register under the same oid - no work needed */
return 0;
}
}
if (PyType_Check(v)) {
if (PyDict_SetItem(self->data, key, v) < 0)
return -1;
self->klass_count++;
return 0;
} else {
PerCache *cache = ((cPersistentObject *)v)->cache;
if (cache) {
if (cache != (PerCache *)self)
/* This object is already in a different cache. */
PyErr_SetString(PyExc_ValueError,
"Cache values may only be in one cache.");
return -1;
}
/* else:
This object is already one of ours, which is ok. It
would be very strange if someone was trying to register
the same object under a different key.
*/
}
if (PyDict_SetItem(self->data, key, v) < 0)
return -1;
/* the dict should have a borrowed reference */
Py_DECREF(v);
p = (cPersistentObject *)v;
Py_INCREF(self);
p->cache = (PerCache *)self;
if (p->state >= 0) {
/* insert this non-ghost object into the ring just
behind the home position. */
self->non_ghost_count++;
ring_add(&self->ring_home, &p->ring);
/* this list should have a new reference to the object */
Py_INCREF(v);
}
return 0;
}
static int
cc_del_item(ccobject *self, PyObject *key)
{
PyObject *v;
cPersistentObject *p;
/* unlink this item from the ring */
v = PyDict_GetItem(self->data, key);
if (v == NULL) {
PyErr_SetObject(PyExc_KeyError, key);
return -1;
}
if (PyType_Check(v)) {
self->klass_count--;
} else {
p = (cPersistentObject *)v;
if (p->state >= 0) {
self->non_ghost_count--;
ring_del(&p->ring);
/* The DelItem below will account for the reference
held by the list. */
} else {
/* This is a ghost object, so we haven't kept a reference
count on it. For it have stayed alive this long
someone else must be keeping a reference to
it. Therefore we need to temporarily give it back a
reference count before calling DelItem below */
Py_INCREF(v);
}
Py_DECREF((PyObject *)p->cache);
p->cache = NULL;
}
if (PyDict_DelItem(self->data, key) < 0) {
PyErr_SetString(PyExc_RuntimeError,
"unexpectedly couldn't remove key in cc_ass_sub");
return -1;
}
return 0;
}
static int
cc_ass_sub(ccobject *self, PyObject *key, PyObject *v)
{
if (!PyString_Check(key)) {
PyErr_Format(PyExc_TypeError,
"cPickleCache key must be a string, not a %s",
key->ob_type->tp_name);
return -1;
}
if (v)
return cc_add_item(self, key, v);
else
return cc_del_item(self, key);
}
static PyMappingMethods cc_as_mapping = {
(inquiry)cc_length, /*mp_length*/
(binaryfunc)cc_subscript, /*mp_subscript*/
(objobjargproc)cc_ass_sub, /*mp_ass_subscript*/
};
static PyObject *
cc_cache_data(ccobject *self, void *context)
{
return PyDict_Copy(self->data);
}
static PyGetSetDef cc_getsets[] = {
{"cache_data", (getter)cc_cache_data},
{NULL}
};
static PyMemberDef cc_members[] = {
{"cache_size", T_INT, offsetof(ccobject, cache_size)},
{"cache_drain_resistance", T_INT,
offsetof(ccobject, cache_drain_resistance)},
{"cache_non_ghost_count", T_INT, offsetof(ccobject, non_ghost_count), RO},
{"cache_klass_count", T_INT, offsetof(ccobject, klass_count), RO},
{NULL}
};
/* This module is compiled as a shared library. Some compilers don't
allow addresses of Python objects defined in other libraries to be
used in static initializers here. The DEFERRED_ADDRESS macro is
used to tag the slots where such addresses appear; the module init
function must fill in the tagged slots at runtime. The argument is
for documentation -- the macro ignores it.
*/
#define DEFERRED_ADDRESS(ADDR) 0
static PyTypeObject Cctype = {
PyObject_HEAD_INIT(DEFERRED_ADDRESS(&PyType_Type))
0, /* ob_size */
"persistent.PickleCache", /* tp_name */
sizeof(ccobject), /* tp_basicsize */
0, /* tp_itemsize */
(destructor)cc_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_compare */
0, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
&cc_as_mapping, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
/* tp_flags */
0, /* tp_doc */
(traverseproc)cc_traverse, /* tp_traverse */
(inquiry)cc_clear, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
cc_methods, /* tp_methods */
cc_members, /* tp_members */
cc_getsets, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
(initproc)cc_init, /* tp_init */
};
void
initcPickleCache(void)
{
PyObject *m;
Cctype.ob_type = &PyType_Type;
Cctype.tp_new = &PyType_GenericNew;
if (PyType_Ready(&Cctype) < 0) {
return;
}
m = Py_InitModule3("cPickleCache", NULL, cPickleCache_doc_string);
capi = (cPersistenceCAPIstruct *)PyCObject_Import(
"persistent.cPersistence", "CAPI");
if (!capi)
return;
capi->percachedel = (percachedelfunc)cc_oid_unreferenced;
py__p_changed = PyString_InternFromString("_p_changed");
if (!py__p_changed)
return;
py__p_deactivate = PyString_InternFromString("_p_deactivate");
if (!py__p_deactivate)
return;
py__p_jar = PyString_InternFromString("_p_jar");
if (!py__p_jar)
return;
py__p_oid = PyString_InternFromString("_p_oid");
if (!py__p_oid)
return;
if (PyModule_AddStringConstant(m, "cache_variant", "stiff/c") < 0)
return;
/* This leaks a reference to Cctype, but it doesn't matter. */
if (PyModule_AddObject(m, "PickleCache", (PyObject *)&Cctype) < 0)
return;
}
ZODB/src/persistent/TimeStamp.c 0000640 0001750 0000144 00000024462 10375443702 016235 0 ustar ctheune users /*****************************************************************************
Copyright (c) 2001, 2004 Zope Corporation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
#include "Python.h"
#include
PyObject *TimeStamp_FromDate(int, int, int, int, int, double);
PyObject *TimeStamp_FromString(const char *);
static char TimeStampModule_doc[] =
"A 64-bit TimeStamp used as a ZODB serial number.\n"
"\n"
"$Id: TimeStamp.c 41650 2006-02-17 22:01:38Z ctheune $\n";
typedef struct {
PyObject_HEAD
unsigned char data[8];
} TimeStamp;
/* The first dimension of the arrays below is non-leapyear / leapyear */
static char month_len[2][12]={
{31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31},
{31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}
};
static short joff[2][12] = {
{0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334},
{0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335}
};
static double gmoff=0;
/* TODO: May be better (faster) to store in a file static. */
#define SCONV ((double)60) / ((double)(1<<16)) / ((double)(1<<16))
static int
leap(int year)
{
return year % 4 == 0 && (year % 100 != 0 || year % 400 == 0);
}
static int
days_in_month(int year, int month)
{
return month_len[leap(year)][month];
}
static double
TimeStamp_yad(int y)
{
double d, s;
y -= 1900;
d = (y - 1) * 365;
if (y > 0) {
s = 1.0;
y -= 1;
} else {
s = -1.0;
y = -y;
}
return d + s * (y / 4 - y / 100 + (y + 300) / 400);
}
static double
TimeStamp_abst(int y, int mo, int d, int m, int s)
{
return (TimeStamp_yad(y) + joff[leap(y)][mo] + d) * 86400 + m * 60 + s;
}
static int
TimeStamp_init_gmoff(void)
{
struct tm *t;
time_t z=0;
t = gmtime(&z);
if (t == NULL) {
PyErr_SetString(PyExc_SystemError, "gmtime failed");
return -1;
}
gmoff = TimeStamp_abst(t->tm_year+1900, t->tm_mon, t->tm_mday - 1,
t->tm_hour * 60 + t->tm_min, t->tm_sec);
return 0;
}
static void
TimeStamp_dealloc(TimeStamp *ts)
{
PyObject_Del(ts);
}
static int
TimeStamp_compare(TimeStamp *v, TimeStamp *w)
{
int cmp = memcmp(v->data, w->data, 8);
if (cmp < 0) return -1;
if (cmp > 0) return 1;
return 0;
}
static long
TimeStamp_hash(TimeStamp *self)
{
register unsigned char *p = (unsigned char *)self->data;
register int len = 8;
register long x = *p << 7;
while (--len >= 0)
x = (1000003*x) ^ *p++;
x ^= 8;
if (x == -1)
x = -2;
return x;
}
typedef struct {
/* TODO: reverse-engineer what's in these things and comment them */
int y;
int m;
int d;
int mi;
} TimeStampParts;
static void
TimeStamp_unpack(TimeStamp *self, TimeStampParts *p)
{
unsigned long v;
v = (self->data[0] * 16777216 + self->data[1] * 65536
+ self->data[2] * 256 + self->data[3]);
p->y = v / 535680 + 1900;
p->m = (v % 535680) / 44640 + 1;
p->d = (v % 44640) / 1440 + 1;
p->mi = v % 1440;
}
static double
TimeStamp_sec(TimeStamp *self)
{
unsigned int v;
v = (self->data[4] * 16777216 + self->data[5] * 65536
+ self->data[6] * 256 + self->data[7]);
return SCONV * v;
}
static PyObject *
TimeStamp_year(TimeStamp *self)
{
TimeStampParts p;
TimeStamp_unpack(self, &p);
return PyInt_FromLong(p.y);
}
static PyObject *
TimeStamp_month(TimeStamp *self)
{
TimeStampParts p;
TimeStamp_unpack(self, &p);
return PyInt_FromLong(p.m);
}
static PyObject *
TimeStamp_day(TimeStamp *self)
{
TimeStampParts p;
TimeStamp_unpack(self, &p);
return PyInt_FromLong(p.d);
}
static PyObject *
TimeStamp_hour(TimeStamp *self)
{
TimeStampParts p;
TimeStamp_unpack(self, &p);
return PyInt_FromLong(p.mi / 60);
}
static PyObject *
TimeStamp_minute(TimeStamp *self)
{
TimeStampParts p;
TimeStamp_unpack(self, &p);
return PyInt_FromLong(p.mi % 60);
}
static PyObject *
TimeStamp_second(TimeStamp *self)
{
return PyFloat_FromDouble(TimeStamp_sec(self));
}
static PyObject *
TimeStamp_timeTime(TimeStamp *self)
{
TimeStampParts p;
TimeStamp_unpack(self, &p);
return PyFloat_FromDouble(TimeStamp_abst(p.y, p.m - 1, p.d - 1, p.mi, 0)
+ TimeStamp_sec(self) - gmoff);
}
static PyObject *
TimeStamp_raw(TimeStamp *self)
{
return PyString_FromStringAndSize((const char*)self->data, 8);
}
static PyObject *
TimeStamp_str(TimeStamp *self)
{
char buf[128];
TimeStampParts p;
int len;
TimeStamp_unpack(self, &p);
len =sprintf(buf, "%4.4d-%2.2d-%2.2d %2.2d:%2.2d:%09.6f",
p.y, p.m, p.d, p.mi / 60, p.mi % 60,
TimeStamp_sec(self));
return PyString_FromStringAndSize(buf, len);
}
static PyObject *
TimeStamp_laterThan(TimeStamp *self, PyObject *obj)
{
TimeStamp *o = NULL;
TimeStampParts p;
unsigned char new[8];
int i;
if (obj->ob_type != self->ob_type) {
PyErr_SetString(PyExc_TypeError, "expected TimeStamp object");
return NULL;
}
o = (TimeStamp *)obj;
if (memcmp(self->data, o->data, 8) > 0) {
Py_INCREF(self);
return (PyObject *)self;
}
memcpy(new, o->data, 8);
for (i = 7; i > 3; i--) {
if (new[i] == 255)
new[i] = 0;
else {
new[i]++;
return TimeStamp_FromString((const char*)new);
}
}
/* All but the first two bytes are the same. Need to increment
the year, month, and day explicitly. */
TimeStamp_unpack(o, &p);
if (p.mi >= 1439) {
p.mi = 0;
if (p.d == month_len[leap(p.y)][p.m - 1]) {
p.d = 1;
if (p.m == 12) {
p.m = 1;
p.y++;
} else
p.m++;
} else
p.d++;
} else
p.mi++;
return TimeStamp_FromDate(p.y, p.m, p.d, p.mi / 60, p.mi % 60, 0);
}
static struct PyMethodDef TimeStamp_methods[] = {
{"year", (PyCFunction)TimeStamp_year, METH_NOARGS},
{"minute", (PyCFunction)TimeStamp_minute, METH_NOARGS},
{"month", (PyCFunction)TimeStamp_month, METH_NOARGS},
{"day", (PyCFunction)TimeStamp_day, METH_NOARGS},
{"hour", (PyCFunction)TimeStamp_hour, METH_NOARGS},
{"second", (PyCFunction)TimeStamp_second, METH_NOARGS},
{"timeTime",(PyCFunction)TimeStamp_timeTime, METH_NOARGS},
{"laterThan", (PyCFunction)TimeStamp_laterThan, METH_O},
{"raw", (PyCFunction)TimeStamp_raw, METH_NOARGS},
{NULL, NULL},
};
static PyTypeObject TimeStamp_type = {
PyObject_HEAD_INIT(NULL)
0,
"persistent.TimeStamp",
sizeof(TimeStamp),
0,
(destructor)TimeStamp_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
(cmpfunc)TimeStamp_compare, /* tp_compare */
(reprfunc)TimeStamp_raw, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
(hashfunc)TimeStamp_hash, /* tp_hash */
0, /* tp_call */
(reprfunc)TimeStamp_str, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
0, /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
TimeStamp_methods, /* tp_methods */
0, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
};
PyObject *
TimeStamp_FromString(const char *buf)
{
/* buf must be exactly 8 characters */
TimeStamp *ts = (TimeStamp *)PyObject_New(TimeStamp, &TimeStamp_type);
memcpy(ts->data, buf, 8);
return (PyObject *)ts;
}
#define CHECK_RANGE(VAR, LO, HI) if ((VAR) < (LO) || (VAR) > (HI)) { \
return PyErr_Format(PyExc_ValueError, \
# VAR " must be between %d and %d: %d", \
(LO), (HI), (VAR)); \
}
PyObject *
TimeStamp_FromDate(int year, int month, int day, int hour, int min,
double sec)
{
TimeStamp *ts = NULL;
int d;
unsigned int v;
if (year < 1900)
return PyErr_Format(PyExc_ValueError,
"year must be greater than 1900: %d", year);
CHECK_RANGE(month, 1, 12);
d = days_in_month(year, month - 1);
if (day < 1 || day > d)
return PyErr_Format(PyExc_ValueError,
"day must be between 1 and %d: %d", d, day);
CHECK_RANGE(hour, 0, 23);
CHECK_RANGE(min, 0, 59);
/* Seconds are allowed to be anything, so chill
If we did want to be pickly, 60 would be a better choice.
if (sec < 0 || sec > 59)
return PyErr_Format(PyExc_ValueError,
"second must be between 0 and 59: %f", sec);
*/
ts = (TimeStamp *)PyObject_New(TimeStamp, &TimeStamp_type);
v = (((year - 1900) * 12 + month - 1) * 31 + day - 1);
v = (v * 24 + hour) * 60 + min;
ts->data[0] = v / 16777216;
ts->data[1] = (v % 16777216) / 65536;
ts->data[2] = (v % 65536) / 256;
ts->data[3] = v % 256;
sec /= SCONV;
v = (unsigned int)sec;
ts->data[4] = v / 16777216;
ts->data[5] = (v % 16777216) / 65536;
ts->data[6] = (v % 65536) / 256;
ts->data[7] = v % 256;
return (PyObject *)ts;
}
PyObject *
TimeStamp_TimeStamp(PyObject *obj, PyObject *args)
{
char *buf = NULL;
int len = 0, y, mo, d, h = 0, m = 0;
double sec = 0;
if (PyArg_ParseTuple(args, "s#:TimeStamp", &buf, &len)) {
if (len != 8) {
PyErr_SetString(PyExc_ValueError, "8-character string expected");
return NULL;
}
return TimeStamp_FromString(buf);
}
PyErr_Clear();
if (!PyArg_ParseTuple(args, "iii|iid", &y, &mo, &d, &h, &m, &sec))
return NULL;
return TimeStamp_FromDate(y, mo, d, h, m, sec);
}
static PyMethodDef TimeStampModule_functions[] = {
{"TimeStamp", TimeStamp_TimeStamp, METH_VARARGS},
{NULL, NULL},
};
void
initTimeStamp(void)
{
PyObject *m;
if (TimeStamp_init_gmoff() < 0)
return;
m = Py_InitModule4("TimeStamp", TimeStampModule_functions,
TimeStampModule_doc, NULL, PYTHON_API_VERSION);
if (m == NULL)
return;
TimeStamp_type.ob_type = &PyType_Type;
TimeStamp_type.tp_getattro = PyObject_GenericGetAttr;
}
ZODB/src/persistent/mapping.py 0000640 0001750 0000144 00000010051 10375443702 016160 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Python implementation of persistent base types
$Id: mapping.py 41650 2006-02-17 22:01:38Z ctheune $"""
import persistent
from UserDict import UserDict
class PersistentMapping(UserDict, persistent.Persistent):
"""A persistent wrapper for mapping objects.
This class allows wrapping of mapping objects so that object
changes are registered. As a side effect, mapping objects may be
subclassed.
A subclass of PersistentMapping or any code that adds new
attributes should not create an attribute named _container. This
is reserved for backwards compatibility reasons.
"""
# UserDict provides all of the mapping behavior. The
# PersistentMapping class is responsible marking the persistent
# state as changed when a method actually changes the state. At
# the mapping API evolves, we may need to add more methods here.
__super_delitem = UserDict.__delitem__
__super_setitem = UserDict.__setitem__
__super_clear = UserDict.clear
__super_update = UserDict.update
__super_setdefault = UserDict.setdefault
__super_pop = UserDict.pop
__super_popitem = UserDict.popitem
def __delitem__(self, key):
self.__super_delitem(key)
self._p_changed = 1
def __setitem__(self, key, v):
self.__super_setitem(key, v)
self._p_changed = 1
def clear(self):
self.__super_clear()
self._p_changed = 1
def update(self, b):
self.__super_update(b)
self._p_changed = 1
def setdefault(self, key, failobj=None):
# We could inline all of UserDict's implementation into the
# method here, but I'd rather not depend at all on the
# implementation in UserDict (simple as it is).
if not self.has_key(key):
self._p_changed = 1
return self.__super_setdefault(key, failobj)
def pop(self, key, *args):
self._p_changed = 1
return self.__super_pop(key, *args)
def popitem(self):
self._p_changed = 1
return self.__super_popitem()
# __iter__ was added in ZODB 3.4.2, but should have been added long
# before. We could inherit from Python's IterableUserDict instead
# (which just adds __iter__ to Python's UserDict), but that class isn't
# documented, and it would add another level of lookup for all the
# other methods.
def __iter__(self):
return iter(self.data)
# If the internal representation of PersistentMapping changes,
# it causes compatibility problems for pickles generated by
# different versions of the code. Compatibility works in both
# directions, because an application may want to share a database
# between applications using different versions of the code.
# Effectively, the original rep is part of the "API." To provide
# full compatibility, the getstate and setstate must read and
# write objects using the old rep.
# As a result, the PersistentMapping must save and restore the
# actual internal dictionary using the name _container.
def __getstate__(self):
state = {}
state.update(self.__dict__)
state['_container'] = state['data']
del state['data']
return state
def __setstate__(self, state):
if state.has_key('_container'):
self.data = state['_container']
del state['_container']
elif not state.has_key('data'):
self.data = {}
self.__dict__.update(state)
ZODB/src/persistent/ring.h 0000640 0001750 0000144 00000005120 10057366465 015273 0 ustar ctheune users /*****************************************************************************
Copyright (c) 2003 Zope Corporation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
/* Support routines for the doubly-linked list of cached objects.
The cache stores a headed, doubly-linked, circular list of persistent
objects, with space for the pointers allocated in the objects themselves.
The cache stores the distinguished head of the list, which is not a valid
persistent object. The other list members are non-ghost persistent
objects, linked in LRU (least-recently used) order.
The r_next pointers traverse the ring starting with the least recently used
object. The r_prev pointers traverse the ring starting with the most
recently used object.
Obscure: While each object is pointed at twice by list pointers (once by
its predecessor's r_next, again by its successor's r_prev), the refcount
on the object is bumped only by 1. This leads to some possibly surprising
sequences of incref and decref code. Note that since the refcount is
bumped at least once, the list does hold a strong reference to each
object in it.
*/
typedef struct CPersistentRing_struct
{
struct CPersistentRing_struct *r_prev;
struct CPersistentRing_struct *r_next;
} CPersistentRing;
/* The list operations here take constant time independent of the
* number of objects in the list:
*/
/* Add elt as the most recently used object. elt must not already be
* in the list, although this isn't checked.
*/
void ring_add(CPersistentRing *ring, CPersistentRing *elt);
/* Remove elt from the list. elt must already be in the list, although
* this isn't checked.
*/
void ring_del(CPersistentRing *elt);
/* elt must already be in the list, although this isn't checked. It's
* unlinked from its current position, and relinked into the list as the
* most recently used object (which is arguably the tail of the list
* instead of the head -- but the name of this function could be argued
* either way). This is equivalent to
*
* ring_del(elt);
* ring_add(ring, elt);
*
* but may be a little quicker.
*/
void ring_move_to_head(CPersistentRing *ring, CPersistentRing *elt);
ZODB/src/persistent/tests/ 0000751 0001750 0000144 00000000000 10424632372 015316 5 ustar ctheune users ZODB/src/persistent/tests/persistenttestbase.py 0000640 0001750 0000144 00000026052 10375443702 021632 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import unittest
from persistent import Persistent
from persistent.interfaces import IPersistent
# Confusing: ZODB doesn't use this file. It appears to be used only
# by Zope3, where it's imported by zope/app/schema/tests/test_wrapper.py.
try:
import zope.interface
except ImportError:
interfaces = False
else:
interfaces = True
class Test(unittest.TestCase):
klass = None # override in subclass
def testSaved(self):
p = self.klass()
p._p_oid = '\0\0\0\0\0\0hi'
dm = DM()
p._p_jar = dm
self.assertEqual(p._p_changed, 0)
self.assertEqual(dm.called, 0)
p.inc()
self.assertEqual(p._p_changed, 1)
self.assertEqual(dm.called, 1)
p.inc()
self.assertEqual(p._p_changed, 1)
self.assertEqual(dm.called, 1)
p._p_deactivate()
self.assertEqual(p._p_changed, 1)
self.assertEqual(dm.called, 1)
p._p_deactivate()
self.assertEqual(p._p_changed, 1)
self.assertEqual(dm.called, 1)
del p._p_changed
# deal with current cPersistence implementation
if p._p_changed != 3:
self.assertEqual(p._p_changed, None)
self.assertEqual(dm.called, 1)
p.inc()
self.assertEqual(p.x, 43)
self.assertEqual(p._p_changed, 1)
self.assertEqual(dm.called, 2)
p._p_changed = 0
self.assertEqual(p._p_changed, 0)
self.assertEqual(dm.called, 2)
self.assertEqual(p.x, 43)
p.inc()
self.assertEqual(p._p_changed, 1)
self.assertEqual(dm.called, 3)
def testUnsaved(self):
p = self.klass()
self.assertEqual(p.x, 0)
self.assertEqual(p._p_changed, 0)
self.assertEqual(p._p_jar, None)
self.assertEqual(p._p_oid, None)
p.inc()
p.inc()
self.assertEqual(p.x, 2)
self.assertEqual(p._p_changed, 0)
p._p_deactivate()
self.assertEqual(p._p_changed, 0)
p._p_changed = 1
self.assertEqual(p._p_changed, 0)
p._p_deactivate()
self.assertEqual(p._p_changed, 0)
del p._p_changed
self.assertEqual(p._p_changed, 0)
if self.has_dict:
self.failUnless(p.__dict__)
self.assertEqual(p.x, 2)
def testState(self):
p = self.klass()
self.assertEqual(p.__getstate__(), {'x': 0})
self.assertEqual(p._p_changed, 0)
p.__setstate__({'x':5})
self.assertEqual(p._p_changed, 0)
if self.has_dict:
p._v_foo = 2
self.assertEqual(p.__getstate__(), {'x': 5})
self.assertEqual(p._p_changed, 0)
def testSetStateSerial(self):
p = self.klass()
p._p_serial = '00000012'
p.__setstate__(p.__getstate__())
self.assertEqual(p._p_serial, '00000012')
def testDirectChanged(self):
p = self.klass()
p._p_oid = 1
dm = DM()
p._p_jar = dm
self.assertEqual(p._p_changed, 0)
self.assertEqual(dm.called, 0)
p._p_changed = 1
self.assertEqual(dm.called, 1)
def testGhostChanged(self):
# If an object is a ghost and its _p_changed is set to True (any
# true value), it should activate (unghostify) the object. This
# behavior is new in ZODB 3.6; before then, an attempt to do
# "ghost._p_changed = True" was ignored.
p = self.klass()
p._p_oid = 1
dm = DM()
p._p_jar = dm
p._p_deactivate()
self.assertEqual(p._p_changed, None)
p._p_changed = True
self.assertEqual(p._p_changed, 1)
def testRegistrationFailure(self):
p = self.klass()
p._p_oid = 1
dm = BrokenDM()
p._p_jar = dm
self.assertEqual(p._p_changed, 0)
self.assertEqual(dm.called, 0)
try:
p._p_changed = 1
except NotImplementedError:
pass
else:
raise AssertionError("Exception not propagated")
self.assertEqual(dm.called, 1)
self.assertEqual(p._p_changed, 0)
def testLoadFailure(self):
p = self.klass()
p._p_oid = 1
dm = BrokenDM()
p._p_jar = dm
p._p_deactivate() # make it a ghost
try:
p._p_activate()
except NotImplementedError:
pass
else:
raise AssertionError("Exception not propagated")
self.assertEqual(p._p_changed, None)
def testActivate(self):
p = self.klass()
dm = DM()
p._p_oid = 1
p._p_jar = dm
p._p_changed = 0
p._p_deactivate()
# Unsure: does this really test the activate method?
p._p_activate()
self.assertEqual(p._p_changed, 0)
self.assertEqual(p.x, 42)
def testDeactivate(self):
p = self.klass()
dm = DM()
p._p_oid = 1
p._p_deactivate() # this deactive has no effect
self.assertEqual(p._p_changed, 0)
p._p_jar = dm
p._p_changed = 0
p._p_deactivate()
self.assertEqual(p._p_changed, None)
p._p_activate()
self.assertEqual(p._p_changed, 0)
self.assertEqual(p.x, 42)
if interfaces:
def testInterface(self):
self.assert_(IPersistent.implementedBy(Persistent),
"%s does not implement IPersistent" % Persistent)
p = Persistent()
self.assert_(IPersistent.providedBy(p),
"%s does not implement IPersistent" % p)
self.assert_(IPersistent.implementedBy(P),
"%s does not implement IPersistent" % P)
p = self.klass()
self.assert_(IPersistent.providedBy(p),
"%s does not implement IPersistent" % p)
def testDataManagerAndAttributes(self):
# Test to cover an odd bug where the instance __dict__ was
# set at the same location as the data manager in the C type.
p = P()
p.inc()
p.inc()
self.assert_('x' in p.__dict__)
self.assert_(p._p_jar is None)
def testMultipleInheritance(self):
# make sure it is possible to inherit from two different
# subclasses of persistent.
class A(Persistent):
pass
class B(Persistent):
pass
class C(A, B):
pass
class D(object):
pass
class E(D, B):
pass
def testMultipleMeta(self):
# make sure it's possible to define persistent classes
# with a base whose metaclass is different
class alternateMeta(type):
pass
class alternate(object):
__metaclass__ = alternateMeta
class mixedMeta(alternateMeta, type):
pass
class mixed(alternate,Persistent):
__metaclass__ = mixedMeta
def testSlots(self):
# Verify that Persistent classes behave the same way
# as pure Python objects where '__slots__' and '__dict__'
# are concerned.
class noDict(object):
__slots__ = ['foo']
class shouldHaveDict(noDict):
pass
class p_noDict(Persistent):
__slots__ = ['foo']
class p_shouldHaveDict(p_noDict):
pass
self.assertEqual(noDict.__dictoffset__, 0)
self.assertEqual(p_noDict.__dictoffset__, 0)
self.assert_(shouldHaveDict.__dictoffset__ <> 0)
self.assert_(p_shouldHaveDict.__dictoffset__ <> 0)
def testBasicTypeStructure(self):
# test that a persistent class has a sane C type structure
# use P (defined below) as simplest example
self.assertEqual(Persistent.__dictoffset__, 0)
self.assertEqual(Persistent.__weakrefoffset__, 0)
self.assert_(Persistent.__basicsize__ > object.__basicsize__)
self.assert_(P.__dictoffset__)
self.assert_(P.__weakrefoffset__)
self.assert_(P.__dictoffset__ < P.__weakrefoffset__)
self.assert_(P.__basicsize__ > Persistent.__basicsize__)
# Unsure: Can anyone defend/explain the test below? The tests classes defined
# here don't define __call__, so this weird test will always pass, but to what
# end? If a klass is given that happens to define __call__, the test *may*
# mysteriously fail. Who cares?
## def testDeactivateErrors(self):
## p = self.klass()
## p._p_oid = '\0\0\0\0\0\0hi'
## dm = DM()
## p._p_jar = dm
## def typeerr(*args, **kwargs):
## self.assertRaises(TypeError, p, *args, **kwargs)
## typeerr(1)
## typeerr(1, 2)
## typeerr(spam=1)
## typeerr(spam=1, force=1)
## p._p_changed = True
## class Err(object):
## def __nonzero__(self):
## raise RuntimeError
## typeerr(force=Err())
class P(Persistent):
def __init__(self):
self.x = 0
def inc(self):
self.x += 1
class P2(P):
def __getstate__(self):
return 42
def __setstate__(self, v):
self.v = v
class B(Persistent):
__slots__ = ["x", "_p_serial"]
def __init__(self):
self.x = 0
def inc(self):
self.x += 1
def __getstate__(self):
return {'x': self.x}
def __setstate__(self, state):
self.x = state['x']
class DM:
def __init__(self):
self.called = 0
def register(self, ob):
self.called += 1
def setstate(self, ob):
ob.__setstate__({'x': 42})
class BrokenDM(DM):
def register(self,ob):
self.called += 1
raise NotImplementedError
def setstate(self,ob):
raise NotImplementedError
class PersistentTest(Test):
klass = P
has_dict = 1
def testPicklable(self):
import pickle
p = self.klass()
p.inc()
p2 = pickle.loads(pickle.dumps(p))
self.assertEqual(p2.__class__, self.klass)
# verify that the inc is reflected:
self.assertEqual(p2.x, p.x)
# This assertion would be invalid. Interfaces
# are compared by identity and copying doesn't
# preserve identity. We would get false negatives due
# to the differing identities of the original and copied
# PersistentInterface:
# self.assertEqual(p2.__dict__, p.__dict__)
def testPicklableWCustomState(self):
import pickle
p = P2()
p2 = pickle.loads(pickle.dumps(p))
self.assertEqual(p2.__class__, P2);
self.assertEqual(p2.__dict__, {'v': 42})
class BasePersistentTest(Test):
klass = B
has_dict = 0
ZODB/src/persistent/tests/test_PickleCache.py 0000640 0001750 0000144 00000002631 10225135606 021061 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Unit tests for PickleCache
$Id: test_PickleCache.py 29896 2005-04-07 04:48:06Z tim_one $
"""
class DummyConnection:
def setklassstate(self, obj):
"""Method used by PickleCache."""
def test_delitem():
"""
>>> from persistent import PickleCache
>>> conn = DummyConnection()
>>> cache = PickleCache(conn)
>>> del cache['']
Traceback (most recent call last):
...
KeyError: ''
>>> from persistent import Persistent
>>> p = Persistent()
>>> p._p_oid = 'foo'
>>> p._p_jar = conn
>>> cache['foo'] = p
>>> del cache['foo']
"""
from zope.testing.doctest import DocTestSuite
import unittest
def test_suite():
return unittest.TestSuite((
DocTestSuite(),
))
if __name__ == '__main__':
unittest.main()
ZODB/src/persistent/tests/test_list.py 0000640 0001750 0000144 00000013376 10057366465 017725 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Test the list interface to PersistentList
"""
import unittest
from persistent.list import PersistentList
l0 = []
l1 = [0]
l2 = [0, 1]
class TestPList(unittest.TestCase):
def testTheWorld(self):
# Test constructors
u = PersistentList()
u0 = PersistentList(l0)
u1 = PersistentList(l1)
u2 = PersistentList(l2)
uu = PersistentList(u)
uu0 = PersistentList(u0)
uu1 = PersistentList(u1)
uu2 = PersistentList(u2)
v = PersistentList(tuple(u))
class OtherList:
def __init__(self, initlist):
self.__data = initlist
def __len__(self):
return len(self.__data)
def __getitem__(self, i):
return self.__data[i]
v0 = PersistentList(OtherList(u0))
vv = PersistentList("this is also a sequence")
# Test __repr__
eq = self.assertEqual
eq(str(u0), str(l0), "str(u0) == str(l0)")
eq(repr(u1), repr(l1), "repr(u1) == repr(l1)")
eq(`u2`, `l2`, "`u2` == `l2`")
# Test __cmp__ and __len__
def mycmp(a, b):
r = cmp(a, b)
if r < 0: return -1
if r > 0: return 1
return r
all = [l0, l1, l2, u, u0, u1, u2, uu, uu0, uu1, uu2]
for a in all:
for b in all:
eq(mycmp(a, b), mycmp(len(a), len(b)),
"mycmp(a, b) == mycmp(len(a), len(b))")
# Test __getitem__
for i in range(len(u2)):
eq(u2[i], i, "u2[i] == i")
# Test __setitem__
uu2[0] = 0
uu2[1] = 100
try:
uu2[2] = 200
except IndexError:
pass
else:
raise TestFailed("uu2[2] shouldn't be assignable")
# Test __delitem__
del uu2[1]
del uu2[0]
try:
del uu2[0]
except IndexError:
pass
else:
raise TestFailed("uu2[0] shouldn't be deletable")
# Test __getslice__
for i in range(-3, 4):
eq(u2[:i], l2[:i], "u2[:i] == l2[:i]")
eq(u2[i:], l2[i:], "u2[i:] == l2[i:]")
for j in range(-3, 4):
eq(u2[i:j], l2[i:j], "u2[i:j] == l2[i:j]")
# Test __setslice__
for i in range(-3, 4):
u2[:i] = l2[:i]
eq(u2, l2, "u2 == l2")
u2[i:] = l2[i:]
eq(u2, l2, "u2 == l2")
for j in range(-3, 4):
u2[i:j] = l2[i:j]
eq(u2, l2, "u2 == l2")
uu2 = u2[:]
uu2[:0] = [-2, -1]
eq(uu2, [-2, -1, 0, 1], "uu2 == [-2, -1, 0, 1]")
uu2[0:] = []
eq(uu2, [], "uu2 == []")
# Test __contains__
for i in u2:
self.failUnless(i in u2, "i in u2")
for i in min(u2)-1, max(u2)+1:
self.failUnless(i not in u2, "i not in u2")
# Test __delslice__
uu2 = u2[:]
del uu2[1:2]
del uu2[0:1]
eq(uu2, [], "uu2 == []")
uu2 = u2[:]
del uu2[1:]
del uu2[:1]
eq(uu2, [], "uu2 == []")
# Test __add__, __radd__, __mul__ and __rmul__
#self.failUnless(u1 + [] == [] + u1 == u1, "u1 + [] == [] + u1 == u1")
self.failUnless(u1 + [1] == u2, "u1 + [1] == u2")
#self.failUnless([-1] + u1 == [-1, 0], "[-1] + u1 == [-1, 0]")
self.failUnless(u2 == u2*1 == 1*u2, "u2 == u2*1 == 1*u2")
self.failUnless(u2+u2 == u2*2 == 2*u2, "u2+u2 == u2*2 == 2*u2")
self.failUnless(u2+u2+u2 == u2*3 == 3*u2, "u2+u2+u2 == u2*3 == 3*u2")
# Test append
u = u1[:]
u.append(1)
eq(u, u2, "u == u2")
# Test insert
u = u2[:]
u.insert(0, -1)
eq(u, [-1, 0, 1], "u == [-1, 0, 1]")
# Test pop
u = PersistentList([0, -1, 1])
u.pop()
eq(u, [0, -1], "u == [0, -1]")
u.pop(0)
eq(u, [-1], "u == [-1]")
# Test remove
u = u2[:]
u.remove(1)
eq(u, u1, "u == u1")
# Test count
u = u2*3
eq(u.count(0), 3, "u.count(0) == 3")
eq(u.count(1), 3, "u.count(1) == 3")
eq(u.count(2), 0, "u.count(2) == 0")
# Test index
eq(u2.index(0), 0, "u2.index(0) == 0")
eq(u2.index(1), 1, "u2.index(1) == 1")
try:
u2.index(2)
except ValueError:
pass
else:
raise TestFailed("expected ValueError")
# Test reverse
u = u2[:]
u.reverse()
eq(u, [1, 0], "u == [1, 0]")
u.reverse()
eq(u, u2, "u == u2")
# Test sort
u = PersistentList([1, 0])
u.sort()
eq(u, u2, "u == u2")
# Test extend
u = u1[:]
u.extend(u2)
eq(u, u1 + u2, "u == u1 + u2")
# Test iadd
u = u1[:]
u += u2
eq(u, u1 + u2, "u == u1 + u2")
# Test imul
u = u1[:]
u *= 3
eq(u, u1 + u1 + u1, "u == u1 + u1 + u1")
def test_suite():
return unittest.makeSuite(TestPList)
if __name__ == "__main__":
loader = unittest.TestLoader()
unittest.main(testLoader=loader)
ZODB/src/persistent/tests/test_mapping.py 0000640 0001750 0000144 00000010326 10375443702 020366 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2005 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Test the mapping interface to PersistentMapping
"""
import unittest
from persistent.mapping import PersistentMapping
l0 = {}
l1 = {0:0}
l2 = {0:0, 1:1}
class TestPMapping(unittest.TestCase):
def testTheWorld(self):
# Test constructors
u = PersistentMapping()
u0 = PersistentMapping(l0)
u1 = PersistentMapping(l1)
u2 = PersistentMapping(l2)
uu = PersistentMapping(u)
uu0 = PersistentMapping(u0)
uu1 = PersistentMapping(u1)
uu2 = PersistentMapping(u2)
class OtherMapping:
def __init__(self, initmapping):
self.__data = initmapping
def items(self):
return self.__data.items()
v0 = PersistentMapping(OtherMapping(u0))
vv = PersistentMapping([(0, 0), (1, 1)])
# Test __repr__
eq = self.assertEqual
eq(str(u0), str(l0), "str(u0) == str(l0)")
eq(repr(u1), repr(l1), "repr(u1) == repr(l1)")
eq(`u2`, `l2`, "`u2` == `l2`")
# Test __cmp__ and __len__
def mycmp(a, b):
r = cmp(a, b)
if r < 0: return -1
if r > 0: return 1
return r
all = [l0, l1, l2, u, u0, u1, u2, uu, uu0, uu1, uu2]
for a in all:
for b in all:
eq(mycmp(a, b), mycmp(len(a), len(b)),
"mycmp(a, b) == mycmp(len(a), len(b))")
# Test __getitem__
for i in range(len(u2)):
eq(u2[i], i, "u2[i] == i")
# Test get
for i in range(len(u2)):
eq(u2.get(i), i, "u2.get(i) == i")
eq(u2.get(i, 5), i, "u2.get(i, 5) == i")
for i in min(u2)-1, max(u2)+1:
eq(u2.get(i), None, "u2.get(i) == None")
eq(u2.get(i, 5), 5, "u2.get(i, 5) == 5")
# Test __setitem__
uu2[0] = 0
uu2[1] = 100
uu2[2] = 200
# Test __delitem__
del uu2[1]
del uu2[0]
try:
del uu2[0]
except KeyError:
pass
else:
raise TestFailed("uu2[0] shouldn't be deletable")
# Test __contains__
for i in u2:
self.failUnless(i in u2, "i in u2")
for i in min(u2)-1, max(u2)+1:
self.failUnless(i not in u2, "i not in u2")
# Test update
l = {"a":"b"}
u = PersistentMapping(l)
u.update(u2)
for i in u:
self.failUnless(i in l or i in u2, "i in l or i in u2")
for i in l:
self.failUnless(i in u, "i in u")
for i in u2:
self.failUnless(i in u, "i in u")
# Test setdefault
x = u2.setdefault(0, 5)
eq(x, 0, "u2.setdefault(0, 5) == 0")
x = u2.setdefault(5, 5)
eq(x, 5, "u2.setdefault(5, 5) == 5")
self.failUnless(5 in u2, "5 in u2")
# Test pop
x = u2.pop(1)
eq(x, 1, "u2.pop(1) == 1")
self.failUnless(1 not in u2, "1 not in u2")
try:
u2.pop(1)
except KeyError:
pass
else:
raise TestFailed("1 should not be poppable from u2")
x = u2.pop(1, 7)
eq(x, 7, "u2.pop(1, 7) == 7")
# Test popitem
items = u2.items()
key, value = u2.popitem()
self.failUnless((key, value) in items, "key, value in items")
self.failUnless(key not in u2, "key not in u2")
# Test clear
u2.clear()
eq(u2, {}, "u2 == {}")
def test_suite():
return unittest.makeSuite(TestPMapping)
if __name__ == "__main__":
loader = unittest.TestLoader()
unittest.main(testLoader=loader)
ZODB/src/persistent/tests/test_persistent.py 0000640 0001750 0000144 00000001603 10225135606 021124 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
from zope.testing import doctest
from persistent import Persistent
class P(Persistent):
def __init__(self):
self.x = 0
def inc(self):
self.x += 1
def test_suite():
return doctest.DocFileSuite("persistent.txt", globs={"P": P})
ZODB/src/persistent/tests/__init__.py 0000640 0001750 0000144 00000000012 10051215041 017402 0 ustar ctheune users # package
ZODB/src/persistent/tests/test_overriding_attrs.py 0000640 0001750 0000144 00000026007 10305413212 022305 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Overriding attr methods
This module tests and documents, through example, overriding attribute
access methods.
$Id: test_overriding_attrs.py 38214 2005-08-31 20:43:54Z mj $
"""
from persistent import Persistent
import transaction
from ZODB.tests.util import DB
class SampleOverridingGetattr(Persistent):
"""Example of overriding __getattr__
"""
def __getattr__(self, name):
"""Get attributes that can't be gotten the usual way
The __getattr__ method works pretty much the same for persistent
classes as it does for other classes. No special handling is
needed. If an object is a ghost, then it will be activated before
__getattr__ is called.
In this example, our objects returns a tuple with the attribute
name, converted to upper case and the value of _p_changed, for any
attribute that isn't handled by the default machinery.
>>> o = SampleOverridingGetattr()
>>> o._p_changed
False
>>> o._p_oid
>>> o._p_jar
>>> o.spam
('SPAM', False)
>>> o.spam = 1
>>> o.spam
1
We'll save the object, so it can be deactivated:
>>> db = DB()
>>> conn = db.open()
>>> conn.root()['o'] = o
>>> transaction.commit()
>>> o._p_deactivate()
>>> o._p_changed
And now, if we ask for an attribute it doesn't have,
>>> o.eggs
('EGGS', False)
And we see that the object was activated before calling the
__getattr__ method.
We always close databases after we use them:
>>> db.close()
"""
# Don't pretend we have any special attributes.
if name.startswith("__") and name.endswrith("__"):
raise AttributeError(name)
else:
return name.upper(), self._p_changed
class SampleOverridingGetattributeSetattrAndDelattr(Persistent):
"""Example of overriding __getattribute__, __setattr__, and __delattr__
In this example, we'll provide an example that shows how to
override the __getattribute__, __setattr__, and __delattr__
methods. We'll create a class that stores it's attributes in a
secret dictionary within it's instance dictionary.
The class will have the policy that variables with names starting
with 'tmp_' will be volatile.
"""
def __init__(self, **kw):
self.__dict__['__secret__'] = kw.copy()
def __getattribute__(self, name):
"""Get an attribute value
The __getattribute__ method is called for all attribute
accesses. It overrides the attribute access support inherited
from Persistent.
Our sample class let's us provide initial values as keyword
arguments to the constructor:
>>> o = SampleOverridingGetattributeSetattrAndDelattr(x=1)
>>> o._p_changed
0
>>> o._p_oid
>>> o._p_jar
>>> o.x
1
>>> o.y
Traceback (most recent call last):
...
AttributeError: y
Next, we'll save the object in a database so that we can
deactivate it:
>>> db = DB()
>>> conn = db.open()
>>> conn.root()['o'] = o
>>> transaction.commit()
>>> o._p_deactivate()
>>> o._p_changed
And we'll get some data:
>>> o.x
1
which activates the object:
>>> o._p_changed
0
It works for missing attribes too:
>>> o._p_deactivate()
>>> o._p_changed
>>> o.y
Traceback (most recent call last):
...
AttributeError: y
>>> o._p_changed
0
See the very important note in the comment below!
We always close databases after we use them:
>>> db.close()
"""
#################################################################
# IMPORTANT! READ THIS! 8->
#
# We *always* give Persistent a chance first.
# Persistent handles certain special attributes, like _p_
# attributes. In particular, the base class handles __dict__
# and __class__.
#
# We call _p_getattr. If it returns True, then we have to
# use Persistent.__getattribute__ to get the value.
#
#################################################################
if Persistent._p_getattr(self, name):
return Persistent.__getattribute__(self, name)
# Data should be in our secret dictionary:
secret = self.__dict__['__secret__']
if name in secret:
return secret[name]
# Maybe it's a method:
meth = getattr(self.__class__, name, None)
if meth is None:
raise AttributeError(name)
return meth.__get__(self, self.__class__)
def __setattr__(self, name, value):
"""Set an attribute value
The __setattr__ method is called for all attribute
assignments. It overrides the attribute assignment support
inherited from Persistent.
Implementors of __setattr__ methods:
1. Must call Persistent._p_setattr first to allow it
to handle some attributes and to make sure that the object
is activated if necessary, and
2. Must set _p_changed to mark objects as changed.
See the comments in the source below.
>>> o = SampleOverridingGetattributeSetattrAndDelattr()
>>> o._p_changed
0
>>> o._p_oid
>>> o._p_jar
>>> o.x
Traceback (most recent call last):
...
AttributeError: x
>>> o.x = 1
>>> o.x
1
Because the implementation doesn't store attributes directly
in the instance dictionary, we don't have a key for the attribute:
>>> 'x' in o.__dict__
False
Next, we'll save the object in a database so that we can
deactivate it:
>>> db = DB()
>>> conn = db.open()
>>> conn.root()['o'] = o
>>> transaction.commit()
>>> o._p_deactivate()
>>> o._p_changed
We'll modify an attribute
>>> o.y = 2
>>> o.y
2
which reactivates it, and markes it as modified, because our
implementation marked it as modified:
>>> o._p_changed
1
Now, if commit:
>>> transaction.commit()
>>> o._p_changed
0
And deactivate the object:
>>> o._p_deactivate()
>>> o._p_changed
and then set a variable with a name starting with 'tmp_',
The object will be activated, but not marked as modified,
because our __setattr__ implementation doesn't mark the
object as changed if the name starts with 'tmp_':
>>> o.tmp_foo = 3
>>> o._p_changed
0
>>> o.tmp_foo
3
We always close databases after we use them:
>>> db.close()
"""
#################################################################
# IMPORTANT! READ THIS! 8->
#
# We *always* give Persistent a chance first.
# Persistent handles certain special attributes, like _p_
# attributes.
#
# We call _p_setattr. If it returns True, then we are done.
# It has already set the attribute.
#
#################################################################
if Persistent._p_setattr(self, name, value):
return
self.__dict__['__secret__'][name] = value
if not name.startswith('tmp_'):
self._p_changed = 1
def __delattr__(self, name):
"""Delete an attribute value
The __delattr__ method is called for all attribute
deletions. It overrides the attribute deletion support
inherited from Persistent.
Implementors of __delattr__ methods:
1. Must call Persistent._p_delattr first to allow it
to handle some attributes and to make sure that the object
is activated if necessary, and
2. Must set _p_changed to mark objects as changed.
See the comments in the source below.
>>> o = SampleOverridingGetattributeSetattrAndDelattr(
... x=1, y=2, tmp_z=3)
>>> o._p_changed
0
>>> o._p_oid
>>> o._p_jar
>>> o.x
1
>>> del o.x
>>> o.x
Traceback (most recent call last):
...
AttributeError: x
Next, we'll save the object in a database so that we can
deactivate it:
>>> db = DB()
>>> conn = db.open()
>>> conn.root()['o'] = o
>>> transaction.commit()
>>> o._p_deactivate()
>>> o._p_changed
If we delete an attribute:
>>> del o.y
The object is activated. It is also marked as changed because
our implementation marked it as changed.
>>> o._p_changed
1
>>> o.y
Traceback (most recent call last):
...
AttributeError: y
>>> o.tmp_z
3
Now, if commit:
>>> transaction.commit()
>>> o._p_changed
0
And deactivate the object:
>>> o._p_deactivate()
>>> o._p_changed
and then delete a variable with a name starting with 'tmp_',
The object will be activated, but not marked as modified,
because our __delattr__ implementation doesn't mark the
object as changed if the name starts with 'tmp_':
>>> del o.tmp_z
>>> o._p_changed
0
>>> o.tmp_z
Traceback (most recent call last):
...
AttributeError: tmp_z
We always close databases after we use them:
>>> db.close()
"""
#################################################################
# IMPORTANT! READ THIS! 8->
#
# We *always* give Persistent a chance first.
# Persistent handles certain special attributes, like _p_
# attributes.
#
# We call _p_delattr. If it returns True, then we are done.
# It has already deleted the attribute.
#
#################################################################
if Persistent._p_delattr(self, name):
return
del self.__dict__['__secret__'][name]
if not name.startswith('tmp_'):
self._p_changed = 1
def test_suite():
from zope.testing.doctest import DocTestSuite
return DocTestSuite()
ZODB/src/persistent/tests/testPersistent.py 0000640 0001750 0000144 00000017016 10214421637 020732 0 ustar ctheune users #############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import pickle
import time
import unittest
from persistent import Persistent, GHOST, UPTODATE, CHANGED, STICKY
from persistent.cPickleCache import PickleCache
from persistent.TimeStamp import TimeStamp
from ZODB.utils import p64
class Jar(object):
"""Testing stub for _p_jar attribute."""
def __init__(self):
self.cache = PickleCache(self)
self.oid = 1
self.registered = {}
def add(self, obj):
obj._p_oid = p64(self.oid)
self.oid += 1
obj._p_jar = self
self.cache[obj._p_oid] = obj
def close(self):
pass
# the following methods must be implemented to be a jar
def setklassstate(self):
# I don't know what this method does, but the pickle cache
# constructor calls it.
pass
def register(self, obj):
self.registered[obj] = 1
def setstate(self, obj):
# Trivial setstate() implementation that just re-initializes
# the object. This isn't what setstate() is supposed to do,
# but it suffices for the tests.
obj.__class__.__init__(obj)
class P(Persistent):
pass
class H1(Persistent):
def __init__(self):
self.n = 0
def __getattr__(self, attr):
self.n += 1
return self.n
class H2(Persistent):
def __init__(self):
self.n = 0
def __getattribute__(self, attr):
supergetattr = super(H2, self).__getattribute__
try:
return supergetattr(attr)
except AttributeError:
n = supergetattr("n")
self.n = n + 1
return n + 1
class PersistenceTest(unittest.TestCase):
def setUp(self):
self.jar = Jar()
def tearDown(self):
self.jar.close()
def testOidAndJarAttrs(self):
obj = P()
self.assertEqual(obj._p_oid, None)
obj._p_oid = 12
self.assertEqual(obj._p_oid, 12)
del obj._p_oid
self.jar.add(obj)
# Can't change oid of cache object.
def deloid():
del obj._p_oid
self.assertRaises(ValueError, deloid)
def setoid():
obj._p_oid = 12
self.assertRaises(ValueError, setoid)
def deloid():
del obj._p_jar
self.assertRaises(ValueError, deloid)
def setoid():
obj._p_jar = 12
self.assertRaises(ValueError, setoid)
def testChangedAndState(self):
obj = P()
self.jar.add(obj)
# The value returned for _p_changed can be one of:
# 0 -- it is not changed
# 1 -- it is changed
# None -- it is a ghost
obj.x = 1
self.assertEqual(obj._p_changed, 1)
self.assertEqual(obj._p_state, CHANGED)
self.assert_(obj in self.jar.registered)
obj._p_changed = 0
self.assertEqual(obj._p_changed, 0)
self.assertEqual(obj._p_state, UPTODATE)
self.jar.registered.clear()
obj._p_changed = 1
self.assertEqual(obj._p_changed, 1)
self.assertEqual(obj._p_state, CHANGED)
self.assert_(obj in self.jar.registered)
# setting obj._p_changed to None ghostifies if the
# object is in the up-to-date state, but not otherwise.
obj._p_changed = None
self.assertEqual(obj._p_changed, 1)
self.assertEqual(obj._p_state, CHANGED)
obj._p_changed = 0
# Now it's a ghost.
obj._p_changed = None
self.assertEqual(obj._p_changed, None)
self.assertEqual(obj._p_state, GHOST)
obj = P()
self.jar.add(obj)
obj._p_changed = 1
# You can transition directly from modified to ghost if
# you delete the _p_changed attribute.
del obj._p_changed
self.assertEqual(obj._p_changed, None)
self.assertEqual(obj._p_state, GHOST)
def testStateReadonly(self):
# make sure we can't write to _p_state; we don't want yet
# another way to change state!
obj = P()
def setstate(value):
obj._p_state = value
self.assertRaises(TypeError, setstate, GHOST)
self.assertRaises(TypeError, setstate, UPTODATE)
self.assertRaises(TypeError, setstate, CHANGED)
self.assertRaises(TypeError, setstate, STICKY)
def testInvalidate(self):
obj = P()
self.jar.add(obj)
self.assertEqual(obj._p_changed, 0)
self.assertEqual(obj._p_state, UPTODATE)
obj._p_invalidate()
self.assertEqual(obj._p_changed, None)
self.assertEqual(obj._p_state, GHOST)
obj._p_activate()
obj.x = 1
obj._p_invalidate()
self.assertEqual(obj._p_changed, None)
self.assertEqual(obj._p_state, GHOST)
def testSerial(self):
noserial = "\000" * 8
obj = P()
self.assertEqual(obj._p_serial, noserial)
def set(val):
obj._p_serial = val
self.assertRaises(ValueError, set, 1)
self.assertRaises(ValueError, set, "0123")
self.assertRaises(ValueError, set, "012345678")
self.assertRaises(ValueError, set, u"01234567")
obj._p_serial = "01234567"
del obj._p_serial
self.assertEqual(obj._p_serial, noserial)
def testMTime(self):
obj = P()
self.assertEqual(obj._p_mtime, None)
t = int(time.time())
ts = TimeStamp(*time.gmtime(t)[:6])
obj._p_serial = repr(ts)
self.assertEqual(obj._p_mtime, t)
self.assert_(isinstance(obj._p_mtime, float))
def testPicklable(self):
obj = P()
obj.attr = "test"
s = pickle.dumps(obj)
obj2 = pickle.loads(s)
self.assertEqual(obj.attr, obj2.attr)
def testGetattr(self):
obj = H1()
self.assertEqual(obj.larry, 1)
self.assertEqual(obj.curly, 2)
self.assertEqual(obj.moe, 3)
self.jar.add(obj)
obj._p_deactivate()
# The simple Jar used for testing re-initializes the object.
self.assertEqual(obj.larry, 1)
# The getattr hook modified the object, so it should now be
# in the changed state.
self.assertEqual(obj._p_changed, 1)
self.assertEqual(obj._p_state, CHANGED)
self.assertEqual(obj.curly, 2)
self.assertEqual(obj.moe, 3)
def testGetattribute(self):
obj = H2()
self.assertEqual(obj.larry, 1)
self.assertEqual(obj.curly, 2)
self.assertEqual(obj.moe, 3)
self.jar.add(obj)
obj._p_deactivate()
# The simple Jar used for testing re-initializes the object.
self.assertEqual(obj.larry, 1)
# The getattr hook modified the object, so it should now be
# in the changed state.
self.assertEqual(obj._p_changed, 1)
self.assertEqual(obj._p_state, CHANGED)
self.assertEqual(obj.curly, 2)
self.assertEqual(obj.moe, 3)
# TODO: Need to decide how __setattr__ and __delattr__ should work,
# then write tests.
def test_suite():
return unittest.makeSuite(PersistenceTest)
ZODB/src/persistent/tests/test_wref.py 0000640 0001750 0000144 00000001550 10225135606 017670 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""
$Id: test_wref.py 29896 2005-04-07 04:48:06Z tim_one $
"""
import unittest
from zope.testing.doctest import DocTestSuite
def test_suite():
return DocTestSuite('persistent.wref')
if __name__ == '__main__':
unittest.main()
ZODB/src/persistent/tests/test_pickle.py 0000640 0001750 0000144 00000014345 10225135606 020202 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Basic pickling tests
$Id: test_pickle.py 29896 2005-04-07 04:48:06Z tim_one $
"""
from persistent import Persistent
import pickle
def print_dict(d):
d = d.items()
d.sort()
print '{%s}' % (', '.join(
[('%r: %r' % (k, v)) for (k, v) in d]
))
def cmpattrs(self, other, *attrs):
for attr in attrs:
if attr[:3] in ('_v_', '_p_'):
continue
c = cmp(getattr(self, attr, None), getattr(other, attr, None))
if c:
return c
return 0
class Simple(Persistent):
def __init__(self, name, **kw):
self.__name__ = name
self.__dict__.update(kw)
self._v_favorite_color = 'blue'
self._p_foo = 'bar'
def __cmp__(self, other):
return cmpattrs(self, other, '__class__', *(self.__dict__.keys()))
def test_basic_pickling():
"""
>>> x = Simple('x', aaa=1, bbb='foo')
>>> print_dict(x.__getstate__())
{'__name__': 'x', 'aaa': 1, 'bbb': 'foo'}
>>> f, (c,), state = x.__reduce__()
>>> f.__name__
'__newobj__'
>>> f.__module__
'copy_reg'
>>> c.__name__
'Simple'
>>> print_dict(state)
{'__name__': 'x', 'aaa': 1, 'bbb': 'foo'}
>>> pickle.loads(pickle.dumps(x)) == x
1
>>> pickle.loads(pickle.dumps(x, 0)) == x
1
>>> pickle.loads(pickle.dumps(x, 1)) == x
1
>>> pickle.loads(pickle.dumps(x, 2)) == x
1
>>> x.__setstate__({'z': 1})
>>> x.__dict__
{'z': 1}
"""
class Custom(Simple):
def __new__(cls, x, y):
r = Persistent.__new__(cls)
r.x, r.y = x, y
return r
def __init__(self, x, y):
self.a = 42
def __getnewargs__(self):
return self.x, self.y
def __getstate__(self):
return self.a
def __setstate__(self, a):
self.a = a
def test_pickling_w_overrides():
"""
>>> x = Custom('x', 'y')
>>> x.a = 99
>>> (f, (c, ax, ay), a) = x.__reduce__()
>>> f.__name__
'__newobj__'
>>> f.__module__
'copy_reg'
>>> c.__name__
'Custom'
>>> ax, ay, a
('x', 'y', 99)
>>> pickle.loads(pickle.dumps(x)) == x
1
>>> pickle.loads(pickle.dumps(x, 0)) == x
1
>>> pickle.loads(pickle.dumps(x, 1)) == x
1
>>> pickle.loads(pickle.dumps(x, 2)) == x
1
"""
class Slotted(Persistent):
__slots__ = 's1', 's2', '_p_splat', '_v_eek'
def __init__(self, s1, s2):
self.s1, self.s2 = s1, s2
self._v_eek = 1
self._p_splat = 2
class SubSlotted(Slotted):
__slots__ = 's3', 's4'
def __init__(self, s1, s2, s3):
Slotted.__init__(self, s1, s2)
self.s3 = s3
def __cmp__(self, other):
return cmpattrs(self, other, '__class__', 's1', 's2', 's3', 's4')
def test_pickling_w_slots_only():
"""
>>> x = SubSlotted('x', 'y', 'z')
>>> d, s = x.__getstate__()
>>> d
>>> print_dict(s)
{'s1': 'x', 's2': 'y', 's3': 'z'}
>>> pickle.loads(pickle.dumps(x)) == x
1
>>> pickle.loads(pickle.dumps(x, 0)) == x
1
>>> pickle.loads(pickle.dumps(x, 1)) == x
1
>>> pickle.loads(pickle.dumps(x, 2)) == x
1
>>> x.s4 = 'spam'
>>> d, s = x.__getstate__()
>>> d
>>> print_dict(s)
{'s1': 'x', 's2': 'y', 's3': 'z', 's4': 'spam'}
>>> pickle.loads(pickle.dumps(x)) == x
1
>>> pickle.loads(pickle.dumps(x, 0)) == x
1
>>> pickle.loads(pickle.dumps(x, 1)) == x
1
>>> pickle.loads(pickle.dumps(x, 2)) == x
1
"""
class SubSubSlotted(SubSlotted):
def __init__(self, s1, s2, s3, **kw):
SubSlotted.__init__(self, s1, s2, s3)
self.__dict__.update(kw)
self._v_favorite_color = 'blue'
self._p_foo = 'bar'
def __cmp__(self, other):
return cmpattrs(self, other,
'__class__', 's1', 's2', 's3', 's4',
*(self.__dict__.keys()))
def test_pickling_w_slots():
"""
>>> x = SubSubSlotted('x', 'y', 'z', aaa=1, bbb='foo')
>>> d, s = x.__getstate__()
>>> print_dict(d)
{'aaa': 1, 'bbb': 'foo'}
>>> print_dict(s)
{'s1': 'x', 's2': 'y', 's3': 'z'}
>>> pickle.loads(pickle.dumps(x)) == x
1
>>> pickle.loads(pickle.dumps(x, 0)) == x
1
>>> pickle.loads(pickle.dumps(x, 1)) == x
1
>>> pickle.loads(pickle.dumps(x, 2)) == x
1
>>> x.s4 = 'spam'
>>> d, s = x.__getstate__()
>>> print_dict(d)
{'aaa': 1, 'bbb': 'foo'}
>>> print_dict(s)
{'s1': 'x', 's2': 'y', 's3': 'z', 's4': 'spam'}
>>> pickle.loads(pickle.dumps(x)) == x
1
>>> pickle.loads(pickle.dumps(x, 0)) == x
1
>>> pickle.loads(pickle.dumps(x, 1)) == x
1
>>> pickle.loads(pickle.dumps(x, 2)) == x
1
"""
def test_pickling_w_slots_w_empty_dict():
"""
>>> x = SubSubSlotted('x', 'y', 'z')
>>> d, s = x.__getstate__()
>>> print_dict(d)
{}
>>> print_dict(s)
{'s1': 'x', 's2': 'y', 's3': 'z'}
>>> pickle.loads(pickle.dumps(x)) == x
1
>>> pickle.loads(pickle.dumps(x, 0)) == x
1
>>> pickle.loads(pickle.dumps(x, 1)) == x
1
>>> pickle.loads(pickle.dumps(x, 2)) == x
1
>>> x.s4 = 'spam'
>>> d, s = x.__getstate__()
>>> print_dict(d)
{}
>>> print_dict(s)
{'s1': 'x', 's2': 'y', 's3': 'z', 's4': 'spam'}
>>> pickle.loads(pickle.dumps(x)) == x
1
>>> pickle.loads(pickle.dumps(x, 0)) == x
1
>>> pickle.loads(pickle.dumps(x, 1)) == x
1
>>> pickle.loads(pickle.dumps(x, 2)) == x
1
"""
from zope.testing.doctest import DocTestSuite
import unittest
def test_suite():
return unittest.TestSuite((
DocTestSuite(),
))
if __name__ == '__main__': unittest.main()
ZODB/src/persistent/tests/persistent.txt 0000640 0001750 0000144 00000024115 10375443702 020264 0 ustar ctheune users Tests for `persistent.Persistent`
=================================
This document is an extended doc test that covers the basics of the
Persistent base class. The test expects a class named `P` to be
provided in its globals. The `P` class implements the `Persistent`
interface.
Test framework
--------------
The class `P` needs to behave like `ExampleP`. (Note that the code below
is *not* part of the tests.)
::
class ExampleP(Persistent):
def __init__(self):
self.x = 0
def inc(self):
self.x += 1
The tests use stub data managers. A data manager is responsible for
loading and storing the state of a persistent object. It's stored in
the ``_p_jar`` attribute of a persistent object.
>>> class DM:
... def __init__(self):
... self.called = 0
... def register(self, ob):
... self.called += 1
... def setstate(self, ob):
... ob.__setstate__({'x': 42})
>>> class BrokenDM(DM):
... def register(self,ob):
... self.called += 1
... raise NotImplementedError
... def setstate(self,ob):
... raise NotImplementedError
>>> from persistent import Persistent
Test Persistent without Data Manager
------------------------------------
First do some simple tests of a Persistent instance that does not have
a data manager (``_p_jar``).
>>> p = P()
>>> p.x
0
>>> p._p_changed
False
>>> p._p_state
0
>>> p._p_jar
>>> p._p_oid
Verify that modifications have no effect on ``_p_state`` of ``_p_changed``.
>>> p.inc()
>>> p.inc()
>>> p.x
2
>>> p._p_changed
False
>>> p._p_state
0
Try all sorts of different ways to change the object's state.
>>> p._p_deactivate()
>>> p._p_state
0
>>> p._p_changed = True
>>> p._p_state
0
>>> del p._p_changed
>>> p._p_changed
False
>>> p._p_state
0
>>> p.x
2
Test Persistent with Data Manager
---------------------------------
Next try some tests of an object with a data manager. The `DM` class is
a simple testing stub.
>>> p = P()
>>> dm = DM()
>>> p._p_oid = "00000012"
>>> p._p_jar = dm
>>> p._p_changed
0
>>> dm.called
0
Modifying the object marks it as changed and registers it with the data
manager. Subsequent modifications don't have additional side-effects.
>>> p.inc()
>>> p._p_changed
1
>>> dm.called
1
>>> p.inc()
>>> p._p_changed
1
>>> dm.called
1
It's not possible to deactivate a modified object.
>>> p._p_deactivate()
>>> p._p_changed
1
It is possible to invalidate it. That's the key difference between
deactivation and invalidation.
>>> p._p_invalidate()
>>> p._p_state
-1
Now that the object is a ghost, any attempt to modify it will require that it
be unghosted first. The test data manager has the odd property that it sets
the object's ``x`` attribute to ``42`` when it is unghosted.
>>> p.inc()
>>> p.x
43
>>> dm.called
2
You can manually reset the changed field to ``False``, although it's not clear
why you would want to do that. The object changes to the ``UPTODATE`` state
but retains its modifications.
>>> p._p_changed = False
>>> p._p_state
0
>>> p._p_changed
False
>>> p.x
43
>>> p.inc()
>>> p._p_changed
True
>>> dm.called
3
``__getstate__()`` and ``__setstate__()``
-----------------------------------------
The next several tests cover the ``__getstate__()`` and ``__setstate__()``
implementations.
>>> p = P()
>>> state = p.__getstate__()
>>> isinstance(state, dict)
True
>>> state['x']
0
>>> p._p_state
0
Calling setstate always leaves the object in the uptodate state?
(I'm not entirely clear on this one.)
>>> p.__setstate__({'x': 5})
>>> p._p_state
0
Assigning to a volatile attribute has no effect on the object state.
>>> p._v_foo = 2
>>> p.__getstate__()
{'x': 5}
>>> p._p_state
0
The ``_p_serial`` attribute is not affected by calling setstate.
>>> p._p_serial = "00000012"
>>> p.__setstate__(p.__getstate__())
>>> p._p_serial
'00000012'
Change Ghost test
-----------------
If an object is a ghost and its ``_p_changed`` is set to ``True`` (any true
value), it should activate (unghostify) the object. This behavior is new in
ZODB 3.6; before then, an attempt to do ``ghost._p_changed = True`` was
ignored.
>>> p = P()
>>> p._p_jar = DM()
>>> p._p_oid = 1
>>> p._p_deactivate()
>>> p._p_changed # None
>>> p._p_state # ghost state
-1
>>> p._p_changed = True
>>> p._p_changed
1
>>> p._p_state # changed state
1
>>> p.x
42
Activate, deactivate, and invalidate
------------------------------------
Some of these tests are redundant, but are included to make sure there
are explicit and simple tests of ``_p_activate()``, ``_p_deactivate()``, and
``_p_invalidate()``.
>>> p = P()
>>> p._p_oid = 1
>>> p._p_jar = DM()
>>> p._p_deactivate()
>>> p._p_state
-1
>>> p._p_activate()
>>> p._p_state
0
>>> p.x
42
>>> p.inc()
>>> p.x
43
>>> p._p_state
1
>>> p._p_invalidate()
>>> p._p_state
-1
>>> p.x
42
Test failures
-------------
The following tests cover various errors cases.
When an object is modified, it registers with its data manager. If that
registration fails, the exception is propagated and the object stays in the
up-to-date state. It shouldn't change to the modified state, because it won't
be saved when the transaction commits.
>>> p = P()
>>> p._p_oid = 1
>>> p._p_jar = BrokenDM()
>>> p._p_state
0
>>> p._p_jar.called
0
>>> p._p_changed = 1
Traceback (most recent call last):
...
NotImplementedError
>>> p._p_jar.called
1
>>> p._p_state
0
Make sure that exceptions that occur inside the data manager's ``setstate()``
method propagate out to the caller.
>>> p = P()
>>> p._p_oid = 1
>>> p._p_jar = BrokenDM()
>>> p._p_deactivate()
>>> p._p_state
-1
>>> p._p_activate()
Traceback (most recent call last):
...
NotImplementedError
>>> p._p_state
-1
Special test to cover layout of ``__dict__``
--------------------------------------------
We once had a bug in the `Persistent` class that calculated an incorrect
offset for the ``__dict__`` attribute. It assigned ``__dict__`` and
``_p_jar`` to the same location in memory. This is a simple test to make sure
they have different locations.
>>> p = P()
>>> p.inc()
>>> p.inc()
>>> 'x' in p.__dict__
True
>>> p._p_jar
Inheritance and metaclasses
---------------------------
Simple tests to make sure it's possible to inherit from the `Persistent` base
class multiple times. There used to be metaclasses involved in `Persistent`
that probably made this a more interesting test.
>>> class A(Persistent):
... pass
>>> class B(Persistent):
... pass
>>> class C(A, B):
... pass
>>> class D(object):
... pass
>>> class E(D, B):
... pass
>>> a = A()
>>> b = B()
>>> c = C()
>>> d = D()
>>> e = E()
Also make sure that it's possible to define `Persistent` classes that have a
custom metaclass.
>>> class alternateMeta(type):
... type
>>> class alternate(object):
... __metaclass__ = alternateMeta
>>> class mixedMeta(alternateMeta, type):
... pass
>>> class mixed(alternate, Persistent):
... pass
>>> class mixed(Persistent, alternate):
... pass
Basic type structure
--------------------
>>> Persistent.__dictoffset__
0
>>> Persistent.__weakrefoffset__
0
>>> Persistent.__basicsize__ > object.__basicsize__
True
>>> P.__dictoffset__ > 0
True
>>> P.__weakrefoffset__ > 0
True
>>> P.__dictoffset__ < P.__weakrefoffset__
True
>>> P.__basicsize__ > Persistent.__basicsize__
True
Slots
-----
These are some simple tests of classes that have an ``__slots__``
attribute. Some of the classes should have slots, others shouldn't.
>>> class noDict(object):
... __slots__ = ['foo']
>>> class p_noDict(Persistent):
... __slots__ = ['foo']
>>> class p_shouldHaveDict(p_noDict):
... pass
>>> p_noDict.__dictoffset__
0
>>> x = p_noDict()
>>> x.foo = 1
>>> x.foo
1
>>> x.bar = 1
Traceback (most recent call last):
...
AttributeError: 'p_noDict' object has no attribute 'bar'
>>> x._v_bar = 1
Traceback (most recent call last):
...
AttributeError: 'p_noDict' object has no attribute '_v_bar'
>>> x.__dict__
Traceback (most recent call last):
...
AttributeError: 'p_noDict' object has no attribute '__dict__'
The various _p_ attributes are unaffected by slots.
>>> p._p_oid
>>> p._p_jar
>>> p._p_state
0
If the most-derived class does not specify
>>> p_shouldHaveDict.__dictoffset__ > 0
True
>>> x = p_shouldHaveDict()
>>> isinstance(x.__dict__, dict)
True
Pickling
--------
There's actually a substantial effort involved in making subclasses of
`Persistent` work with plain-old pickle. The ZODB serialization layer never
calls pickle on an object; it pickles the object's class description and its
state as two separate pickles.
>>> import pickle
>>> p = P()
>>> p.inc()
>>> p2 = pickle.loads(pickle.dumps(p))
>>> p2.__class__ is P
True
>>> p2.x == p.x
True
We should also test that pickle works with custom getstate and setstate.
Perhaps even reduce. The problem is that pickling depends on finding the
class in a particular module, and classes defined here won't appear in any
module. We could require each user of the tests to define a base class, but
that might be tedious.
Interfaces
----------
Some versions of Zope and ZODB have the `zope.interfaces` package available.
If it is available, then persistent will be associated with several
interfaces. It's hard to write a doctest test that runs the tests only if
`zope.interface` is available, so this test looks a little unusual. One
problem is that the assert statements won't do anything if you run with `-O`.
>>> try:
... import zope.interface
... except ImportError:
... pass
... else:
... from persistent.interfaces import IPersistent
... assert IPersistent.implementedBy(Persistent)
... p = Persistent()
... assert IPersistent.providedBy(p)
... assert IPersistent.implementedBy(P)
... p = P()
... assert IPersistent.providedBy(p)
ZODB/src/persistent/DEPENDENCIES.cfg 0000640 0001750 0000144 00000000125 10223352372 016455 0 ustar ctheune users # the following are needed by the tests
transaction
ZODB
zope.interface
zope.testing
ZODB/src/persistent/__init__.py 0000640 0001750 0000144 00000002273 10057366465 016302 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Provide access to Persistent and PersistentMapping.
$Id: __init__.py 25186 2004-06-02 15:07:33Z jim $
"""
from cPersistence import Persistent, GHOST, UPTODATE, CHANGED, STICKY
from cPickleCache import PickleCache
from cPersistence import simple_new
import copy_reg
copy_reg.constructor(simple_new)
# Make an interface declaration for Persistent,
# if zope.interface is available.
try:
from zope.interface import classImplements
except ImportError:
pass
else:
from persistent.interfaces import IPersistent
classImplements(Persistent, IPersistent)
ZODB/src/persistent/cPersistence.c 0000640 0001750 0000144 00000074300 10311652333 016745 0 ustar ctheune users /*****************************************************************************
Copyright (c) 2001, 2002 Zope Corporation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
static char cPersistence_doc_string[] =
"Defines Persistent mixin class for persistent objects.\n"
"\n"
"$Id: cPersistence.c 38459 2005-09-13 22:38:19Z tim_one $\n";
#include "cPersistence.h"
#include "structmember.h"
struct ccobject_head_struct {
CACHE_HEAD
};
/* These two objects are initialized when the module is loaded */
static PyObject *TimeStamp, *py_simple_new;
/* Strings initialized by init_strings() below. */
static PyObject *py_keys, *py_setstate, *py___dict__, *py_timeTime;
static PyObject *py__p_changed, *py__p_deactivate;
static PyObject *py___getattr__, *py___setattr__, *py___delattr__;
static PyObject *py___slotnames__, *copy_reg_slotnames, *__newobj__;
static PyObject *py___getnewargs__, *py___getstate__;
static int
init_strings(void)
{
#define INIT_STRING(S) \
if (!(py_ ## S = PyString_InternFromString(#S))) \
return -1;
INIT_STRING(keys);
INIT_STRING(setstate);
INIT_STRING(timeTime);
INIT_STRING(__dict__);
INIT_STRING(_p_changed);
INIT_STRING(_p_deactivate);
INIT_STRING(__getattr__);
INIT_STRING(__setattr__);
INIT_STRING(__delattr__);
INIT_STRING(__slotnames__);
INIT_STRING(__getnewargs__);
INIT_STRING(__getstate__);
#undef INIT_STRING
return 0;
}
#ifdef Py_DEBUG
static void
fatal_1350(cPersistentObject *self, const char *caller, const char *detail)
{
char buf[1000];
PyOS_snprintf(buf, sizeof(buf),
"cPersistence.c %s(): object at %p with type %.200s\n"
"%s.\n"
"The only known cause is multiple threads trying to ghost and\n"
"unghost the object simultaneously.\n"
"That's not legal, but ZODB can't stop it.\n"
"See Collector #1350.\n",
caller, self, self->ob_type->tp_name, detail);
Py_FatalError(buf);
}
#endif
static void ghostify(cPersistentObject*);
/* Load the state of the object, unghostifying it. Upon success, return 1.
* If an error occurred, re-ghostify the object and return -1.
*/
static int
unghostify(cPersistentObject *self)
{
if (self->state < 0 && self->jar) {
PyObject *r;
/* Is it ever possibly to not have a cache? */
if (self->cache) {
/* Create a node in the ring for this unghostified object. */
self->cache->non_ghost_count++;
ring_add(&self->cache->ring_home, &self->ring);
Py_INCREF(self);
}
/* set state to CHANGED while setstate() call is in progress
to prevent a recursive call to _PyPersist_Load().
*/
self->state = cPersistent_CHANGED_STATE;
/* Call the object's __setstate__() */
r = PyObject_CallMethod(self->jar, "setstate", "O", (PyObject *)self);
if (r == NULL) {
ghostify(self);
return -1;
}
self->state = cPersistent_UPTODATE_STATE;
Py_DECREF(r);
if (self->cache && self->ring.r_next == NULL) {
#ifdef Py_DEBUG
fatal_1350(self, "unghostify",
"is not in the cache despite that we just "
"unghostified it");
#else
PyErr_Format(PyExc_SystemError, "object at %p with type "
"%.200s not in the cache despite that we just "
"unghostified it", self, self->ob_type->tp_name);
return -1;
#endif
}
}
return 1;
}
/****************************************************************************/
static PyTypeObject Pertype;
static void
accessed(cPersistentObject *self)
{
/* Do nothing unless the object is in a cache and not a ghost. */
if (self->cache && self->state >= 0 && self->ring.r_next)
ring_move_to_head(&self->cache->ring_home, &self->ring);
}
static void
unlink_from_ring(cPersistentObject *self)
{
/* If the cache has been cleared, then a non-ghost object
isn't in the ring any longer.
*/
if (self->ring.r_next == NULL)
return;
/* if we're ghostifying an object, we better have some non-ghosts */
assert(self->cache->non_ghost_count > 0);
self->cache->non_ghost_count--;
ring_del(&self->ring);
}
static void
ghostify(cPersistentObject *self)
{
PyObject **dictptr;
/* are we already a ghost? */
if (self->state == cPersistent_GHOST_STATE)
return;
/* Is it ever possible to not have a cache? */
if (self->cache == NULL) {
self->state = cPersistent_GHOST_STATE;
return;
}
if (self->ring.r_next == NULL) {
/* There's no way to raise an error in this routine. */
#ifdef Py_DEBUG
fatal_1350(self, "ghostify", "claims to be in a cache but isn't");
#else
return;
#endif
}
/* If we're ghostifying an object, we better have some non-ghosts. */
assert(self->cache->non_ghost_count > 0);
self->cache->non_ghost_count--;
ring_del(&self->ring);
self->state = cPersistent_GHOST_STATE;
dictptr = _PyObject_GetDictPtr((PyObject *)self);
if (dictptr && *dictptr) {
Py_DECREF(*dictptr);
*dictptr = NULL;
}
/* We remove the reference to the just ghosted object that the ring
* holds. Note that the dictionary of oids->objects has an uncounted
* reference, so if the ring's reference was the only one, this frees
* the ghost object. Note further that the object's dealloc knows to
* inform the dictionary that it is going away.
*/
Py_DECREF(self);
}
static int
changed(cPersistentObject *self)
{
if ((self->state == cPersistent_UPTODATE_STATE ||
self->state == cPersistent_STICKY_STATE)
&& self->jar)
{
PyObject *meth, *arg, *result;
static PyObject *s_register;
if (s_register == NULL)
s_register = PyString_InternFromString("register");
meth = PyObject_GetAttr((PyObject *)self->jar, s_register);
if (meth == NULL)
return -1;
arg = PyTuple_New(1);
if (arg == NULL) {
Py_DECREF(meth);
return -1;
}
Py_INCREF(self);
PyTuple_SET_ITEM(arg, 0, (PyObject *)self);
result = PyEval_CallObject(meth, arg);
Py_DECREF(arg);
Py_DECREF(meth);
if (result == NULL)
return -1;
Py_DECREF(result);
self->state = cPersistent_CHANGED_STATE;
}
return 0;
}
static PyObject *
Per__p_deactivate(cPersistentObject *self)
{
if (self->state == cPersistent_UPTODATE_STATE && self->jar) {
PyObject **dictptr = _PyObject_GetDictPtr((PyObject *)self);
if (dictptr && *dictptr) {
Py_DECREF(*dictptr);
*dictptr = NULL;
}
/* Note that we need to set to ghost state unless we are
called directly. Methods that override this need to
do the same! */
ghostify(self);
}
Py_INCREF(Py_None);
return Py_None;
}
static PyObject *
Per__p_activate(cPersistentObject *self)
{
if (unghostify(self) < 0)
return NULL;
Py_INCREF(Py_None);
return Py_None;
}
static int Per_set_changed(cPersistentObject *self, PyObject *v);
static PyObject *
Per__p_invalidate(cPersistentObject *self)
{
signed char old_state = self->state;
if (old_state != cPersistent_GHOST_STATE) {
if (Per_set_changed(self, NULL) < 0)
return NULL;
ghostify(self);
}
Py_INCREF(Py_None);
return Py_None;
}
static PyObject *
pickle_slotnames(PyTypeObject *cls)
{
PyObject *slotnames;
slotnames = PyDict_GetItem(cls->tp_dict, py___slotnames__);
if (slotnames) {
Py_INCREF(slotnames);
return slotnames;
}
slotnames = PyObject_CallFunctionObjArgs(copy_reg_slotnames,
(PyObject*)cls, NULL);
if (slotnames && !(slotnames == Py_None || PyList_Check(slotnames))) {
PyErr_SetString(PyExc_TypeError,
"copy_reg._slotnames didn't return a list or None");
Py_DECREF(slotnames);
return NULL;
}
return slotnames;
}
static PyObject *
pickle_copy_dict(PyObject *state)
{
PyObject *copy, *key, *value;
char *ckey;
int pos = 0;
copy = PyDict_New();
if (!copy)
return NULL;
if (!state)
return copy;
while (PyDict_Next(state, &pos, &key, &value)) {
if (key && PyString_Check(key)) {
ckey = PyString_AS_STRING(key);
if (*ckey == '_' &&
(ckey[1] == 'v' || ckey[1] == 'p') &&
ckey[2] == '_')
/* skip volatile and persistent */
continue;
}
if (PyObject_SetItem(copy, key, value) < 0)
goto err;
}
return copy;
err:
Py_DECREF(copy);
return NULL;
}
static char pickle___getstate__doc[] =
"Get the object serialization state\n"
"\n"
"If the object has no assigned slots and has no instance dictionary, then \n"
"None is returned.\n"
"\n"
"If the object has no assigned slots and has an instance dictionary, then \n"
"the a copy of the instance dictionary is returned. The copy has any items \n"
"with names starting with '_v_' or '_p_' ommitted.\n"
"\n"
"If the object has assigned slots, then a two-element tuple is returned. \n"
"The first element is either None or a copy of the instance dictionary, \n"
"as described above. The second element is a dictionary with items \n"
"for each of the assigned slots.\n"
;
static PyObject *
pickle___getstate__(PyObject *self)
{
PyObject *slotnames=NULL, *slots=NULL, *state=NULL;
PyObject **dictp;
int n=0;
slotnames = pickle_slotnames(self->ob_type);
if (!slotnames)
return NULL;
dictp = _PyObject_GetDictPtr(self);
if (dictp)
state = pickle_copy_dict(*dictp);
else {
state = Py_None;
Py_INCREF(state);
}
if (slotnames != Py_None) {
int i;
slots = PyDict_New();
if (!slots)
goto end;
for (i = 0; i < PyList_GET_SIZE(slotnames); i++) {
PyObject *name, *value;
char *cname;
name = PyList_GET_ITEM(slotnames, i);
if (PyString_Check(name)) {
cname = PyString_AS_STRING(name);
if (*cname == '_' &&
(cname[1] == 'v' || cname[1] == 'p') &&
cname[2] == '_')
/* skip volatile and persistent */
continue;
}
/* Unclear: Will this go through our getattr hook? */
value = PyObject_GetAttr(self, name);
if (value == NULL)
PyErr_Clear();
else {
int err = PyDict_SetItem(slots, name, value);
Py_DECREF(value);
if (err < 0)
goto end;
n++;
}
}
}
if (n)
state = Py_BuildValue("(NO)", state, slots);
end:
Py_XDECREF(slotnames);
Py_XDECREF(slots);
return state;
}
static int
pickle_setattrs_from_dict(PyObject *self, PyObject *dict)
{
PyObject *key, *value;
int pos = 0;
if (!PyDict_Check(dict)) {
PyErr_SetString(PyExc_TypeError, "Expected dictionary");
return -1;
}
while (PyDict_Next(dict, &pos, &key, &value)) {
if (PyObject_SetAttr(self, key, value) < 0)
return -1;
}
return 0;
}
static char pickle___setstate__doc[] =
"Set the object serialization state\n\n"
"The state should be in one of 3 forms:\n\n"
"- None\n\n"
" Ignored\n\n"
"- A dictionary\n\n"
" In this case, the object's instance dictionary will be cleared and \n"
" updated with the new state.\n\n"
"- A two-tuple with a string as the first element. \n\n"
" In this case, the method named by the string in the first element will be\n"
" called with the second element.\n\n"
" This form supports migration of data formats.\n\n"
"- A two-tuple with None or a Dictionary as the first element and\n"
" with a dictionary as the second element.\n\n"
" If the first element is not None, then the object's instance dictionary \n"
" will be cleared and updated with the value.\n\n"
" The items in the second element will be assigned as attributes.\n"
;
static PyObject *
pickle___setstate__(PyObject *self, PyObject *state)
{
PyObject *slots=NULL;
if (PyTuple_Check(state)) {
if (!PyArg_ParseTuple(state, "OO:__setstate__", &state, &slots))
return NULL;
}
if (state != Py_None) {
PyObject **dict;
dict = _PyObject_GetDictPtr(self);
if (dict) {
if (!*dict) {
*dict = PyDict_New();
if (!*dict)
return NULL;
}
}
if (*dict) {
PyDict_Clear(*dict);
if (PyDict_Update(*dict, state) < 0)
return NULL;
}
else if (pickle_setattrs_from_dict(self, state) < 0)
return NULL;
}
if (slots && pickle_setattrs_from_dict(self, slots) < 0)
return NULL;
Py_INCREF(Py_None);
return Py_None;
}
static char pickle___reduce__doc[] =
"Reduce an object to contituent parts for serialization\n"
;
static PyObject *
pickle___reduce__(PyObject *self)
{
PyObject *args=NULL, *bargs=NULL, *state=NULL, *getnewargs=NULL;
int l, i;
getnewargs = PyObject_GetAttr(self, py___getnewargs__);
if (getnewargs) {
bargs = PyObject_CallFunctionObjArgs(getnewargs, NULL);
Py_DECREF(getnewargs);
if (!bargs)
return NULL;
l = PyTuple_Size(bargs);
if (l < 0)
goto end;
}
else {
PyErr_Clear();
l = 0;
}
args = PyTuple_New(l+1);
if (args == NULL)
goto end;
Py_INCREF(self->ob_type);
PyTuple_SET_ITEM(args, 0, (PyObject*)(self->ob_type));
for (i = 0; i < l; i++) {
Py_INCREF(PyTuple_GET_ITEM(bargs, i));
PyTuple_SET_ITEM(args, i+1, PyTuple_GET_ITEM(bargs, i));
}
state = PyObject_CallMethodObjArgs(self, py___getstate__, NULL);
if (!state)
goto end;
state = Py_BuildValue("(OON)", __newobj__, args, state);
end:
Py_XDECREF(bargs);
Py_XDECREF(args);
return state;
}
/* Return the object's state, a dict or None.
If the object has no dict, it's state is None.
Otherwise, return a dict containing all the attributes that
don't start with "_v_".
The caller should not modify this dict, as it may be a reference to
the object's __dict__.
*/
static PyObject *
Per__getstate__(cPersistentObject *self)
{
/* TODO: Should it be an error to call __getstate__() on a ghost? */
if (unghostify(self) < 0)
return NULL;
/* TODO: should we increment stickyness? Tim doesn't understand that
question. S*/
return pickle___getstate__((PyObject*)self);
}
/* The Persistent base type provides a traverse function, but not a
clear function. An instance of a Persistent subclass will have
its dict cleared through subtype_clear().
There is always a cycle between a persistent object and its cache.
When the cycle becomes unreachable, the clear function for the
cache will break the cycle. Thus, the persistent object need not
have a clear function. It would be complex to write a clear function
for the objects, if we needed one, because of the reference count
tricks done by the cache.
*/
static void
Per_dealloc(cPersistentObject *self)
{
if (self->state >= 0)
unlink_from_ring(self);
if (self->cache)
cPersistenceCAPI->percachedel(self->cache, self->oid);
Py_XDECREF(self->cache);
Py_XDECREF(self->jar);
Py_XDECREF(self->oid);
self->ob_type->tp_free(self);
}
static int
Per_traverse(cPersistentObject *self, visitproc visit, void *arg)
{
int err;
#define VISIT(SLOT) \
if (SLOT) { \
err = visit((PyObject *)(SLOT), arg); \
if (err) \
return err; \
}
VISIT(self->jar);
VISIT(self->oid);
VISIT(self->cache);
#undef VISIT
return 0;
}
/* convert_name() returns a new reference to a string name
or sets an exception and returns NULL.
*/
static PyObject *
convert_name(PyObject *name)
{
#ifdef Py_USING_UNICODE
/* The Unicode to string conversion is done here because the
existing tp_setattro slots expect a string object as name
and we wouldn't want to break those. */
if (PyUnicode_Check(name)) {
name = PyUnicode_AsEncodedString(name, NULL, NULL);
}
else
#endif
if (!PyString_Check(name)) {
PyErr_SetString(PyExc_TypeError, "attribute name must be a string");
return NULL;
} else
Py_INCREF(name);
return name;
}
/* Returns true if the object requires unghostification.
There are several special attributes that we allow access to without
requiring that the object be unghostified:
__class__
__del__
__dict__
__of__
__setstate__
*/
static int
unghost_getattr(const char *s)
{
if (*s++ != '_')
return 1;
if (*s == 'p') {
s++;
if (*s == '_')
return 0; /* _p_ */
else
return 1;
}
else if (*s == '_') {
s++;
switch (*s) {
case 'c':
return strcmp(s, "class__");
case 'd':
s++;
if (!strcmp(s, "el__"))
return 0; /* __del__ */
if (!strcmp(s, "ict__"))
return 0; /* __dict__ */
return 1;
case 'o':
return strcmp(s, "of__");
case 's':
return strcmp(s, "setstate__");
default:
return 1;
}
}
return 1;
}
static PyObject*
Per_getattro(cPersistentObject *self, PyObject *name)
{
PyObject *result = NULL; /* guilty until proved innocent */
char *s;
name = convert_name(name);
if (!name)
goto Done;
s = PyString_AS_STRING(name);
if (unghost_getattr(s)) {
if (unghostify(self) < 0)
goto Done;
accessed(self);
}
result = PyObject_GenericGetAttr((PyObject *)self, name);
Done:
Py_XDECREF(name);
return result;
}
/* Exposed as _p_getattr method. Test whether base getattr should be used */
static PyObject *
Per__p_getattr(cPersistentObject *self, PyObject *name)
{
PyObject *result = NULL; /* guilty until proved innocent */
char *s;
name = convert_name(name);
if (!name)
goto Done;
s = PyString_AS_STRING(name);
if (*s != '_' || unghost_getattr(s)) {
if (unghostify(self) < 0)
goto Done;
accessed(self);
result = Py_False;
}
else
result = Py_True;
Py_INCREF(result);
Done:
Py_XDECREF(name);
return result;
}
/*
TODO: we should probably not allow assignment of __class__ and __dict__.
*/
static int
Per_setattro(cPersistentObject *self, PyObject *name, PyObject *v)
{
int result = -1; /* guilty until proved innocent */
char *s;
name = convert_name(name);
if (!name)
goto Done;
s = PyString_AS_STRING(name);
if (strncmp(s, "_p_", 3) != 0) {
if (unghostify(self) < 0)
goto Done;
accessed(self);
if (strncmp(s, "_v_", 3) != 0
&& self->state != cPersistent_CHANGED_STATE) {
if (changed(self) < 0)
goto Done;
}
}
result = PyObject_GenericSetAttr((PyObject *)self, name, v);
Done:
Py_XDECREF(name);
return result;
}
static int
Per_p_set_or_delattro(cPersistentObject *self, PyObject *name, PyObject *v)
{
int result = -1; /* guilty until proved innocent */
char *s;
name = convert_name(name);
if (!name)
goto Done;
s = PyString_AS_STRING(name);
if (strncmp(s, "_p_", 3)) {
if (unghostify(self) < 0)
goto Done;
accessed(self);
result = 0;
}
else {
if (PyObject_GenericSetAttr((PyObject *)self, name, v) < 0)
goto Done;
result = 1;
}
Done:
Py_XDECREF(name);
return result;
}
static PyObject *
Per__p_setattr(cPersistentObject *self, PyObject *args)
{
PyObject *name, *v, *result;
int r;
if (!PyArg_ParseTuple(args, "OO:_p_setattr", &name, &v))
return NULL;
r = Per_p_set_or_delattro(self, name, v);
if (r < 0)
return NULL;
result = r ? Py_True : Py_False;
Py_INCREF(result);
return result;
}
static PyObject *
Per__p_delattr(cPersistentObject *self, PyObject *name)
{
int r;
PyObject *result;
r = Per_p_set_or_delattro(self, name, NULL);
if (r < 0)
return NULL;
result = r ? Py_True : Py_False;
Py_INCREF(result);
return result;
}
static PyObject *
Per_get_changed(cPersistentObject *self)
{
if (self->state < 0) {
Py_INCREF(Py_None);
return Py_None;
}
return PyBool_FromLong(self->state == cPersistent_CHANGED_STATE);
}
static int
Per_set_changed(cPersistentObject *self, PyObject *v)
{
int deactivate = 0;
int true;
if (!v) {
/* delattr is used to invalidate an object even if it has changed. */
if (self->state != cPersistent_GHOST_STATE)
self->state = cPersistent_UPTODATE_STATE;
deactivate = 1;
}
else if (v == Py_None)
deactivate = 1;
if (deactivate) {
PyObject *res, *meth;
meth = PyObject_GetAttr((PyObject *)self, py__p_deactivate);
if (meth == NULL)
return -1;
res = PyObject_CallObject(meth, NULL);
if (res)
Py_DECREF(res);
else {
/* an error occured in _p_deactivate().
It's not clear what we should do here. The code is
obviously ignoring the exception, but it shouldn't return
0 for a getattr and set an exception. The simplest change
is to clear the exception, but that simply masks the
error.
This prints an error to stderr just like exceptions in
__del__(). It would probably be better to log it but that
would be painful from C.
*/
PyErr_WriteUnraisable(meth);
}
Py_DECREF(meth);
return 0;
}
/* !deactivate. If passed a true argument, mark self as changed (starting
* with ZODB 3.6, that includes activating the object if it's a ghost).
* If passed a false argument, and the object isn't a ghost, set the
* state as up-to-date.
*/
true = PyObject_IsTrue(v);
if (true == -1)
return -1;
if (true) {
if (self->state < 0) {
if (unghostify(self) < 0)
return -1;
}
return changed(self);
}
/* We were passed a false, non-None argument. If we're not a ghost,
* mark self as up-to-date.
*/
if (self->state >= 0)
self->state = cPersistent_UPTODATE_STATE;
return 0;
}
static PyObject *
Per_get_oid(cPersistentObject *self)
{
PyObject *oid = self->oid ? self->oid : Py_None;
Py_INCREF(oid);
return oid;
}
static int
Per_set_oid(cPersistentObject *self, PyObject *v)
{
if (self->cache) {
int result;
if (v == NULL) {
PyErr_SetString(PyExc_ValueError,
"can't delete _p_oid of cached object");
return -1;
}
if (PyObject_Cmp(self->oid, v, &result) < 0)
return -1;
if (result) {
PyErr_SetString(PyExc_ValueError,
"can not change _p_oid of cached object");
return -1;
}
}
Py_XDECREF(self->oid);
Py_XINCREF(v);
self->oid = v;
return 0;
}
static PyObject *
Per_get_jar(cPersistentObject *self)
{
PyObject *jar = self->jar ? self->jar : Py_None;
Py_INCREF(jar);
return jar;
}
static int
Per_set_jar(cPersistentObject *self, PyObject *v)
{
if (self->cache) {
int result;
if (v == NULL) {
PyErr_SetString(PyExc_ValueError,
"can't delete _p_jar of cached object");
return -1;
}
if (PyObject_Cmp(self->jar, v, &result) < 0)
return -1;
if (result) {
PyErr_SetString(PyExc_ValueError,
"can not change _p_jar of cached object");
return -1;
}
}
Py_XDECREF(self->jar);
Py_XINCREF(v);
self->jar = v;
return 0;
}
static PyObject *
Per_get_serial(cPersistentObject *self)
{
return PyString_FromStringAndSize(self->serial, 8);
}
static int
Per_set_serial(cPersistentObject *self, PyObject *v)
{
if (v) {
if (PyString_Check(v) && PyString_GET_SIZE(v) == 8)
memcpy(self->serial, PyString_AS_STRING(v), 8);
else {
PyErr_SetString(PyExc_ValueError,
"_p_serial must be an 8-character string");
return -1;
}
} else
memset(self->serial, 0, 8);
return 0;
}
static PyObject *
Per_get_mtime(cPersistentObject *self)
{
PyObject *t, *v;
if (unghostify(self) < 0)
return NULL;
accessed(self);
if (memcmp(self->serial, "\0\0\0\0\0\0\0\0", 8) == 0) {
Py_INCREF(Py_None);
return Py_None;
}
t = PyObject_CallFunction(TimeStamp, "s#", self->serial, 8);
if (!t)
return NULL;
v = PyObject_CallMethod(t, "timeTime", "");
Py_DECREF(t);
return v;
}
static PyObject *
Per_get_state(cPersistentObject *self)
{
return PyInt_FromLong(self->state);
}
static PyGetSetDef Per_getsets[] = {
{"_p_changed", (getter)Per_get_changed, (setter)Per_set_changed},
{"_p_jar", (getter)Per_get_jar, (setter)Per_set_jar},
{"_p_mtime", (getter)Per_get_mtime},
{"_p_oid", (getter)Per_get_oid, (setter)Per_set_oid},
{"_p_serial", (getter)Per_get_serial, (setter)Per_set_serial},
{"_p_state", (getter)Per_get_state},
{NULL}
};
static struct PyMethodDef Per_methods[] = {
{"_p_deactivate", (PyCFunction)Per__p_deactivate, METH_NOARGS,
"_p_deactivate() -- Deactivate the object"},
{"_p_activate", (PyCFunction)Per__p_activate, METH_NOARGS,
"_p_activate() -- Activate the object"},
{"_p_invalidate", (PyCFunction)Per__p_invalidate, METH_NOARGS,
"_p_invalidate() -- Invalidate the object"},
{"_p_getattr", (PyCFunction)Per__p_getattr, METH_O,
"_p_getattr(name) -- Test whether the base class must handle the name\n"
"\n"
"The method unghostifies the object, if necessary.\n"
"The method records the object access, if necessary.\n"
"\n"
"This method should be called by subclass __getattribute__\n"
"implementations before doing anything else. If the method\n"
"returns True, then __getattribute__ implementations must delegate\n"
"to the base class, Persistent.\n"
},
{"_p_setattr", (PyCFunction)Per__p_setattr, METH_VARARGS,
"_p_setattr(name, value) -- Save persistent meta data\n"
"\n"
"This method should be called by subclass __setattr__ implementations\n"
"before doing anything else. If it returns true, then the attribute\n"
"was handled by the base class.\n"
"\n"
"The method unghostifies the object, if necessary.\n"
"The method records the object access, if necessary.\n"
},
{"_p_delattr", (PyCFunction)Per__p_delattr, METH_O,
"_p_delattr(name) -- Delete persistent meta data\n"
"\n"
"This method should be called by subclass __delattr__ implementations\n"
"before doing anything else. If it returns true, then the attribute\n"
"was handled by the base class.\n"
"\n"
"The method unghostifies the object, if necessary.\n"
"The method records the object access, if necessary.\n"
},
{"__getstate__", (PyCFunction)Per__getstate__, METH_NOARGS,
pickle___getstate__doc },
{"__setstate__", (PyCFunction)pickle___setstate__, METH_O,
pickle___setstate__doc},
{"__reduce__", (PyCFunction)pickle___reduce__, METH_NOARGS,
pickle___reduce__doc},
{NULL, NULL} /* sentinel */
};
/* This module is compiled as a shared library. Some compilers don't
allow addresses of Python objects defined in other libraries to be
used in static initializers here. The DEFERRED_ADDRESS macro is
used to tag the slots where such addresses appear; the module init
function must fill in the tagged slots at runtime. The argument is
for documentation -- the macro ignores it.
*/
#define DEFERRED_ADDRESS(ADDR) 0
static PyTypeObject Pertype = {
PyObject_HEAD_INIT(DEFERRED_ADDRESS(&PyPersist_MetaType))
0, /* ob_size */
"persistent.Persistent", /* tp_name */
sizeof(cPersistentObject), /* tp_basicsize */
0, /* tp_itemsize */
(destructor)Per_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_compare */
0, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
(getattrofunc)Per_getattro, /* tp_getattro */
(setattrofunc)Per_setattro, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
/* tp_flags */
0, /* tp_doc */
(traverseproc)Per_traverse, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
Per_methods, /* tp_methods */
0, /* tp_members */
Per_getsets, /* tp_getset */
};
/* End of code for Persistent objects */
/* -------------------------------------------------------- */
typedef int (*intfunctionwithpythonarg)(PyObject*);
/* Load the object's state if necessary and become sticky */
static int
Per_setstate(cPersistentObject *self)
{
if (unghostify(self) < 0)
return -1;
self->state = cPersistent_STICKY_STATE;
return 0;
}
static PyObject *
simple_new(PyObject *self, PyObject *type_object)
{
return PyType_GenericNew((PyTypeObject *)type_object, NULL, NULL);
}
static PyMethodDef cPersistence_methods[] = {
{"simple_new", simple_new, METH_O,
"Create an object by simply calling a class's __new__ method without "
"arguments."},
{NULL, NULL}
};
static cPersistenceCAPIstruct
truecPersistenceCAPI = {
&Pertype,
(getattrofunc)Per_getattro, /*tp_getattr with object key*/
(setattrofunc)Per_setattro, /*tp_setattr with object key*/
changed,
accessed,
ghostify,
(intfunctionwithpythonarg)Per_setstate,
NULL /* The percachedel slot is initialized in cPickleCache.c when
the module is loaded. It uses a function in a different
shared library. */
};
void
initcPersistence(void)
{
PyObject *m, *s;
PyObject *copy_reg;
if (init_strings() < 0)
return;
m = Py_InitModule3("cPersistence", cPersistence_methods,
cPersistence_doc_string);
Pertype.ob_type = &PyType_Type;
Pertype.tp_new = PyType_GenericNew;
if (PyType_Ready(&Pertype) < 0)
return;
if (PyModule_AddObject(m, "Persistent", (PyObject *)&Pertype) < 0)
return;
cPersistenceCAPI = &truecPersistenceCAPI;
s = PyCObject_FromVoidPtr(cPersistenceCAPI, NULL);
if (!s)
return;
if (PyModule_AddObject(m, "CAPI", s) < 0)
return;
if (PyModule_AddIntConstant(m, "GHOST", cPersistent_GHOST_STATE) < 0)
return;
if (PyModule_AddIntConstant(m, "UPTODATE", cPersistent_UPTODATE_STATE) < 0)
return;
if (PyModule_AddIntConstant(m, "CHANGED", cPersistent_CHANGED_STATE) < 0)
return;
if (PyModule_AddIntConstant(m, "STICKY", cPersistent_STICKY_STATE) < 0)
return;
py_simple_new = PyObject_GetAttrString(m, "simple_new");
if (!py_simple_new)
return;
copy_reg = PyImport_ImportModule("copy_reg");
if (!copy_reg)
return;
copy_reg_slotnames = PyObject_GetAttrString(copy_reg, "_slotnames");
if (!copy_reg_slotnames) {
Py_DECREF(copy_reg);
return;
}
__newobj__ = PyObject_GetAttrString(copy_reg, "__newobj__");
if (!__newobj__) {
Py_DECREF(copy_reg);
return;
}
if (!TimeStamp) {
m = PyImport_ImportModule("persistent.TimeStamp");
if (!m)
return;
TimeStamp = PyObject_GetAttrString(m, "TimeStamp");
Py_DECREF(m);
/* fall through to immediate return on error */
}
}
ZODB/src/persistent/list.py 0000640 0001750 0000144 00000005561 10057366465 015521 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Python implementation of persistent list.
$Id: list.py 25186 2004-06-02 15:07:33Z jim $"""
import persistent
from UserList import UserList
class PersistentList(UserList, persistent.Persistent):
__super_setitem = UserList.__setitem__
__super_delitem = UserList.__delitem__
__super_setslice = UserList.__setslice__
__super_delslice = UserList.__delslice__
__super_iadd = UserList.__iadd__
__super_imul = UserList.__imul__
__super_append = UserList.append
__super_insert = UserList.insert
__super_pop = UserList.pop
__super_remove = UserList.remove
__super_reverse = UserList.reverse
__super_sort = UserList.sort
__super_extend = UserList.extend
def __setitem__(self, i, item):
self.__super_setitem(i, item)
self._p_changed = 1
def __delitem__(self, i):
self.__super_delitem(i)
self._p_changed = 1
def __setslice__(self, i, j, other):
self.__super_setslice(i, j, other)
self._p_changed = 1
def __delslice__(self, i, j):
self.__super_delslice(i, j)
self._p_changed = 1
def __iadd__(self, other):
L = self.__super_iadd(other)
self._p_changed = 1
return L
def __imul__(self, n):
L = self.__super_imul(n)
self._p_changed = 1
return L
def append(self, item):
self.__super_append(item)
self._p_changed = 1
def insert(self, i, item):
self.__super_insert(i, item)
self._p_changed = 1
def pop(self, i=-1):
rtn = self.__super_pop(i)
self._p_changed = 1
return rtn
def remove(self, item):
self.__super_remove(item)
self._p_changed = 1
def reverse(self):
self.__super_reverse()
self._p_changed = 1
def sort(self, *args):
self.__super_sort(*args)
self._p_changed = 1
def extend(self, other):
self.__super_extend(other)
self._p_changed = 1
# This works around a bug in Python 2.1.x (up to 2.1.2 at least) where the
# __cmp__ bogusly raises a RuntimeError, and because this is an extension
# class, none of the rich comparison stuff works anyway.
def __cmp__(self, other):
return cmp(self.data, self._UserList__cast(other))
ZODB/src/persistent/interfaces.py 0000640 0001750 0000144 00000024474 10375443702 016666 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Persistence Interfaces
$Id: interfaces.py 41650 2006-02-17 22:01:38Z ctheune $
"""
from zope.interface import Interface
from zope.interface import Attribute
class IPersistent(Interface):
"""Python persistent interface
A persistent object can be in one of several states:
- Unsaved
The object has been created but not saved in a data manager.
In this state, the _p_changed attribute is non-None and false
and the _p_jar attribute is None.
- Saved
The object has been saved and has not been changed since it was saved.
In this state, the _p_changed attribute is non-None and false
and the _p_jar attribute is set to a data manager.
- Sticky
This state is identical to the saved state except that the
object cannot transition to the ghost state. This is a special
state used by C methods of persistent objects to make sure that
state is not unloaded in the middle of computation.
In this state, the _p_changed attribute is non-None and false
and the _p_jar attribute is set to a data manager.
There is no Python API for detecting whether an object is in the
sticky state.
- Changed
The object has been changed.
In this state, the _p_changed attribute is true
and the _p_jar attribute is set to a data manager.
- Ghost
the object is in memory but its state has not been loaded from
the database (or its state has been unloaded). In this state,
the object doesn't contain any application data.
In this state, the _p_changed attribute is None, and the _p_jar
attribute is set to the data manager from which the object was
obtained.
In all the above, _p_oid (the persistent object id) is set when
_p_jar first gets set.
The following state transitions are possible:
- Unsaved -> Saved
This transition occurs when an object is saved in the
database. This usually happens when an unsaved object is added
to (e.g. as an attribute or item of) a saved (or changed) object
and the transaction is committed.
- Saved -> Changed
Sticky -> Changed
Ghost -> Changed
This transition occurs when someone sets an attribute or sets
_p_changed to a true value on a saved, sticky or ghost object. When
the transition occurs, the persistent object is required to call the
register() method on its data manager, passing itself as the
only argument.
Prior to ZODB 3.6, setting _p_changed to a true value on a ghost object
was ignored (the object remained a ghost, and getting its _p_changed
attribute continued to return None).
- Saved -> Sticky
This transition occurs when C code marks the object as sticky to
prevent its deactivation.
- Saved -> Ghost
This transition occurs when a saved object is deactivated or
invalidated. See discussion below.
- Sticky -> Saved
This transition occurs when C code unmarks the object as sticky to
allow its deactivation.
- Changed -> Saved
This transition occurs when a transaction is committed. After
saving the state of a changed object during transaction commit,
the data manager sets the object's _p_changed to a non-None false
value.
- Changed -> Ghost
This transition occurs when a transaction is aborted. All changed
objects are invalidated by the data manager by an abort.
- Ghost -> Saved
This transition occurs when an attribute or operation of a ghost
is accessed and the object's state is loaded from the database.
Note that there is a separate C API that is not included here.
The C API requires a specific data layout and defines the sticky
state.
About Invalidation, Deactivation and the Sticky & Ghost States
The sticky state is intended to be a short-lived state, to prevent
an object's state from being discarded while we're in C routines. It
is an error to invalidate an object in the sticky state.
Deactivation is a request that an object discard its state (become
a ghost). Deactivation is an optimization, and a request to
deactivate may be ignored. There are two equivalent ways to
request deactivation:
- call _p_deactivate()
- set _p_changed to None
There are two ways to invalidate an object: call the
_p_invalidate() method (preferred) or delete its _p_changed
attribute. This cannot be ignored, and is used when semantics
require invalidation. Normally, an invalidated object transitions
to the ghost state. However, some objects cannot be ghosts. When
these objects are invalidated, they immediately reload their state
from their data manager, and are then in the saved state.
"""
_p_jar = Attribute(
"""The data manager for the object.
The data manager implements the IPersistentDataManager interface.
If there is no data manager, then this is None.
""")
_p_oid = Attribute(
"""The object id.
It is up to the data manager to assign this.
The special value None is reserved to indicate that an object
id has not been assigned. Non-None object ids must be non-empty
strings. The 8-byte string '\0'*8 (8 NUL bytes) is reserved to
identify the database root object.
""")
_p_changed = Attribute(
"""The persistent state of the object.
This is one of:
None -- The object is a ghost.
false but not None -- The object is saved (or has never been saved).
true -- The object has been modified since it was last saved.
The object state may be changed by assigning or deleting this
attribute; however, assigning None is ignored if the object is
not in the saved state, and may be ignored even if the object is
in the saved state.
At and after ZODB 3.6, setting _p_changed to a true value for a ghost
object activates the object; prior to 3.6, setting _p_changed to a
true value on a ghost object was ignored.
Note that an object can transition to the changed state only if
it has a data manager. When such a state change occurs, the
'register' method of the data manager must be called, passing the
persistent object.
Deleting this attribute forces invalidation independent of
existing state, although it is an error if the sticky state is
current.
""")
_p_serial = Attribute(
"""The object serial number.
This member is used by the data manager to distiguish distinct
revisions of a given persistent object.
This is an 8-byte string (not Unicode).
""")
def __getstate__():
"""Get the object data.
The state should not include persistent attributes ("_p_name").
The result must be picklable.
"""
def __setstate__(state):
"""Set the object data.
"""
def _p_activate():
"""Activate the object.
Change the object to the saved state if it is a ghost.
"""
def _p_deactivate():
"""Deactivate the object.
Possibly change an object in the saved state to the
ghost state. It may not be possible to make some persistent
objects ghosts, and, for optimization reasons, the implementation
may choose to keep an object in the saved state.
"""
def _p_invalidate():
"""Invalidate the object.
Invalidate the object. This causes any data to be thrown
away, even if the object is in the changed state. The object
is moved to the ghost state; further accesses will cause
object data to be reloaded.
"""
class IPersistentNoReadConflicts(IPersistent):
def _p_independent():
"""Hook for subclasses to prevent read conflict errors.
A specific persistent object type can define this method and
have it return true if the data manager should ignore read
conflicts for this object.
"""
# TODO: document conflict resolution.
class IPersistentDataManager(Interface):
"""Provide services for managing persistent state.
This interface is used by a persistent object to interact with its
data manager in the context of a transaction.
"""
def setstate(object):
"""Load the state for the given object.
The object should be in the ghost state. The object's state will be
set and the object will end up in the saved state.
The object must provide the IPersistent interface.
"""
def oldstate(obj, tid):
"""Return copy of 'obj' that was written by transaction 'tid'.
The returned object does not have the typical metadata (_p_jar, _p_oid,
_p_serial) set. I'm not sure how references to other peristent objects
are handled.
Parameters
obj: a persistent object from this Connection.
tid: id of a transaction that wrote an earlier revision.
Raises KeyError if tid does not exist or if tid deleted a revision of
obj.
"""
def register(object):
"""Register an IPersistent with the current transaction.
This method must be called when the object transitions to
the changed state.
A subclass could override this method to customize the default
policy of one transaction manager for each thread.
"""
# Maybe later:
## def mtime(object):
## """Return the modification time of the object.
## The modification time may not be known, in which case None
## is returned. If non-None, the return value is the kind of
## timestamp supplied by Python's time.time().
## """
ZODB/src/persistent/dict.py 0000640 0001750 0000144 00000005413 10375443702 015456 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Python implementation of persistent container type
$Id: dict.py 41650 2006-02-17 22:01:38Z ctheune $
"""
import persistent
from UserDict import IterableUserDict
__metaclass__ = type
class PersistentDict(persistent.Persistent, IterableUserDict):
"""A persistent wrapper for mapping objects.
This class allows wrapping of mapping objects so that object
changes are registered. As a side effect, mapping objects may be
subclassed.
"""
# IterableUserDict provides all of the mapping behavior. The
# PersistentDict class is responsible marking the persistent
# state as changed when a method actually changes the state. At
# the mapping API evolves, we may need to add more methods here.
__super_delitem = IterableUserDict.__delitem__
__super_setitem = IterableUserDict.__setitem__
__super_clear = IterableUserDict.clear
__super_update = IterableUserDict.update
__super_setdefault = IterableUserDict.setdefault
__super_pop = IterableUserDict.pop
__super_popitem = IterableUserDict.popitem
__super_p_init = persistent.Persistent.__init__
__super_init = IterableUserDict.__init__
def __init__(self, dict=None):
self.__super_init(dict)
self.__super_p_init()
def __delitem__(self, key):
self.__super_delitem(key)
self._p_changed = True
def __setitem__(self, key, v):
self.__super_setitem(key, v)
self._p_changed = True
def clear(self):
self.__super_clear()
self._p_changed = True
def update(self, b):
self.__super_update(b)
self._p_changed = True
def setdefault(self, key, failobj=None):
# We could inline all of UserDict's implementation into the
# method here, but I'd rather not depend at all on the
# implementation in UserDict (simple as it is).
if not self.has_key(key):
self._p_changed = True
return self.__super_setdefault(key, failobj)
def pop(self, key, *args):
self._p_changed = True
return self.__super_pop(key, *args)
def popitem(self):
self._p_changed = True
return self.__super_popitem()
ZODB/src/persistent/README.txt 0000640 0001750 0000144 00000001342 10375443702 015654 0 ustar ctheune users ===================
Persistence support
===================
(This document is under construction. More basic documentation will eventually
appear here.)
Overriding `__getattr__`, `__getattribute__`, `__setattr__`, and `__delattr__`
------------------------------------------------------------------------------
Subclasses can override the attribute-management methods. For the
`__getattr__` method, the behavior is like that for regular Python
classes and for earlier versions of ZODB 3.
For `__getattribute__`, __setattr__`, and `__delattr__`, it is necessary
to call certain methods defined by `persistent.Persistent`. Detailed
examples and documentation is provided in the test module,
`persistent.tests.test_overriding_attrs`.
ZODB/src/persistent/cPersistence.h 0000640 0001750 0000144 00000007360 10057366465 016773 0 ustar ctheune users /*****************************************************************************
Copyright (c) 2001, 2002 Zope Corporation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
#ifndef CPERSISTENCE_H
#define CPERSISTENCE_H
#include "Python.h"
#include "ring.h"
#define CACHE_HEAD \
PyObject_HEAD \
CPersistentRing ring_home; \
int non_ghost_count;
struct ccobject_head_struct;
typedef struct ccobject_head_struct PerCache;
/* How big is a persistent object?
12 PyGC_Head is two pointers and an int
8 PyObject_HEAD is an int and a pointer
12 jar, oid, cache pointers
8 ring struct
8 serialno
4 state + extra
(52) so far
4 dict ptr
4 weaklist ptr
-------------------------
64 only need 62, but obmalloc rounds up to multiple of eight
Even a ghost requires 64 bytes. It's possible to make a persistent
instance with slots and no dict, which changes the storage needed.
*/
#define cPersistent_HEAD \
PyObject_HEAD \
PyObject *jar; \
PyObject *oid; \
PerCache *cache; \
CPersistentRing ring; \
char serial[8]; \
signed char state; \
unsigned char reserved[3];
#define cPersistent_GHOST_STATE -1
#define cPersistent_UPTODATE_STATE 0
#define cPersistent_CHANGED_STATE 1
#define cPersistent_STICKY_STATE 2
typedef struct {
cPersistent_HEAD
} cPersistentObject;
typedef void (*percachedelfunc)(PerCache *, PyObject *);
typedef struct {
PyTypeObject *pertype;
getattrofunc getattro;
setattrofunc setattro;
int (*changed)(cPersistentObject*);
void (*accessed)(cPersistentObject*);
void (*ghostify)(cPersistentObject*);
int (*setstate)(PyObject*);
percachedelfunc percachedel;
} cPersistenceCAPIstruct;
#define cPersistenceType cPersistenceCAPI->pertype
#ifndef DONT_USE_CPERSISTENCECAPI
static cPersistenceCAPIstruct *cPersistenceCAPI;
#endif
#define cPersistanceModuleName "cPersistence"
#define PER_TypeCheck(O) PyObject_TypeCheck((O), cPersistenceCAPI->pertype)
#define PER_USE_OR_RETURN(O,R) {if((O)->state==cPersistent_GHOST_STATE && cPersistenceCAPI->setstate((PyObject*)(O)) < 0) return (R); else if ((O)->state==cPersistent_UPTODATE_STATE) (O)->state=cPersistent_STICKY_STATE;}
#define PER_CHANGED(O) (cPersistenceCAPI->changed((cPersistentObject*)(O)))
#define PER_GHOSTIFY(O) (cPersistenceCAPI->ghostify((cPersistentObject*)(O)))
/* If the object is sticky, make it non-sticky, so that it can be ghostified.
The value is not meaningful
*/
#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE))
#define PER_PREVENT_DEACTIVATION(O) ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE))
/*
Make a persistent object usable from C by:
- Making sure it is not a ghost
- Making it sticky.
IMPORTANT: If you call this and don't call PER_ALLOW_DEACTIVATION,
your object will not be ghostified.
PER_USE returns a 1 on success and 0 failure, where failure means
error.
*/
#define PER_USE(O) \
(((O)->state != cPersistent_GHOST_STATE \
|| (cPersistenceCAPI->setstate((PyObject*)(O)) >= 0)) \
? (((O)->state==cPersistent_UPTODATE_STATE) \
? ((O)->state=cPersistent_STICKY_STATE) : 1) : 0)
#define PER_ACCESSED(O) (cPersistenceCAPI->accessed((cPersistentObject*)(O)))
#endif
ZODB/src/ZopeUndo/ 0000751 0001750 0000144 00000000000 10424632372 013517 5 ustar ctheune users ZODB/src/ZopeUndo/tests/ 0000751 0001750 0000144 00000000000 10424632372 014661 5 ustar ctheune users ZODB/src/ZopeUndo/tests/testPrefix.py 0000640 0001750 0000144 00000003265 10375443702 017400 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
from ZopeUndo.Prefix import Prefix
import unittest
class PrefixTest(unittest.TestCase):
def test(self):
p1 = Prefix("/a/b")
for equal in ("/a/b", "/a/b/c", "/a/b/c/d"):
self.assertEqual(p1, equal)
for notEqual in ("", "/a/c", "/a/bbb", "///"):
self.assertNotEqual(p1, notEqual)
p2 = Prefix("")
for equal in ("", "/", "/def", "/a/b", "/a/b/c", "/a/b/c/d"):
self.assertEqual(p2, equal)
def test_username_info(self):
# Zope Collector 1810; user paths have username appended
p1 = Prefix('/a/b')
for equal in ('/a/b spam', '/a/b/c spam', '/a/b/c/b spam'):
self.assertEqual(p1, equal)
for notEqual in (" spam", "/a/c spam", "/a/bbb spam", "/// spam"):
self.assertNotEqual(p1, notEqual)
p2 = Prefix("")
for equal in (" eggs", "/ eggs", "/def eggs", "/a/b eggs",
"/a/b/c eggs", "/a/b/c/d eggs"):
self.assertEqual(p2, equal)
def test_suite():
return unittest.makeSuite(PrefixTest)
ZODB/src/ZopeUndo/tests/__init__.py 0000640 0001750 0000144 00000001202 10057366465 016776 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
ZODB/src/ZopeUndo/__init__.py 0000640 0001750 0000144 00000001202 10057366465 015634 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
ZODB/src/ZopeUndo/Prefix.py 0000640 0001750 0000144 00000003456 10375443702 015340 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""ZODB undo support for Zope2.
This package is used to support the Prefix object that Zope uses for
undo. It is a separate package only to aid configuration management.
This package is included in Zope and ZODB3, so that ZODB3 is suitable
for running a ZEO server that handles Zope undo.
"""
class Prefix:
"""A Prefix() is equal to any path it is a prefix of.
This class can be compared to a string.
The comparison will return True if all path elements of the
Prefix are found at the beginning of the string being compared.
Two Prefixes can not be compared.
"""
__no_side_effects__ = 1
def __init__(self, path):
path_list = path.split('/')
self.length = len(path_list)
self.path = path_list
def __cmp__(self, o):
other_path = o.split('/')
if other_path and ' ' in other_path[-1]:
# don't include logged username in comparison
pos = other_path[-1].rfind(' ')
other_path[-1] = other_path[-1][:pos]
return cmp(other_path[:self.length], self.path)
def __repr__(self):
# makes failing tests easier to read
return "Prefix('%s')" % '/'.join(self.path)
ZODB/src/BTrees/ 0000751 0001750 0000144 00000000000 10424632372 013140 5 ustar ctheune users ZODB/src/BTrees/BucketTemplate.c 0000640 0001750 0000144 00000136473 10306324073 016226 0 ustar ctheune users /*****************************************************************************
Copyright (c) 2001, 2002 Zope Corporation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
#define BUCKETTEMPLATE_C "$Id: BucketTemplate.c 38291 2005-09-03 13:42:19Z alienoid $\n"
/* Use BUCKET_SEARCH to find the index at which a key belongs.
* INDEX An int lvalue to hold the index i such that KEY belongs at
* SELF->keys[i]. Note that this will equal SELF->len if KEY
* is larger than the bucket's largest key. Else it's the
* smallest i such that SELF->keys[i] >= KEY.
* ABSENT An int lvalue to hold a Boolean result, true (!= 0) if the
* key is absent, false (== 0) if the key is at INDEX.
* SELF A pointer to a Bucket node.
* KEY The key you're looking for, of type KEY_TYPE.
* ONERROR What to do if key comparison raises an exception; for example,
* perhaps 'return NULL'.
*
* See Maintainer.txt for discussion: this is optimized in subtle ways.
* It's recommended that you call this at the start of a routine, waiting
* to check for self->len == 0 after (if an empty bucket is special in
* context; INDEX becomes 0 and ABSENT becomes true if this macro is run
* with an empty SELF, and that may be all the invoker needs to know).
*/
#define BUCKET_SEARCH(INDEX, ABSENT, SELF, KEY, ONERROR) { \
int _lo = 0; \
int _hi = (SELF)->len; \
int _i; \
int _cmp = 1; \
for (_i = _hi >> 1; _lo < _hi; _i = (_lo + _hi) >> 1) { \
TEST_KEY_SET_OR(_cmp, (SELF)->keys[_i], (KEY)) \
ONERROR; \
if (_cmp < 0) _lo = _i + 1; \
else if (_cmp == 0) break; \
else _hi = _i; \
} \
(INDEX) = _i; \
(ABSENT) = _cmp; \
}
/*
** _bucket_get
**
** Search a bucket for a given key.
**
** Arguments
** self The bucket
** keyarg The key to look for
** has_key Boolean; if true, return a true/false result; else return
** the value associated with the key.
**
** Return
** If has_key:
** Returns the Python int 0 if the key is absent, else returns
** has_key itself as a Python int. A BTree caller generally passes
** the depth of the bucket for has_key, so a true result returns
** the bucket depth then.
** Note that has_key should be true when searching set buckets.
** If not has_key:
** If the key is present, returns the associated value, and the
** caller owns the reference. Else returns NULL and sets KeyError.
** Whether or not has_key:
** If a comparison sets an exception, returns NULL.
*/
static PyObject *
_bucket_get(Bucket *self, PyObject *keyarg, int has_key)
{
int i, cmp;
KEY_TYPE key;
PyObject *r = NULL;
int copied = 1;
COPY_KEY_FROM_ARG(key, keyarg, copied);
UNLESS (copied) return NULL;
UNLESS (PER_USE(self)) return NULL;
BUCKET_SEARCH(i, cmp, self, key, goto Done);
if (has_key)
r = PyInt_FromLong(cmp ? 0 : has_key);
else {
if (cmp == 0) {
COPY_VALUE_TO_OBJECT(r, self->values[i]);
}
else
PyErr_SetObject(PyExc_KeyError, keyarg);
}
Done:
PER_UNUSE(self);
return r;
}
static PyObject *
bucket_getitem(Bucket *self, PyObject *key)
{
return _bucket_get(self, key, 0);
}
/*
** Bucket_grow
**
** Resize a bucket.
**
** Arguments: self The bucket.
** newsize The new maximum capacity. If < 0, double the
** current size unless the bucket is currently empty,
** in which case use MIN_BUCKET_ALLOC.
** noval Boolean; if true, allocate only key space and not
** value space
**
** Returns: -1 on error, and MemoryError exception is set
** 0 on success
*/
static int
Bucket_grow(Bucket *self, int newsize, int noval)
{
KEY_TYPE *keys;
VALUE_TYPE *values;
if (self->size) {
if (newsize < 0)
newsize = self->size * 2;
if (newsize < 0) /* int overflow */
goto Overflow;
UNLESS (keys = BTree_Realloc(self->keys, sizeof(KEY_TYPE) * newsize))
return -1;
UNLESS (noval) {
values = BTree_Realloc(self->values, sizeof(VALUE_TYPE) * newsize);
if (values == NULL) {
free(keys);
return -1;
}
self->values = values;
}
self->keys = keys;
}
else {
if (newsize < 0)
newsize = MIN_BUCKET_ALLOC;
UNLESS (self->keys = BTree_Malloc(sizeof(KEY_TYPE) * newsize))
return -1;
UNLESS (noval) {
self->values = BTree_Malloc(sizeof(VALUE_TYPE) * newsize);
if (self->values == NULL) {
free(self->keys);
self->keys = NULL;
return -1;
}
}
}
self->size = newsize;
return 0;
Overflow:
PyErr_NoMemory();
return -1;
}
/* So far, bucket_append is called only by multiunion_m(), so is called
* only when MULTI_INT_UNION is defined. Flavors of BTree/Bucket that
* don't support MULTI_INT_UNION don't call bucket_append (yet), and
* gcc complains if bucket_append is compiled in those cases. So only
* compile bucket_append if it's going to be used.
*/
#ifdef MULTI_INT_UNION
/*
* Append a slice of the "from" bucket to self.
*
* self Append (at least keys) to this bucket. self must be activated
* upon entry, and remains activated at exit. If copyValues
* is true, self must be empty or already have a non-NULL values
* pointer. self's access and modification times aren't updated.
* from The bucket from which to take keys, and possibly values. from
* must be activated upon entry, and remains activated at exit.
* If copyValues is true, from must have a non-NULL values
* pointer. self and from must not be the same. from's access
* time isn't updated.
* i, n The slice from[i : i+n] is appended to self. Must have
* i >= 0, n > 0 and i+n <= from->len.
* copyValues Boolean. If true, copy values from the slice as well as keys.
* In this case, from must have a non-NULL values pointer, and
* self must too (unless self is empty, in which case a values
* vector will be allocated for it).
* overallocate Boolean. If self doesn't have enough room upon entry to hold
* all the appended stuff, then if overallocate is false exactly
* enough room will be allocated to hold the new stuff, else if
* overallocate is true an excess will be allocated. overallocate
* may be a good idea if you expect to append more stuff to self
* later; else overallocate should be false.
*
* CAUTION: If self is empty upon entry (self->size == 0), and copyValues is
* false, then no space for values will get allocated. This can be a trap if
* the caller intends to copy values itself.
*
* Return
* -1 Error.
* 0 OK.
*/
static int
bucket_append(Bucket *self, Bucket *from, int i, int n,
int copyValues, int overallocate)
{
int newlen;
assert(self && from && self != from);
assert(i >= 0);
assert(n > 0);
assert(i+n <= from->len);
/* Make room. */
newlen = self->len + n;
if (newlen > self->size) {
int newsize = newlen;
if (overallocate) /* boost by 25% -- pretty arbitrary */
newsize += newsize >> 2;
if (Bucket_grow(self, newsize, ! copyValues) < 0)
return -1;
}
assert(newlen <= self->size);
/* Copy stuff. */
memcpy(self->keys + self->len, from->keys + i, n * sizeof(KEY_TYPE));
if (copyValues) {
assert(self->values);
assert(from->values);
memcpy(self->values + self->len, from->values + i,
n * sizeof(VALUE_TYPE));
}
self->len = newlen;
/* Bump refcounts. */
#ifdef KEY_TYPE_IS_PYOBJECT
{
int j;
PyObject **p = from->keys + i;
for (j = 0; j < n; ++j, ++p) {
Py_INCREF(*p);
}
}
#endif
#ifdef VALUE_TYPE_IS_PYOBJECT
if (copyValues) {
int j;
PyObject **p = from->values + i;
for (j = 0; j < n; ++j, ++p) {
Py_INCREF(*p);
}
}
#endif
return 0;
}
#endif /* MULTI_INT_UNION */
/*
** _bucket_set: Assign a value to a key in a bucket, delete a key+value
** pair, or just insert a key.
**
** Arguments
** self The bucket
** keyarg The key to look for
** v The value to associate with key; NULL means delete the key.
** If NULL, it's an error (KeyError) if the key isn't present.
** Note that if this is a set bucket, and you want to insert
** a new set element, v must be non-NULL although its exact
** value will be ignored. Passing Py_None is good for this.
** unique Boolean; when true, don't replace the value if the key is
** already present.
** noval Boolean; when true, operate on keys only (ignore values)
** changed ignored on input
**
** Return
** -1 on error
** 0 on success and the # of bucket entries didn't change
** 1 on success and the # of bucket entries did change
** *changed If non-NULL, set to 1 on any mutation of the bucket.
*/
static int
_bucket_set(Bucket *self, PyObject *keyarg, PyObject *v,
int unique, int noval, int *changed)
{
int i, cmp;
KEY_TYPE key;
/* Subtle: there may or may not be a value. If there is, we need to
* check its type early, so that in case of error we can get out before
* mutating the bucket. But because value isn't used on all paths, if
* we don't initialize value then gcc gives a nuisance complaint that
* value may be used initialized (it can't be, but gcc doesn't know
* that). So we initialize it. However, VALUE_TYPE can be various types,
* including int, PyObject*, and char[6], so it's a puzzle to spell
* initialization. It so happens that {0} is a valid initializer for all
* these types.
*/
VALUE_TYPE value = {0}; /* squash nuisance warning */
int result = -1; /* until proven innocent */
int copied = 1;
COPY_KEY_FROM_ARG(key, keyarg, copied);
UNLESS(copied) return -1;
/* Copy the value early (if needed), so that in case of error a
* pile of bucket mutations don't need to be undone.
*/
if (v && !noval) {
COPY_VALUE_FROM_ARG(value, v, copied);
UNLESS(copied) return -1;
}
UNLESS (PER_USE(self)) return -1;
BUCKET_SEARCH(i, cmp, self, key, goto Done);
if (cmp == 0) {
/* The key exists, at index i. */
if (v) {
/* The key exists at index i, and there's a new value.
* If unique, we're not supposed to replace it. If noval, or this
* is a set bucket (self->values is NULL), there's nothing to do.
*/
if (unique || noval || self->values == NULL) {
result = 0;
goto Done;
}
/* The key exists at index i, and we need to replace the value. */
#ifdef VALUE_SAME
/* short-circuit if no change */
if (VALUE_SAME(self->values[i], value)) {
result = 0;
goto Done;
}
#endif
if (changed)
*changed = 1;
DECREF_VALUE(self->values[i]);
COPY_VALUE(self->values[i], value);
INCREF_VALUE(self->values[i]);
if (PER_CHANGED(self) >= 0)
result = 0;
goto Done;
}
/* The key exists at index i, and should be deleted. */
DECREF_KEY(self->keys[i]);
self->len--;
if (i < self->len)
memmove(self->keys + i, self->keys + i+1,
sizeof(KEY_TYPE)*(self->len - i));
if (self->values) {
DECREF_VALUE(self->values[i]);
if (i < self->len)
memmove(self->values + i, self->values + i+1,
sizeof(VALUE_TYPE)*(self->len - i));
}
if (! self->len) {
self->size = 0;
free(self->keys);
self->keys = NULL;
if (self->values) {
free(self->values);
self->values = NULL;
}
}
if (changed)
*changed = 1;
if (PER_CHANGED(self) >= 0)
result = 1;
goto Done;
}
/* The key doesn't exist, and belongs at index i. */
if (!v) {
/* Can't delete a non-existent key. */
PyErr_SetObject(PyExc_KeyError, keyarg);
goto Done;
}
/* The key doesn't exist and should be inserted at index i. */
if (self->len == self->size && Bucket_grow(self, -1, noval) < 0)
goto Done;
if (self->len > i) {
memmove(self->keys + i + 1, self->keys + i,
sizeof(KEY_TYPE) * (self->len - i));
if (self->values) {
memmove(self->values + i + 1, self->values + i,
sizeof(VALUE_TYPE) * (self->len - i));
}
}
COPY_KEY(self->keys[i], key);
INCREF_KEY(self->keys[i]);
if (! noval) {
COPY_VALUE(self->values[i], value);
INCREF_VALUE(self->values[i]);
}
self->len++;
if (changed)
*changed = 1;
if (PER_CHANGED(self) >= 0)
result = 1;
Done:
PER_UNUSE(self);
return result;
}
/*
** bucket_setitem
**
** wrapper for _bucket_setitem (eliminates +1 return code)
**
** Arguments: self The bucket
** key The key to insert under
** v The value to insert
**
** Returns 0 on success
** -1 on failure
*/
static int
bucket_setitem(Bucket *self, PyObject *key, PyObject *v)
{
if (_bucket_set(self, key, v, 0, 0, 0) < 0)
return -1;
return 0;
}
/**
** Accepts a sequence of 2-tuples, or any object with an items()
** method that returns an iterable object producing 2-tuples.
*/
static int
update_from_seq(PyObject *map, PyObject *seq)
{
PyObject *iter, *o, *k, *v;
int err = -1;
/* One path creates a new seq object. The other path has an
INCREF of the seq argument. So seq must always be DECREFed on
the way out.
*/
/* Use items() if it's not a sequence. Alas, PySequence_Check()
* returns true for a PeristentMapping or PersistentDict, and we
* want to use items() in those cases too.
*/
if (!PySequence_Check(seq) || /* or it "looks like a dict" */
PyObject_HasAttrString(seq, "iteritems")) {
PyObject *items;
items = PyObject_GetAttrString(seq, "items");
if (items == NULL)
return -1;
seq = PyObject_CallObject(items, NULL);
Py_DECREF(items);
if (seq == NULL)
return -1;
}
else
Py_INCREF(seq);
iter = PyObject_GetIter(seq);
if (iter == NULL)
goto err;
while (1) {
o = PyIter_Next(iter);
if (o == NULL) {
if (PyErr_Occurred())
goto err;
else
break;
}
if (!PyTuple_Check(o) || PyTuple_GET_SIZE(o) != 2) {
Py_DECREF(o);
PyErr_SetString(PyExc_TypeError,
"Sequence must contain 2-item tuples");
goto err;
}
k = PyTuple_GET_ITEM(o, 0);
v = PyTuple_GET_ITEM(o, 1);
if (PyObject_SetItem(map, k, v) < 0) {
Py_DECREF(o);
goto err;
}
Py_DECREF(o);
}
err = 0;
err:
Py_DECREF(iter);
Py_DECREF(seq);
return err;
}
static PyObject *
Mapping_update(PyObject *self, PyObject *seq)
{
if (update_from_seq(self, seq) < 0)
return NULL;
Py_INCREF(Py_None);
return Py_None;
}
/*
** bucket_split
**
** Splits one bucket into two
**
** Arguments: self The bucket
** index the index of the key to split at (O.O.B use midpoint)
** next the new bucket to split into
**
** Returns: 0 on success
** -1 on failure
*/
static int
bucket_split(Bucket *self, int index, Bucket *next)
{
int next_size;
ASSERT(self->len > 1, "split of empty bucket", -1);
if (index < 0 || index >= self->len)
index = self->len / 2;
next_size = self->len - index;
next->keys = BTree_Malloc(sizeof(KEY_TYPE) * next_size);
if (!next->keys)
return -1;
memcpy(next->keys, self->keys + index, sizeof(KEY_TYPE) * next_size);
if (self->values) {
next->values = BTree_Malloc(sizeof(VALUE_TYPE) * next_size);
if (!next->values) {
free(next->keys);
next->keys = NULL;
return -1;
}
memcpy(next->values, self->values + index,
sizeof(VALUE_TYPE) * next_size);
}
next->size = next_size;
next->len = next_size;
self->len = index;
next->next = self->next;
Py_INCREF(next);
self->next = next;
if (PER_CHANGED(self) < 0)
return -1;
return 0;
}
/* Set self->next to self->next->next, i.e. unlink self's successor from
* the chain.
*
* Return:
* -1 error
* 0 OK
*/
static int
Bucket_deleteNextBucket(Bucket *self)
{
int result = -1; /* until proven innocent */
Bucket *successor;
PER_USE_OR_RETURN(self, -1);
successor = self->next;
if (successor) {
Bucket *next;
/* Before: self -> successor -> next
* After: self --------------> next
*/
UNLESS (PER_USE(successor)) goto Done;
next = successor->next;
PER_UNUSE(successor);
Py_XINCREF(next); /* it may be NULL, of course */
self->next = next;
Py_DECREF(successor);
if (PER_CHANGED(self) < 0)
goto Done;
}
result = 0;
Done:
PER_UNUSE(self);
return result;
}
/*
Bucket_findRangeEnd -- Find the index of a range endpoint
(possibly) contained in a bucket.
Arguments: self The bucket
keyarg The key to match against
low Boolean; true for low end of range, false for high
exclude_equal Boolean; if true, don't accept an exact match,
and if there is one then move right if low and
left if !low.
offset The output offset
If low true, *offset <- index of the smallest item >= key,
if low false the index of the largest item <= key. In either case, if there
is no such index, *offset is left alone and 0 is returned.
Return:
0 No suitable index exists; *offset has not been changed
1 The correct index was stored into *offset
-1 Error
Example: Suppose the keys are [2, 4], and exclude_equal is false. Searching
for 2 sets *offset to 0 and returns 1 regardless of low. Searching for 4
sets *offset to 1 and returns 1 regardless of low.
Searching for 1:
If low true, sets *offset to 0, returns 1.
If low false, returns 0.
Searching for 3:
If low true, sets *offset to 1, returns 1.
If low false, sets *offset to 0, returns 1.
Searching for 5:
If low true, returns 0.
If low false, sets *offset to 1, returns 1.
The 1, 3 and 5 examples are the same when exclude_equal is true.
*/
static int
Bucket_findRangeEnd(Bucket *self, PyObject *keyarg, int low, int exclude_equal,
int *offset)
{
int i, cmp;
int result = -1; /* until proven innocent */
KEY_TYPE key;
int copied = 1;
COPY_KEY_FROM_ARG(key, keyarg, copied);
UNLESS (copied) return -1;
UNLESS (PER_USE(self)) return -1;
BUCKET_SEARCH(i, cmp, self, key, goto Done);
if (cmp == 0) {
/* exact match at index i */
if (exclude_equal) {
/* but we don't want an exact match */
if (low)
++i;
else
--i;
}
}
/* Else keys[i-1] < key < keys[i], picturing infinities at OOB indices,
* and i has the smallest item > key, which is correct for low.
*/
else if (! low)
/* i-1 has the largest item < key (unless i-1 is 0OB) */
--i;
result = 0 <= i && i < self->len;
if (result)
*offset = i;
Done:
PER_UNUSE(self);
return result;
}
static PyObject *
Bucket_maxminKey(Bucket *self, PyObject *args, int min)
{
PyObject *key=0;
int rc, offset;
int empty_bucket = 1;
if (args && ! PyArg_ParseTuple(args, "|O", &key)) return NULL;
PER_USE_OR_RETURN(self, NULL);
UNLESS (self->len) goto empty;
/* Find the low range */
if (key)
{
if ((rc = Bucket_findRangeEnd(self, key, min, 0, &offset)) <= 0)
{
if (rc < 0) return NULL;
empty_bucket = 0;
goto empty;
}
}
else if (min) offset = 0;
else offset = self->len -1;
COPY_KEY_TO_OBJECT(key, self->keys[offset]);
PER_UNUSE(self);
return key;
empty:
PyErr_SetString(PyExc_ValueError,
empty_bucket ? "empty bucket" :
"no key satisfies the conditions");
PER_UNUSE(self);
return NULL;
}
static PyObject *
Bucket_minKey(Bucket *self, PyObject *args)
{
return Bucket_maxminKey(self, args, 1);
}
static PyObject *
Bucket_maxKey(Bucket *self, PyObject *args)
{
return Bucket_maxminKey(self, args, 0);
}
static int
Bucket_rangeSearch(Bucket *self, PyObject *args, PyObject *kw,
int *low, int *high)
{
PyObject *min = Py_None;
PyObject *max = Py_None;
int excludemin = 0;
int excludemax = 0;
int rc;
if (args) {
if (! PyArg_ParseTupleAndKeywords(args, kw, "|OOii", search_keywords,
&min,
&max,
&excludemin,
&excludemax))
return -1;
}
UNLESS (self->len) goto empty;
/* Find the low range */
if (min != Py_None) {
rc = Bucket_findRangeEnd(self, min, 1, excludemin, low);
if (rc < 0)
return -1;
if (rc == 0)
goto empty;
}
else {
*low = 0;
if (excludemin) {
if (self->len < 2)
goto empty;
++*low;
}
}
/* Find the high range */
if (max != Py_None) {
rc = Bucket_findRangeEnd(self, max, 0, excludemax, high);
if (rc < 0)
return -1;
if (rc == 0)
goto empty;
}
else {
*high = self->len - 1;
if (excludemax) {
if (self->len < 2)
goto empty;
--*high;
}
}
/* If min < max to begin with, it's quite possible that low > high now. */
if (*low <= *high)
return 0;
empty:
*low = 0;
*high = -1;
return 0;
}
/*
** bucket_keys
**
** Generate a list of all keys in the bucket
**
** Arguments: self The Bucket
** args (unused)
**
** Returns: list of bucket keys
*/
static PyObject *
bucket_keys(Bucket *self, PyObject *args, PyObject *kw)
{
PyObject *r = NULL, *key;
int i, low, high;
PER_USE_OR_RETURN(self, NULL);
if (Bucket_rangeSearch(self, args, kw, &low, &high) < 0)
goto err;
r = PyList_New(high-low+1);
if (r == NULL)
goto err;
for (i=low; i <= high; i++) {
COPY_KEY_TO_OBJECT(key, self->keys[i]);
if (PyList_SetItem(r, i-low , key) < 0)
goto err;
}
PER_UNUSE(self);
return r;
err:
PER_UNUSE(self);
Py_XDECREF(r);
return NULL;
}
/*
** bucket_values
**
** Generate a list of all values in the bucket
**
** Arguments: self The Bucket
** args (unused)
**
** Returns list of values
*/
static PyObject *
bucket_values(Bucket *self, PyObject *args, PyObject *kw)
{
PyObject *r=0, *v;
int i, low, high;
PER_USE_OR_RETURN(self, NULL);
if (Bucket_rangeSearch(self, args, kw, &low, &high) < 0) goto err;
UNLESS (r=PyList_New(high-low+1)) goto err;
for (i=low; i <= high; i++)
{
COPY_VALUE_TO_OBJECT(v, self->values[i]);
UNLESS (v) goto err;
if (PyList_SetItem(r, i-low, v) < 0) goto err;
}
PER_UNUSE(self);
return r;
err:
PER_UNUSE(self);
Py_XDECREF(r);
return NULL;
}
/*
** bucket_items
**
** Returns a list of all items in a bucket
**
** Arguments: self The Bucket
** args (unused)
**
** Returns: list of all items in the bucket
*/
static PyObject *
bucket_items(Bucket *self, PyObject *args, PyObject *kw)
{
PyObject *r=0, *o=0, *item=0;
int i, low, high;
PER_USE_OR_RETURN(self, NULL);
if (Bucket_rangeSearch(self, args, kw, &low, &high) < 0) goto err;
UNLESS (r=PyList_New(high-low+1)) goto err;
for (i=low; i <= high; i++)
{
UNLESS (item = PyTuple_New(2)) goto err;
COPY_KEY_TO_OBJECT(o, self->keys[i]);
UNLESS (o) goto err;
PyTuple_SET_ITEM(item, 0, o);
COPY_VALUE_TO_OBJECT(o, self->values[i]);
UNLESS (o) goto err;
PyTuple_SET_ITEM(item, 1, o);
if (PyList_SetItem(r, i-low, item) < 0) goto err;
item = 0;
}
PER_UNUSE(self);
return r;
err:
PER_UNUSE(self);
Py_XDECREF(r);
Py_XDECREF(item);
return NULL;
}
static PyObject *
bucket_byValue(Bucket *self, PyObject *omin)
{
PyObject *r=0, *o=0, *item=0;
VALUE_TYPE min;
VALUE_TYPE v;
int i, l, copied=1;
PER_USE_OR_RETURN(self, NULL);
COPY_VALUE_FROM_ARG(min, omin, copied);
UNLESS(copied) return NULL;
for (i=0, l=0; i < self->len; i++)
if (TEST_VALUE(self->values[i], min) >= 0)
l++;
UNLESS (r=PyList_New(l)) goto err;
for (i=0, l=0; i < self->len; i++)
{
if (TEST_VALUE(self->values[i], min) < 0) continue;
UNLESS (item = PyTuple_New(2)) goto err;
COPY_KEY_TO_OBJECT(o, self->keys[i]);
UNLESS (o) goto err;
PyTuple_SET_ITEM(item, 1, o);
COPY_VALUE(v, self->values[i]);
NORMALIZE_VALUE(v, min);
COPY_VALUE_TO_OBJECT(o, v);
DECREF_VALUE(v);
UNLESS (o) goto err;
PyTuple_SET_ITEM(item, 0, o);
if (PyList_SetItem(r, l, item) < 0) goto err;
l++;
item = 0;
}
item=PyObject_GetAttr(r,sort_str);
UNLESS (item) goto err;
ASSIGN(item, PyObject_CallObject(item, NULL));
UNLESS (item) goto err;
ASSIGN(item, PyObject_GetAttr(r, reverse_str));
UNLESS (item) goto err;
ASSIGN(item, PyObject_CallObject(item, NULL));
UNLESS (item) goto err;
Py_DECREF(item);
PER_UNUSE(self);
return r;
err:
PER_UNUSE(self);
Py_XDECREF(r);
Py_XDECREF(item);
return NULL;
}
static int
_bucket_clear(Bucket *self)
{
const int len = self->len;
/* Don't declare i at this level. If neither keys nor values are
* PyObject*, i won't be referenced, and you'll get a nuisance compiler
* wng for declaring it here.
*/
self->len = self->size = 0;
if (self->next) {
Py_DECREF(self->next);
self->next = NULL;
}
/* Silence compiler warning about unused variable len for the case
when neither key nor value is an object, i.e. II. */
(void)len;
if (self->keys) {
#ifdef KEY_TYPE_IS_PYOBJECT
int i;
for (i = 0; i < len; ++i)
DECREF_KEY(self->keys[i]);
#endif
free(self->keys);
self->keys = NULL;
}
if (self->values) {
#ifdef VALUE_TYPE_IS_PYOBJECT
int i;
for (i = 0; i < len; ++i)
DECREF_VALUE(self->values[i]);
#endif
free(self->values);
self->values = NULL;
}
return 0;
}
#ifdef PERSISTENT
static PyObject *
bucket__p_deactivate(Bucket *self, PyObject *args, PyObject *keywords)
{
int ghostify = 1;
PyObject *force = NULL;
if (args && PyTuple_GET_SIZE(args) > 0) {
PyErr_SetString(PyExc_TypeError,
"_p_deactivate takes not positional arguments");
return NULL;
}
if (keywords) {
int size = PyDict_Size(keywords);
force = PyDict_GetItemString(keywords, "force");
if (force)
size--;
if (size) {
PyErr_SetString(PyExc_TypeError,
"_p_deactivate only accepts keyword arg force");
return NULL;
}
}
if (self->jar && self->oid) {
ghostify = self->state == cPersistent_UPTODATE_STATE;
if (!ghostify && force) {
if (PyObject_IsTrue(force))
ghostify = 1;
if (PyErr_Occurred())
return NULL;
}
if (ghostify) {
if (_bucket_clear(self) < 0)
return NULL;
PER_GHOSTIFY(self);
}
}
Py_INCREF(Py_None);
return Py_None;
}
#endif
static PyObject *
bucket_clear(Bucket *self, PyObject *args)
{
PER_USE_OR_RETURN(self, NULL);
if (self->len) {
if (_bucket_clear(self) < 0)
return NULL;
if (PER_CHANGED(self) < 0)
goto err;
}
PER_UNUSE(self);
Py_INCREF(Py_None);
return Py_None;
err:
PER_UNUSE(self);
return NULL;
}
/*
* Return:
*
* For a set bucket (self->values is NULL), a one-tuple or two-tuple. The
* first element is a tuple of keys, of length self->len. The second element
* is the next bucket, present if and only if next is non-NULL:
*
* (
* (keys[0], keys[1], ..., keys[len-1]),
* next iff non-NULL>
* )
*
* For a mapping bucket (self->values is not NULL), a one-tuple or two-tuple.
* The first element is a tuple interleaving keys and values, of length
* 2 * self->len. The second element is the next bucket, present iff next is
* non-NULL:
*
* (
* (keys[0], values[0], keys[1], values[1], ...,
* keys[len-1], values[len-1]),
* next iff non-NULL>
* )
*/
static PyObject *
bucket_getstate(Bucket *self)
{
PyObject *o = NULL, *items = NULL, *state;
int i, len, l;
PER_USE_OR_RETURN(self, NULL);
len = self->len;
if (self->values) { /* Bucket */
items = PyTuple_New(len * 2);
if (items == NULL)
goto err;
for (i = 0, l = 0; i < len; i++) {
COPY_KEY_TO_OBJECT(o, self->keys[i]);
if (o == NULL)
goto err;
PyTuple_SET_ITEM(items, l, o);
l++;
COPY_VALUE_TO_OBJECT(o, self->values[i]);
if (o == NULL)
goto err;
PyTuple_SET_ITEM(items, l, o);
l++;
}
} else { /* Set */
items = PyTuple_New(len);
if (items == NULL)
goto err;
for (i = 0; i < len; i++) {
COPY_KEY_TO_OBJECT(o, self->keys[i]);
if (o == NULL)
goto err;
PyTuple_SET_ITEM(items, i, o);
}
}
if (self->next)
state = Py_BuildValue("OO", items, self->next);
else
state = Py_BuildValue("(O)", items);
Py_DECREF(items);
PER_UNUSE(self);
return state;
err:
PER_UNUSE(self);
Py_XDECREF(items);
return NULL;
}
static int
_bucket_setstate(Bucket *self, PyObject *state)
{
PyObject *k, *v, *items;
Bucket *next = NULL;
int i, l, len, copied=1;
KEY_TYPE *keys;
VALUE_TYPE *values;
if (!PyArg_ParseTuple(state, "O|O:__setstate__", &items, &next))
return -1;
len = PyTuple_Size(items);
if (len < 0)
return -1;
len /= 2;
for (i = self->len; --i >= 0; ) {
DECREF_KEY(self->keys[i]);
DECREF_VALUE(self->values[i]);
}
self->len = 0;
if (self->next) {
Py_DECREF(self->next);
self->next = NULL;
}
if (len > self->size) {
keys = BTree_Realloc(self->keys, sizeof(KEY_TYPE)*len);
if (keys == NULL)
return -1;
values = BTree_Realloc(self->values, sizeof(VALUE_TYPE)*len);
if (values == NULL)
return -1;
self->keys = keys;
self->values = values;
self->size = len;
}
for (i=0, l=0; i < len; i++) {
k = PyTuple_GET_ITEM(items, l);
l++;
v = PyTuple_GET_ITEM(items, l);
l++;
COPY_KEY_FROM_ARG(self->keys[i], k, copied);
if (!copied)
return -1;
COPY_VALUE_FROM_ARG(self->values[i], v, copied);
if (!copied)
return -1;
INCREF_KEY(self->keys[i]);
INCREF_VALUE(self->values[i]);
}
self->len = len;
if (next) {
self->next = next;
Py_INCREF(next);
}
return 0;
}
static PyObject *
bucket_setstate(Bucket *self, PyObject *state)
{
int r;
PER_PREVENT_DEACTIVATION(self);
r = _bucket_setstate(self, state);
PER_UNUSE(self);
if (r < 0)
return NULL;
Py_INCREF(Py_None);
return Py_None;
}
static PyObject *
bucket_has_key(Bucket *self, PyObject *key)
{
return _bucket_get(self, key, 1);
}
static PyObject *
bucket_setdefault(Bucket *self, PyObject *args)
{
PyObject *key;
PyObject *failobj; /* default */
PyObject *value; /* return value */
int dummy_changed; /* in order to call _bucket_set */
if (! PyArg_UnpackTuple(args, "setdefault", 2, 2, &key, &failobj))
return NULL;
value = _bucket_get(self, key, 0);
if (value != NULL)
return value;
/* The key isn't in the bucket. If that's not due to a KeyError exception,
* pass back the unexpected exception.
*/
if (! PyErr_ExceptionMatches(PyExc_KeyError))
return NULL;
PyErr_Clear();
/* Associate `key` with `failobj` in the bucket, and return `failobj`. */
value = failobj;
if (_bucket_set(self, key, failobj, 0, 0, &dummy_changed) < 0)
value = NULL;
Py_XINCREF(value);
return value;
}
/* forward declaration */
static int
Bucket_length(Bucket *self);
static PyObject *
bucket_pop(Bucket *self, PyObject *args)
{
PyObject *key;
PyObject *failobj = NULL; /* default */
PyObject *value; /* return value */
int dummy_changed; /* in order to call _bucket_set */
if (! PyArg_UnpackTuple(args, "pop", 1, 2, &key, &failobj))
return NULL;
value = _bucket_get(self, key, 0);
if (value != NULL) {
/* Delete key and associated value. */
if (_bucket_set(self, key, NULL, 0, 0, &dummy_changed) < 0) {
Py_DECREF(value);
return NULL;
}
return value;
}
/* The key isn't in the bucket. If that's not due to a KeyError exception,
* pass back the unexpected exception.
*/
if (! PyErr_ExceptionMatches(PyExc_KeyError))
return NULL;
if (failobj != NULL) {
/* Clear the KeyError and return the explicit default. */
PyErr_Clear();
Py_INCREF(failobj);
return failobj;
}
/* No default given. The only difference in this case is the error
* message, which depends on whether the bucket is empty.
*/
if (Bucket_length(self) == 0)
PyErr_SetString(PyExc_KeyError, "pop(): Bucket is empty");
return NULL;
}
/* Search bucket self for key. This is the sq_contains slot of the
* PySequenceMethods.
*
* Return:
* -1 error
* 0 not found
* 1 found
*/
static int
bucket_contains(Bucket *self, PyObject *key)
{
PyObject *asobj = _bucket_get(self, key, 1);
int result = -1;
if (asobj != NULL) {
result = PyInt_AsLong(asobj) ? 1 : 0;
Py_DECREF(asobj);
}
return result;
}
/*
** bucket_getm
**
*/
static PyObject *
bucket_getm(Bucket *self, PyObject *args)
{
PyObject *key, *d=Py_None, *r;
if (!PyArg_ParseTuple(args, "O|O:get", &key, &d))
return NULL;
r = _bucket_get(self, key, 0);
if (r)
return r;
if (!PyErr_ExceptionMatches(PyExc_KeyError))
return NULL;
PyErr_Clear();
Py_INCREF(d);
return d;
}
/**************************************************************************/
/* Iterator support. */
/* A helper to build all the iterators for Buckets and Sets.
* If args is NULL, the iterator spans the entire structure. Else it's an
* argument tuple, with optional low and high arguments.
* kind is 'k', 'v' or 'i'.
* Returns a BTreeIter object, or NULL if error.
*/
static PyObject *
buildBucketIter(Bucket *self, PyObject *args, PyObject *kw, char kind)
{
BTreeItems *items;
int lowoffset, highoffset;
BTreeIter *result = NULL;
PER_USE_OR_RETURN(self, NULL);
if (Bucket_rangeSearch(self, args, kw, &lowoffset, &highoffset) < 0)
goto Done;
items = (BTreeItems *)newBTreeItems(kind, self, lowoffset,
self, highoffset);
if (items == NULL) goto Done;
result = BTreeIter_new(items); /* win or lose, we're done */
Py_DECREF(items);
Done:
PER_UNUSE(self);
return (PyObject *)result;
}
/* The implementation of iter(Bucket_or_Set); the Bucket tp_iter slot. */
static PyObject *
Bucket_getiter(Bucket *self)
{
return buildBucketIter(self, NULL, NULL, 'k');
}
/* The implementation of Bucket.iterkeys(). */
static PyObject *
Bucket_iterkeys(Bucket *self, PyObject *args, PyObject *kw)
{
return buildBucketIter(self, args, kw, 'k');
}
/* The implementation of Bucket.itervalues(). */
static PyObject *
Bucket_itervalues(Bucket *self, PyObject *args, PyObject *kw)
{
return buildBucketIter(self, args, kw, 'v');
}
/* The implementation of Bucket.iteritems(). */
static PyObject *
Bucket_iteritems(Bucket *self, PyObject *args, PyObject *kw)
{
return buildBucketIter(self, args, kw, 'i');
}
/* End of iterator support. */
#ifdef PERSISTENT
static PyObject *merge_error(int p1, int p2, int p3, int reason);
static PyObject *bucket_merge(Bucket *s1, Bucket *s2, Bucket *s3);
static PyObject *
_bucket__p_resolveConflict(PyObject *ob_type, PyObject *s[3])
{
PyObject *result = NULL; /* guilty until proved innocent */
Bucket *b[3] = {NULL, NULL, NULL};
PyObject *meth = NULL;
PyObject *a = NULL;
int i;
for (i = 0; i < 3; i++) {
PyObject *r;
b[i] = (Bucket*)PyObject_CallObject((PyObject *)ob_type, NULL);
if (b[i] == NULL)
goto Done;
if (s[i] == Py_None) /* None is equivalent to empty, for BTrees */
continue;
meth = PyObject_GetAttr((PyObject *)b[i], __setstate___str);
if (meth == NULL)
goto Done;
a = PyTuple_New(1);
if (a == NULL)
goto Done;
PyTuple_SET_ITEM(a, 0, s[i]);
Py_INCREF(s[i]);
r = PyObject_CallObject(meth, a); /* b[i].__setstate__(s[i]) */
if (r == NULL)
goto Done;
Py_DECREF(r);
Py_DECREF(a);
Py_DECREF(meth);
a = meth = NULL;
}
if (b[0]->next != b[1]->next || b[0]->next != b[2]->next)
merge_error(-1, -1, -1, 0);
else
result = bucket_merge(b[0], b[1], b[2]);
Done:
Py_XDECREF(meth);
Py_XDECREF(a);
Py_XDECREF(b[0]);
Py_XDECREF(b[1]);
Py_XDECREF(b[2]);
return result;
}
static PyObject *
bucket__p_resolveConflict(Bucket *self, PyObject *args)
{
PyObject *s[3];
if (!PyArg_ParseTuple(args, "OOO", &s[0], &s[1], &s[2]))
return NULL;
return _bucket__p_resolveConflict((PyObject *)self->ob_type, s);
}
#endif
/* Caution: Even though the _next attribute is read-only, a program could
do arbitrary damage to the btree internals. For example, it could call
clear() on a bucket inside a BTree.
We need to decide if the convenience for inspecting BTrees is worth
the risk.
*/
static struct PyMemberDef Bucket_members[] = {
{"_next", T_OBJECT, offsetof(Bucket, next)},
{NULL}
};
static struct PyMethodDef Bucket_methods[] = {
{"__getstate__", (PyCFunction) bucket_getstate, METH_NOARGS,
"__getstate__() -- Return the picklable state of the object"},
{"__setstate__", (PyCFunction) bucket_setstate, METH_O,
"__setstate__() -- Set the state of the object"},
{"keys", (PyCFunction) bucket_keys, METH_KEYWORDS,
"keys([min, max]) -- Return the keys"},
{"has_key", (PyCFunction) bucket_has_key, METH_O,
"has_key(key) -- Test whether the bucket contains the given key"},
{"clear", (PyCFunction) bucket_clear, METH_VARARGS,
"clear() -- Remove all of the items from the bucket"},
{"update", (PyCFunction) Mapping_update, METH_O,
"update(collection) -- Add the items from the given collection"},
{"maxKey", (PyCFunction) Bucket_maxKey, METH_VARARGS,
"maxKey([key]) -- Find the maximum key\n\n"
"If an argument is given, find the maximum <= the argument"},
{"minKey", (PyCFunction) Bucket_minKey, METH_VARARGS,
"minKey([key]) -- Find the minimum key\n\n"
"If an argument is given, find the minimum >= the argument"},
{"values", (PyCFunction) bucket_values, METH_KEYWORDS,
"values([min, max]) -- Return the values"},
{"items", (PyCFunction) bucket_items, METH_KEYWORDS,
"items([min, max])) -- Return the items"},
{"byValue", (PyCFunction) bucket_byValue, METH_O,
"byValue(min) -- "
"Return value-keys with values >= min and reverse sorted by values"},
{"get", (PyCFunction) bucket_getm, METH_VARARGS,
"get(key[,default]) -- Look up a value\n\n"
"Return the default (or None) if the key is not found."},
{"setdefault", (PyCFunction) bucket_setdefault, METH_VARARGS,
"D.setdefault(k, d) -> D.get(k, d), also set D[k]=d if k not in D.\n\n"
"Return the value like get() except that if key is missing, d is both\n"
"returned and inserted into the bucket as the value of k."},
{"pop", (PyCFunction) bucket_pop, METH_VARARGS,
"D.pop(k[, d]) -> v, remove key and return the corresponding value.\n\n"
"If key is not found, d is returned if given, otherwise KeyError\n"
"is raised."},
{"iterkeys", (PyCFunction) Bucket_iterkeys, METH_KEYWORDS,
"B.iterkeys([min[,max]]) -> an iterator over the keys of B"},
{"itervalues", (PyCFunction) Bucket_itervalues, METH_KEYWORDS,
"B.itervalues([min[,max]]) -> an iterator over the values of B"},
{"iteritems", (PyCFunction) Bucket_iteritems, METH_KEYWORDS,
"B.iteritems([min[,max]]) -> an iterator over the (key, value) items of B"},
#ifdef PERSISTENT
{"_p_resolveConflict", (PyCFunction) bucket__p_resolveConflict,
METH_VARARGS,
"_p_resolveConflict() -- Reinitialize from a newly created copy"},
{"_p_deactivate", (PyCFunction) bucket__p_deactivate, METH_KEYWORDS,
"_p_deactivate() -- Reinitialize from a newly created copy"},
#endif
{NULL, NULL}
};
static int
Bucket_init(PyObject *self, PyObject *args, PyObject *kwds)
{
PyObject *v = NULL;
if (!PyArg_ParseTuple(args, "|O:" MOD_NAME_PREFIX "Bucket", &v))
return -1;
if (v)
return update_from_seq(self, v);
else
return 0;
}
static void
bucket_dealloc(Bucket *self)
{
if (self->state != cPersistent_GHOST_STATE)
_bucket_clear(self);
cPersistenceCAPI->pertype->tp_dealloc((PyObject *)self);
}
static int
bucket_traverse(Bucket *self, visitproc visit, void *arg)
{
int err = 0;
int i, len;
#define VISIT(SLOT) \
if (SLOT) { \
err = visit((PyObject *)(SLOT), arg); \
if (err) \
goto Done; \
}
/* Call our base type's traverse function. Because buckets are
* subclasses of Peristent, there must be one.
*/
err = cPersistenceCAPI->pertype->tp_traverse((PyObject *)self, visit, arg);
if (err)
goto Done;
/* If this is registered with the persistence system, cleaning up cycles
* is the database's problem. It would be horrid to unghostify buckets
* here just to chase pointers every time gc runs.
*/
if (self->state == cPersistent_GHOST_STATE)
goto Done;
len = self->len;
(void)i; /* if neither keys nor values are PyObject*, "i" is otherwise
unreferenced and we get a nuisance compiler wng */
#ifdef KEY_TYPE_IS_PYOBJECT
/* Keys are Python objects so need to be traversed. */
for (i = 0; i < len; i++)
VISIT(self->keys[i]);
#endif
#ifdef VALUE_TYPE_IS_PYOBJECT
if (self->values != NULL) {
/* self->values exists (this is a mapping bucket, not a set bucket),
* and are Python objects, so need to be traversed. */
for (i = 0; i < len; i++)
VISIT(self->values[i]);
}
#endif
VISIT(self->next);
Done:
return err;
#undef VISIT
}
static int
bucket_tp_clear(Bucket *self)
{
if (self->state != cPersistent_GHOST_STATE)
_bucket_clear(self);
return 0;
}
/* Code to access Bucket objects as mappings */
static int
Bucket_length( Bucket *self)
{
int r;
UNLESS (PER_USE(self)) return -1;
r = self->len;
PER_UNUSE(self);
return r;
}
static PyMappingMethods Bucket_as_mapping = {
(inquiry)Bucket_length, /*mp_length*/
(binaryfunc)bucket_getitem, /*mp_subscript*/
(objobjargproc)bucket_setitem, /*mp_ass_subscript*/
};
static PySequenceMethods Bucket_as_sequence = {
(inquiry)0, /* sq_length */
(binaryfunc)0, /* sq_concat */
(intargfunc)0, /* sq_repeat */
(intargfunc)0, /* sq_item */
(intintargfunc)0, /* sq_slice */
(intobjargproc)0, /* sq_ass_item */
(intintobjargproc)0, /* sq_ass_slice */
(objobjproc)bucket_contains, /* sq_contains */
0, /* sq_inplace_concat */
0, /* sq_inplace_repeat */
};
static PyObject *
bucket_repr(Bucket *self)
{
PyObject *i, *r;
char repr[10000];
int rv;
i = bucket_items(self, NULL, NULL);
if (!i)
return NULL;
r = PyObject_Repr(i);
Py_DECREF(i);
if (!r) {
return NULL;
}
rv = PyOS_snprintf(repr, sizeof(repr),
"%s(%s)", self->ob_type->tp_name,
PyString_AS_STRING(r));
if (rv > 0 && rv < sizeof(repr)) {
Py_DECREF(r);
return PyString_FromStringAndSize(repr, strlen(repr));
}
else {
/* The static buffer wasn't big enough */
int size;
PyObject *s;
/* 3 for the parens and the null byte */
size = strlen(self->ob_type->tp_name) + PyString_GET_SIZE(r) + 3;
s = PyString_FromStringAndSize(NULL, size);
if (!s) {
Py_DECREF(r);
return r;
}
PyOS_snprintf(PyString_AS_STRING(s), size,
"%s(%s)", self->ob_type->tp_name, PyString_AS_STRING(r));
Py_DECREF(r);
return s;
}
}
static PyTypeObject BucketType = {
PyObject_HEAD_INIT(NULL) /* PyPersist_Type */
0, /* ob_size */
MODULE_NAME MOD_NAME_PREFIX "Bucket",/* tp_name */
sizeof(Bucket), /* tp_basicsize */
0, /* tp_itemsize */
(destructor)bucket_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_compare */
(reprfunc)bucket_repr, /* tp_repr */
0, /* tp_as_number */
&Bucket_as_sequence, /* tp_as_sequence */
&Bucket_as_mapping, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
Py_TPFLAGS_BASETYPE, /* tp_flags */
0, /* tp_doc */
(traverseproc)bucket_traverse, /* tp_traverse */
(inquiry)bucket_tp_clear, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
(getiterfunc)Bucket_getiter, /* tp_iter */
0, /* tp_iternext */
Bucket_methods, /* tp_methods */
Bucket_members, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
Bucket_init, /* tp_init */
0, /* tp_alloc */
0, /*PyType_GenericNew,*/ /* tp_new */
};
static int
nextBucket(SetIteration *i)
{
if (i->position >= 0)
{
UNLESS(PER_USE(BUCKET(i->set))) return -1;
if (i->position)
{
DECREF_KEY(i->key);
DECREF_VALUE(i->value);
}
if (i->position < BUCKET(i->set)->len)
{
COPY_KEY(i->key, BUCKET(i->set)->keys[i->position]);
INCREF_KEY(i->key);
COPY_VALUE(i->value, BUCKET(i->set)->values[i->position]);
INCREF_VALUE(i->value);
i->position ++;
}
else
{
i->position = -1;
PER_ACCESSED(BUCKET(i->set));
}
PER_ALLOW_DEACTIVATION(BUCKET(i->set));
}
return 0;
}
ZODB/src/BTrees/__init__.py 0000640 0001750 0000144 00000000034 10113174703 015240 0 ustar ctheune users # This is a Python package.
ZODB/src/BTrees/_OOBTree.c 0000640 0001750 0000144 00000002130 10057366465 014707 0 ustar ctheune users /*############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
############################################################################*/
#define MASTER_ID "$Id: _OOBTree.c 25186 2004-06-02 15:07:33Z jim $\n"
/* OOBTree - object key, object value BTree
Implements a collection using object type keys
and object type values
*/
#define PERSISTENT
#define MOD_NAME_PREFIX "OO"
#define INITMODULE init_OOBTree
#define DEFAULT_MAX_BUCKET_SIZE 30
#define DEFAULT_MAX_BTREE_SIZE 250
#include "objectkeymacros.h"
#include "objectvaluemacros.h"
#include "BTreeModuleTemplate.c"
ZODB/src/BTrees/IOBTree.py 0000640 0001750 0000144 00000001307 10057366465 014755 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# hack to overcome dynamic-linking headache.
from _IOBTree import *
ZODB/src/BTrees/tests/ 0000751 0001750 0000144 00000000000 10424632372 014302 5 ustar ctheune users ZODB/src/BTrees/tests/test_btreesubclass.py 0000640 0001750 0000144 00000002661 10214421637 020556 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
from BTrees.OOBTree import OOBTree, OOBucket
class B(OOBucket):
pass
class T(OOBTree):
_bucket_type = B
import unittest
class SubclassTest(unittest.TestCase):
def testSubclass(self):
# test that a subclass that defines _bucket_type gets buckets
# of that type
t = T()
# There's no good way to get a bucket at the moment.
# __getstate__() is as good as it gets, but the default
# getstate explicitly includes the pickle of the bucket
# for small trees, so we have to be clever :-(
# make sure there is more than one bucket in the tree
for i in range(1000):
t[i] = i
state = t.__getstate__()
self.assert_(state[0][0].__class__ is B)
def test_suite():
return unittest.makeSuite(SubclassTest)
ZODB/src/BTrees/tests/testBTreesUnicode.py 0000640 0001750 0000144 00000004653 10057366465 020270 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import unittest
from BTrees.OOBTree import OOBTree
# When an OOBtree contains unicode strings as keys,
# it is neccessary accessing non-unicode strings are
# either ascii strings or encoded as unicoded using the
# corresponding encoding
encoding = 'ISO-8859-1'
class TestBTreesUnicode(unittest.TestCase):
""" test unicode"""
def setUp(self):
"""setup an OOBTree with some unicode strings"""
self.s = unicode('dreit\xe4gigen', 'latin1')
self.data = [('alien', 1),
('k\xf6nnten', 2),
('fox', 3),
('future', 4),
('quick', 5),
('zerst\xf6rt', 6),
(unicode('dreit\xe4gigen','latin1'), 7),
]
self.tree = OOBTree()
for k, v in self.data:
if isinstance(k, str):
k = unicode(k, 'latin1')
self.tree[k] = v
def testAllKeys(self):
# check every item of the tree
for k, v in self.data:
if isinstance(k, str):
k = unicode(k, encoding)
self.assert_(self.tree.has_key(k))
self.assertEqual(self.tree[k], v)
def testUnicodeKeys(self):
# try to access unicode keys in tree
k, v = self.data[-1]
self.assertEqual(k, self.s)
self.assertEqual(self.tree[k], v)
self.assertEqual(self.tree[self.s], v)
def testAsciiKeys(self):
# try to access some "plain ASCII" keys in the tree
for k, v in self.data[0], self.data[2]:
self.assert_(isinstance(k, str))
self.assertEqual(self.tree[k], v)
def test_suite():
return unittest.makeSuite(TestBTreesUnicode)
def main():
unittest.TextTestRunner().run(test_suite())
if __name__ == '__main__':
main()
ZODB/src/BTrees/tests/test_check.py 0000640 0001750 0000144 00000006651 10057366465 017011 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Test the BTree check.check() function."""
import unittest
from BTrees.OOBTree import OOBTree
from BTrees.check import check
class CheckTest(unittest.TestCase):
def setUp(self):
self.t = t = OOBTree()
for i in range(31):
t[i] = 2*i
self.state = t.__getstate__()
def testNormal(self):
s = self.state
# Looks like (state, first_bucket)
# where state looks like (bucket0, 15, bucket1).
self.assertEqual(len(s), 2)
self.assertEqual(len(s[0]), 3)
self.assertEqual(s[0][1], 15)
self.t._check() # shouldn't blow up
check(self.t) # shouldn't blow up
def testKeyTooLarge(self):
# Damage an invariant by dropping the BTree key to 14.
s = self.state
news = (s[0][0], 14, s[0][2]), s[1]
self.t.__setstate__(news)
self.t._check() # not caught
try:
# Expecting "... key %r >= upper bound %r at index %d"
check(self.t)
except AssertionError, detail:
self.failUnless(str(detail).find(">= upper bound") > 0)
else:
self.fail("expected self.t_check() to catch the problem")
def testKeyTooSmall(self):
# Damage an invariant by bumping the BTree key to 16.
s = self.state
news = (s[0][0], 16, s[0][2]), s[1]
self.t.__setstate__(news)
self.t._check() # not caught
try:
# Expecting "... key %r < lower bound %r at index %d"
check(self.t)
except AssertionError, detail:
self.failUnless(str(detail).find("< lower bound") > 0)
else:
self.fail("expected self.t_check() to catch the problem")
def testKeysSwapped(self):
# Damage an invariant by swapping two key/value pairs.
s = self.state
# Looks like (state, first_bucket)
# where state looks like (bucket0, 15, bucket1).
(b0, num, b1), firstbucket = s
self.assertEqual(b0[4], 8)
self.assertEqual(b0[5], 10)
b0state = b0.__getstate__()
self.assertEqual(len(b0state), 2)
# b0state looks like
# ((k0, v0, k1, v1, ...), nextbucket)
pairs, nextbucket = b0state
self.assertEqual(pairs[8], 4)
self.assertEqual(pairs[9], 8)
self.assertEqual(pairs[10], 5)
self.assertEqual(pairs[11], 10)
newpairs = pairs[:8] + (5, 10, 4, 8) + pairs[12:]
b0.__setstate__((newpairs, nextbucket))
self.t._check() # not caught
try:
check(self.t)
except AssertionError, detail:
self.failUnless(str(detail).find(
"key 5 at index 4 >= key 4 at index 5") > 0)
else:
self.fail("expected self.t_check() to catch the problem")
def test_suite():
return unittest.makeSuite(CheckTest)
ZODB/src/BTrees/tests/testSetOps.py 0000640 0001750 0000144 00000042414 10057366465 017007 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
from unittest import TestCase, TestSuite, TextTestRunner, makeSuite
from BTrees.OOBTree import OOBTree, OOBucket, OOSet, OOTreeSet
from BTrees.IOBTree import IOBTree, IOBucket, IOSet, IOTreeSet
from BTrees.IIBTree import IIBTree, IIBucket, IISet, IITreeSet
from BTrees.OIBTree import OIBTree, OIBucket, OISet, OITreeSet
# Subclasses have to set up:
# builders - functions to build inputs, taking an optional keys arg
# intersection, union, difference - set to the type-correct versions
class SetResult(TestCase):
def setUp(self):
self.Akeys = [1, 3, 5, 6 ]
self.Bkeys = [ 2, 3, 4, 6, 7]
self.As = [makeset(self.Akeys) for makeset in self.builders]
self.Bs = [makeset(self.Bkeys) for makeset in self.builders]
self.emptys = [makeset() for makeset in self.builders]
# Slow but obviously correct Python implementations of basic ops.
def _union(self, x, y):
result = list(x)
for e in y:
if e not in result:
result.append(e)
result.sort()
return result
def _intersection(self, x, y):
result = []
for e in x:
if e in y:
result.append(e)
return result
def _difference(self, x, y):
result = list(x)
for e in y:
if e in result:
result.remove(e)
# Difference preserves LHS values.
if hasattr(x, "values"):
result = [(k, x[k]) for k in result]
return result
def testNone(self):
for op in self.union, self.intersection, self.difference:
C = op(None, None)
self.assert_(C is None)
for op in self.union, self.intersection, self.difference:
for A in self.As:
C = op(A, None)
self.assert_(C is A)
C = op(None, A)
if op is self.difference:
self.assert_(C is None)
else:
self.assert_(C is A)
def testEmptyUnion(self):
for A in self.As:
for E in self.emptys:
C = self.union(A, E)
self.assert_(not hasattr(C, "values"))
self.assertEqual(list(C), self.Akeys)
C = self.union(E, A)
self.assert_(not hasattr(C, "values"))
self.assertEqual(list(C), self.Akeys)
def testEmptyIntersection(self):
for A in self.As:
for E in self.emptys:
C = self.intersection(A, E)
self.assert_(not hasattr(C, "values"))
self.assertEqual(list(C), [])
C = self.intersection(E, A)
self.assert_(not hasattr(C, "values"))
self.assertEqual(list(C), [])
def testEmptyDifference(self):
for A in self.As:
for E in self.emptys:
C = self.difference(A, E)
# Difference preserves LHS values.
self.assertEqual(hasattr(C, "values"), hasattr(A, "values"))
if hasattr(A, "values"):
self.assertEqual(list(C.items()), list(A.items()))
else:
self.assertEqual(list(C), self.Akeys)
C = self.difference(E, A)
self.assertEqual(hasattr(C, "values"), hasattr(E, "values"))
self.assertEqual(list(C), [])
def testUnion(self):
inputs = self.As + self.Bs
for A in inputs:
for B in inputs:
C = self.union(A, B)
self.assert_(not hasattr(C, "values"))
self.assertEqual(list(C), self._union(A, B))
def testIntersection(self):
inputs = self.As + self.Bs
for A in inputs:
for B in inputs:
C = self.intersection(A, B)
self.assert_(not hasattr(C, "values"))
self.assertEqual(list(C), self._intersection(A, B))
def testDifference(self):
inputs = self.As + self.Bs
for A in inputs:
for B in inputs:
C = self.difference(A, B)
# Difference preserves LHS values.
self.assertEqual(hasattr(C, "values"), hasattr(A, "values"))
want = self._difference(A, B)
if hasattr(A, "values"):
self.assertEqual(list(C.items()), want)
else:
self.assertEqual(list(C), want)
def testLargerInputs(self):
from random import randint
MAXSIZE = 200
MAXVAL = 400
for i in range(3):
n = randint(0, MAXSIZE)
Akeys = [randint(1, MAXVAL) for j in range(n)]
As = [makeset(Akeys) for makeset in self.builders]
Akeys = IISet(Akeys)
n = randint(0, MAXSIZE)
Bkeys = [randint(1, MAXVAL) for j in range(n)]
Bs = [makeset(Bkeys) for makeset in self.builders]
Bkeys = IISet(Bkeys)
for op, simulator in ((self.union, self._union),
(self.intersection, self._intersection),
(self.difference, self._difference)):
for A in As:
for B in Bs:
got = op(A, B)
want = simulator(Akeys, Bkeys)
self.assertEqual(list(got), want,
(A, B, Akeys, Bkeys, list(got), want))
# Given a mapping builder (IIBTree, OOBucket, etc), return a function
# that builds an object of that type given only a list of keys.
def makeBuilder(mapbuilder):
def result(keys=[], mapbuilder=mapbuilder):
return mapbuilder(zip(keys, keys))
return result
class PureII(SetResult):
from BTrees.IIBTree import union, intersection, difference
builders = IISet, IITreeSet, makeBuilder(IIBTree), makeBuilder(IIBucket)
class PureIO(SetResult):
from BTrees.IOBTree import union, intersection, difference
builders = IOSet, IOTreeSet, makeBuilder(IOBTree), makeBuilder(IOBucket)
class PureOO(SetResult):
from BTrees.OOBTree import union, intersection, difference
builders = OOSet, OOTreeSet, makeBuilder(OOBTree), makeBuilder(OOBucket)
class PureOI(SetResult):
from BTrees.OIBTree import union, intersection, difference
builders = OISet, OITreeSet, makeBuilder(OIBTree), makeBuilder(OIBucket)
# Subclasses must set up (as class variables):
# multiunion, union
# mkset, mktreeset
# mkbucket, mkbtree
class MultiUnion(TestCase):
def testEmpty(self):
self.assertEqual(len(self.multiunion([])), 0)
def testOne(self):
for sequence in [3], range(20), range(-10, 0, 2) + range(1, 10, 2):
seq1 = sequence[:]
seq2 = sequence[:]
seq2.reverse()
seqsorted = sequence[:]
seqsorted.sort()
for seq in seq1, seq2, seqsorted:
for builder in self.mkset, self.mktreeset:
input = builder(seq)
output = self.multiunion([input])
self.assertEqual(len(seq), len(output))
self.assertEqual(seqsorted, list(output))
def testValuesIgnored(self):
for builder in self.mkbucket, self.mkbtree:
input = builder([(1, 2), (3, 4), (5, 6)])
output = self.multiunion([input])
self.assertEqual([1, 3, 5], list(output))
def testBigInput(self):
N = 100000
input = self.mkset(range(N))
output = self.multiunion([input] * 10)
self.assertEqual(len(output), N)
self.assertEqual(output.minKey(), 0)
self.assertEqual(output.maxKey(), N-1)
self.assertEqual(list(output), range(N))
def testLotsOfLittleOnes(self):
from random import shuffle
N = 5000
inputs = []
mkset, mktreeset = self.mkset, self.mktreeset
for i in range(N):
base = i * 4 - N
inputs.append(mkset([base, base+1]))
inputs.append(mktreeset([base+2, base+3]))
shuffle(inputs)
output = self.multiunion(inputs)
self.assertEqual(len(output), N*4)
self.assertEqual(list(output), range(-N, 3*N))
def testFunkyKeyIteration(self):
# The internal set iteration protocol allows "iterating over" a
# a single key as if it were a set.
N = 100
union, mkset = self.union, self.mkset
slow = mkset()
for i in range(N):
slow = union(slow, mkset([i]))
fast = self.multiunion(range(N)) # acts like N distinct singleton sets
self.assertEqual(len(slow), N)
self.assertEqual(len(fast), N)
self.assertEqual(list(slow), list(fast))
self.assertEqual(list(fast), range(N))
class TestIIMultiUnion(MultiUnion):
from BTrees.IIBTree import multiunion, union
from BTrees.IIBTree import IISet as mkset, IITreeSet as mktreeset
from BTrees.IIBTree import IIBucket as mkbucket, IIBTree as mkbtree
class TestIOMultiUnion(MultiUnion):
from BTrees.IOBTree import multiunion, union
from BTrees.IOBTree import IOSet as mkset, IOTreeSet as mktreeset
from BTrees.IOBTree import IOBucket as mkbucket, IOBTree as mkbtree
# Check that various special module functions are and aren't imported from
# the expected BTree modules.
class TestImports(TestCase):
def testWeightedUnion(self):
from BTrees.IIBTree import weightedUnion
from BTrees.OIBTree import weightedUnion
try:
from BTrees.IOBTree import weightedUnion
except ImportError:
pass
else:
self.fail("IOBTree shouldn't have weightedUnion")
try:
from BTrees.OOBTree import weightedUnion
except ImportError:
pass
else:
self.fail("OOBTree shouldn't have weightedUnion")
def testWeightedIntersection(self):
from BTrees.IIBTree import weightedIntersection
from BTrees.OIBTree import weightedIntersection
try:
from BTrees.IOBTree import weightedIntersection
except ImportError:
pass
else:
self.fail("IOBTree shouldn't have weightedIntersection")
try:
from BTrees.OOBTree import weightedIntersection
except ImportError:
pass
else:
self.fail("OOBTree shouldn't have weightedIntersection")
def testMultiunion(self):
from BTrees.IIBTree import multiunion
from BTrees.IOBTree import multiunion
try:
from BTrees.OIBTree import multiunion
except ImportError:
pass
else:
self.fail("OIBTree shouldn't have multiunion")
try:
from BTrees.OOBTree import multiunion
except ImportError:
pass
else:
self.fail("OOBTree shouldn't have multiunion")
# Subclasses must set up (as class variables):
# weightedUnion, weightedIntersection
# builders -- sequence of constructors, taking items
# union, intersection -- the module routines of those names
# mkbucket -- the module bucket builder
class Weighted(TestCase):
def setUp(self):
self.Aitems = [(1, 10), (3, 30), (5, 50), (6, 60)]
self.Bitems = [(2, 21), (3, 31), (4, 41), (6, 61), (7, 71)]
self.As = [make(self.Aitems) for make in self.builders]
self.Bs = [make(self.Bitems) for make in self.builders]
self.emptys = [make([]) for make in self.builders]
weights = []
for w1 in -3, -1, 0, 1, 7:
for w2 in -3, -1, 0, 1, 7:
weights.append((w1, w2))
self.weights = weights
def testBothNone(self):
for op in self.weightedUnion, self.weightedIntersection:
w, C = op(None, None)
self.assert_(C is None)
self.assertEqual(w, 0)
w, C = op(None, None, 42, 666)
self.assert_(C is None)
self.assertEqual(w, 0)
def testLeftNone(self):
for op in self.weightedUnion, self.weightedIntersection:
for A in self.As + self.emptys:
w, C = op(None, A)
self.assert_(C is A)
self.assertEqual(w, 1)
w, C = op(None, A, 42, 666)
self.assert_(C is A)
self.assertEqual(w, 666)
def testRightNone(self):
for op in self.weightedUnion, self.weightedIntersection:
for A in self.As + self.emptys:
w, C = op(A, None)
self.assert_(C is A)
self.assertEqual(w, 1)
w, C = op(A, None, 42, 666)
self.assert_(C is A)
self.assertEqual(w, 42)
# If obj is a set, return a bucket with values all 1; else return obj.
def _normalize(self, obj):
if isaset(obj):
obj = self.mkbucket(zip(obj, [1] * len(obj)))
return obj
# Python simulation of weightedUnion.
def _wunion(self, A, B, w1=1, w2=1):
if isaset(A) and isaset(B):
return 1, self.union(A, B).keys()
A = self._normalize(A)
B = self._normalize(B)
result = []
for key in self.union(A, B):
v1 = A.get(key, 0)
v2 = B.get(key, 0)
result.append((key, v1*w1 + v2*w2))
return 1, result
def testUnion(self):
inputs = self.As + self.Bs + self.emptys
for A in inputs:
for B in inputs:
want_w, want_s = self._wunion(A, B)
got_w, got_s = self.weightedUnion(A, B)
self.assertEqual(got_w, want_w)
if isaset(got_s):
self.assertEqual(got_s.keys(), want_s)
else:
self.assertEqual(got_s.items(), want_s)
for w1, w2 in self.weights:
want_w, want_s = self._wunion(A, B, w1, w2)
got_w, got_s = self.weightedUnion(A, B, w1, w2)
self.assertEqual(got_w, want_w)
if isaset(got_s):
self.assertEqual(got_s.keys(), want_s)
else:
self.assertEqual(got_s.items(), want_s)
# Python simulation weightedIntersection.
def _wintersection(self, A, B, w1=1, w2=1):
if isaset(A) and isaset(B):
return w1 + w2, self.intersection(A, B).keys()
A = self._normalize(A)
B = self._normalize(B)
result = []
for key in self.intersection(A, B):
result.append((key, A[key]*w1 + B[key]*w2))
return 1, result
def testIntersection(self):
inputs = self.As + self.Bs + self.emptys
for A in inputs:
for B in inputs:
want_w, want_s = self._wintersection(A, B)
got_w, got_s = self.weightedIntersection(A, B)
self.assertEqual(got_w, want_w)
if isaset(got_s):
self.assertEqual(got_s.keys(), want_s)
else:
self.assertEqual(got_s.items(), want_s)
for w1, w2 in self.weights:
want_w, want_s = self._wintersection(A, B, w1, w2)
got_w, got_s = self.weightedIntersection(A, B, w1, w2)
self.assertEqual(got_w, want_w)
if isaset(got_s):
self.assertEqual(got_s.keys(), want_s)
else:
self.assertEqual(got_s.items(), want_s)
# Given a set builder (like OITreeSet or OISet), return a function that
# takes a list of (key, value) pairs and builds a set out of the keys.
def itemsToSet(setbuilder):
def result(items, setbuilder=setbuilder):
return setbuilder([key for key, value in items])
return result
class TestWeightedII(Weighted):
from BTrees.IIBTree import weightedUnion, weightedIntersection
from BTrees.IIBTree import union, intersection
from BTrees.IIBTree import IIBucket as mkbucket
builders = IIBucket, IIBTree, itemsToSet(IISet), itemsToSet(IITreeSet)
class TestWeightedOI(Weighted):
from BTrees.OIBTree import weightedUnion, weightedIntersection
from BTrees.OIBTree import union, intersection
from BTrees.OIBTree import OIBucket as mkbucket
builders = OIBucket, OIBTree, itemsToSet(OISet), itemsToSet(OITreeSet)
# 'thing' is a bucket, btree, set or treeset. Return true iff it's one of the
# latter two.
def isaset(thing):
return not hasattr(thing, 'values')
def test_suite():
s = TestSuite()
for klass in (TestIIMultiUnion, TestIOMultiUnion,
TestImports,
PureII, PureIO, PureOI, PureOO,
TestWeightedII, TestWeightedOI):
s.addTest(makeSuite(klass))
return s
def main():
TextTestRunner().run(test_suite())
if __name__ == '__main__':
main()
ZODB/src/BTrees/tests/__init__.py 0000640 0001750 0000144 00000000064 10051215041 016375 0 ustar ctheune users # If tests is a package, debugging is a bit easier.
ZODB/src/BTrees/tests/testConflict.py 0000640 0001750 0000144 00000066326 10220643624 017326 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import os
from unittest import TestCase, TestSuite, makeSuite
from BTrees.OOBTree import OOBTree, OOBucket, OOSet, OOTreeSet
from BTrees.IOBTree import IOBTree, IOBucket, IOSet, IOTreeSet
from BTrees.IIBTree import IIBTree, IIBucket, IISet, IITreeSet
from BTrees.IFBTree import IFBTree, IFBucket, IFSet, IFTreeSet
from BTrees.OIBTree import OIBTree, OIBucket, OISet, OITreeSet
import transaction
from ZODB.POSException import ConflictError
class Base:
""" Tests common to all types: sets, buckets, and BTrees """
storage = None
def tearDown(self):
transaction.abort()
del self.t
if self.storage is not None:
self.storage.close()
self.storage.cleanup()
def openDB(self):
from ZODB.FileStorage import FileStorage
from ZODB.DB import DB
n = 'fs_tmp__%s' % os.getpid()
self.storage = FileStorage(n)
self.db = DB(self.storage)
class MappingBase(Base):
""" Tests common to mappings (buckets, btrees) """
def _deletefail(self):
del self.t[1]
def _setupConflict(self):
l=[ -5124, -7377, 2274, 8801, -9901, 7327, 1565, 17, -679,
3686, -3607, 14, 6419, -5637, 6040, -4556, -8622, 3847, 7191,
-4067]
e1=[(-1704, 0), (5420, 1), (-239, 2), (4024, 3), (-6984, 4)]
e2=[(7745, 0), (4868, 1), (-2548, 2), (-2711, 3), (-3154, 4)]
base=self.t
base.update([(i, i*i) for i in l[:20]])
b1=base.__class__(base)
b2=base.__class__(base)
bm=base.__class__(base)
items=base.items()
return base, b1, b2, bm, e1, e2, items
def testSimpleConflict(self):
# Unlike all the other tests, invoke conflict resolution
# by committing a transaction and catching a conflict
# in the storage.
self.openDB()
r1 = self.db.open().root()
r1["t"] = self.t
transaction.commit()
r2 = self.db.open().root()
copy = r2["t"]
list(copy) # unghostify
self.assertEqual(self.t._p_serial, copy._p_serial)
self.t.update({1:2, 2:3})
transaction.commit()
copy.update({3:4})
transaction.commit()
def testMergeDelete(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
del b1[items[0][0]]
del b2[items[5][0]]
del b1[items[-1][0]]
del b2[items[-2][0]]
del bm[items[0][0]]
del bm[items[5][0]]
del bm[items[-1][0]]
del bm[items[-2][0]]
test_merge(base, b1, b2, bm, 'merge delete')
def testMergeDeleteAndUpdate(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
del b1[items[0][0]]
b2[items[5][0]]=1
del b1[items[-1][0]]
b2[items[-2][0]]=2
del bm[items[0][0]]
bm[items[5][0]]=1
del bm[items[-1][0]]
bm[items[-2][0]]=2
test_merge(base, b1, b2, bm, 'merge update and delete')
def testMergeUpdate(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1[items[0][0]]=1
b2[items[5][0]]=2
b1[items[-1][0]]=3
b2[items[-2][0]]=4
bm[items[0][0]]=1
bm[items[5][0]]=2
bm[items[-1][0]]=3
bm[items[-2][0]]=4
test_merge(base, b1, b2, bm, 'merge update')
def testFailMergeDelete(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
del b1[items[0][0]]
del b2[items[0][0]]
test_merge(base, b1, b2, bm, 'merge conflicting delete',
should_fail=1)
def testFailMergeUpdate(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1[items[0][0]]=1
b2[items[0][0]]=2
test_merge(base, b1, b2, bm, 'merge conflicting update',
should_fail=1)
def testFailMergeDeleteAndUpdate(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
del b1[items[0][0]]
b2[items[0][0]]=-9
test_merge(base, b1, b2, bm, 'merge conflicting update and delete',
should_fail=1)
def testMergeInserts(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1[-99999]=-99999
b1[e1[0][0]]=e1[0][1]
b2[99999]=99999
b2[e1[2][0]]=e1[2][1]
bm[-99999]=-99999
bm[e1[0][0]]=e1[0][1]
bm[99999]=99999
bm[e1[2][0]]=e1[2][1]
test_merge(base, b1, b2, bm, 'merge insert')
def testMergeInsertsFromEmpty(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
base.clear()
b1.clear()
b2.clear()
bm.clear()
b1.update(e1)
bm.update(e1)
b2.update(e2)
bm.update(e2)
test_merge(base, b1, b2, bm, 'merge insert from empty')
def testFailMergeEmptyAndFill(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1.clear()
bm.clear()
b2.update(e2)
bm.update(e2)
test_merge(base, b1, b2, bm, 'merge insert from empty', should_fail=1)
def testMergeEmpty(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1.clear()
bm.clear()
test_merge(base, b1, b2, bm, 'empty one and not other', should_fail=1)
def testFailMergeInsert(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1[-99999]=-99999
b1[e1[0][0]]=e1[0][1]
b2[99999]=99999
b2[e1[0][0]]=e1[0][1]
test_merge(base, b1, b2, bm, 'merge conflicting inserts',
should_fail=1)
class SetTests(Base):
"Set (as opposed to TreeSet) specific tests."
def _setupConflict(self):
l=[ -5124, -7377, 2274, 8801, -9901, 7327, 1565, 17, -679,
3686, -3607, 14, 6419, -5637, 6040, -4556, -8622, 3847, 7191,
-4067]
e1=[-1704, 5420, -239, 4024, -6984]
e2=[7745, 4868, -2548, -2711, -3154]
base=self.t
base.update(l)
b1=base.__class__(base)
b2=base.__class__(base)
bm=base.__class__(base)
items=base.keys()
return base, b1, b2, bm, e1, e2, items
def testMergeDelete(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1.remove(items[0])
b2.remove(items[5])
b1.remove(items[-1])
b2.remove(items[-2])
bm.remove(items[0])
bm.remove(items[5])
bm.remove(items[-1])
bm.remove(items[-2])
test_merge(base, b1, b2, bm, 'merge delete')
def testFailMergeDelete(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1.remove(items[0])
b2.remove(items[0])
test_merge(base, b1, b2, bm, 'merge conflicting delete',
should_fail=1)
def testMergeInserts(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1.insert(-99999)
b1.insert(e1[0])
b2.insert(99999)
b2.insert(e1[2])
bm.insert(-99999)
bm.insert(e1[0])
bm.insert(99999)
bm.insert(e1[2])
test_merge(base, b1, b2, bm, 'merge insert')
def testMergeInsertsFromEmpty(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
base.clear()
b1.clear()
b2.clear()
bm.clear()
b1.update(e1)
bm.update(e1)
b2.update(e2)
bm.update(e2)
test_merge(base, b1, b2, bm, 'merge insert from empty')
def testFailMergeEmptyAndFill(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1.clear()
bm.clear()
b2.update(e2)
bm.update(e2)
test_merge(base, b1, b2, bm, 'merge insert from empty', should_fail=1)
def testMergeEmpty(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1.clear()
bm.clear()
test_merge(base, b1, b2, bm, 'empty one and not other', should_fail=1)
def testFailMergeInsert(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1.insert(-99999)
b1.insert(e1[0])
b2.insert(99999)
b2.insert(e1[0])
test_merge(base, b1, b2, bm, 'merge conflicting inserts',
should_fail=1)
def test_merge(o1, o2, o3, expect, message='failed to merge', should_fail=0):
s1 = o1.__getstate__()
s2 = o2.__getstate__()
s3 = o3.__getstate__()
expected = expect.__getstate__()
if expected is None:
expected = ((((),),),)
if should_fail:
try:
merged = o1._p_resolveConflict(s1, s2, s3)
except ConflictError, err:
pass
else:
assert 0, message
else:
merged = o1._p_resolveConflict(s1, s2, s3)
assert merged == expected, message
class BucketTests(MappingBase):
""" Tests common to all buckets """
class BTreeTests(MappingBase):
""" Tests common to all BTrees """
## BTree tests
class TestIOBTrees(BTreeTests, TestCase):
def setUp(self):
self.t = IOBTree()
class TestOOBTrees(BTreeTests, TestCase):
def setUp(self):
self.t = OOBTree()
class TestOIBTrees(BTreeTests, TestCase):
def setUp(self):
self.t = OIBTree()
class TestIIBTrees(BTreeTests, TestCase):
def setUp(self):
self.t = IIBTree()
class TestIFBTrees(BTreeTests, TestCase):
def setUp(self):
self.t = IFBTree()
## Set tests
class TestIOSets(SetTests, TestCase):
def setUp(self):
self.t = IOSet()
class TestOOSets(SetTests, TestCase):
def setUp(self):
self.t = OOSet()
class TestIISets(SetTests, TestCase):
def setUp(self):
self.t = IISet()
class TestIFSets(SetTests, TestCase):
def setUp(self):
self.t = IFSet()
class TestOISets(SetTests, TestCase):
def setUp(self):
self.t = OISet()
class TestIOTreeSets(SetTests, TestCase):
def setUp(self):
self.t = IOTreeSet()
class TestOOTreeSets(SetTests, TestCase):
def setUp(self):
self.t = OOTreeSet()
class TestIITreeSets(SetTests, TestCase):
def setUp(self):
self.t = IITreeSet()
class TestIFTreeSets(SetTests, TestCase):
def setUp(self):
self.t = IFTreeSet()
class TestOITreeSets(SetTests, TestCase):
def setUp(self):
self.t = OITreeSet()
## Bucket tests
class TestIOBuckets(BucketTests, TestCase):
def setUp(self):
self.t = IOBucket()
class TestOOBuckets(BucketTests, TestCase):
def setUp(self):
self.t = OOBucket()
class TestIIBuckets(BucketTests, TestCase):
def setUp(self):
self.t = IIBucket()
class TestIFBuckets(BucketTests, TestCase):
def setUp(self):
self.t = IFBucket()
class TestOIBuckets(BucketTests, TestCase):
def setUp(self):
self.t = OIBucket()
class NastyConfict(Base, TestCase):
def setUp(self):
self.t = OOBTree()
# This tests a problem that cropped up while trying to write
# testBucketSplitConflict (below): conflict resolution wasn't
# working at all in non-trivial cases. Symptoms varied from
# strange complaints about pickling (despite that the test isn't
# doing any *directly*), thru SystemErrors from Python and
# AssertionErrors inside the BTree code.
def testResolutionBlowsUp(self):
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Invoke conflict resolution by committing a transaction.
self.openDB()
r1 = self.db.open().root()
r1["t"] = self.t
transaction.commit()
r2 = self.db.open().root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
self.t.update({1:2, 2:3})
transaction.commit()
copy.update({3:4})
transaction.commit() # if this doesn't blow up
list(copy.values()) # and this doesn't either, then fine
def testBucketSplitConflict(self):
# Tests that a bucket split is viewed as a conflict.
# It's (almost necessarily) a white-box test, and sensitive to
# implementation details.
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Invoke conflict resolution by committing a transaction.
self.openDB()
r1 = self.db.open().root()
r1["t"] = self.t
transaction.commit()
r2 = self.db.open(synch=False).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
# In one transaction, add 16 new keys to bucket1, to force a bucket
# split.
b = self.t
numtoadd = 16
candidate = 60
while numtoadd:
if not b.has_key(candidate):
b[candidate] = candidate
numtoadd -= 1
candidate += 1
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 61 .. 74
# bucket 2 has 16 values: [75, 76 .. 81] + [84, 88 ..116]
# bucket 3 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((b0, 60, b1, 75, b2, 120, b3), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state) , 2)
self.assertEqual(len(state[0]), 7)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 75)
self.assertEqual(state[0][5], 120)
transaction.commit()
# In the other transaction, add 3 values near the tail end of bucket1.
# This doesn't cause a split.
b = copy
for i in range(112, 116):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 18 values: 60, 64 .. 112, 113, 114, 115, 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
self.assertRaises(ConflictError, transaction.commit)
transaction.abort() # horrible things happen w/o this
def testEmptyBucketConflict(self):
# Tests that an emptied bucket *created by* conflict resolution is
# viewed as a conflict: conflict resolution doesn't have enough
# info to unlink the empty bucket from the BTree correctly.
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Invoke conflict resolution by committing a transaction.
self.openDB()
r1 = self.db.open().root()
r1["t"] = self.t
transaction.commit()
r2 = self.db.open(synch=False).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
# In one transaction, delete half of bucket 1.
b = self.t
for k in 60, 64, 68, 72, 76, 80, 84, 88:
del b[k]
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 7 values: 92, 96, 100, 104, 108, 112, 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state) , 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
transaction.commit()
# In the other transaction, delete the other half of bucket 1.
b = copy
for k in 92, 96, 100, 104, 108, 112, 116:
del b[k]
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 8 values: 60, 64, 68, 72, 76, 80, 84, 88
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Conflict resolution empties bucket1 entirely. This used to
# create an "insane" BTree (a legit BTree cannot contain an empty
# bucket -- it contains NULL pointers the BTree code doesn't
# expect, and segfaults result).
self.assertRaises(ConflictError, transaction.commit)
transaction.abort() # horrible things happen w/o this
def testEmptyBucketNoConflict(self):
# Tests that a plain empty bucket (on input) is not viewed as a
# conflict.
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Invoke conflict resolution by committing a transaction.
self.openDB()
r1 = self.db.open().root()
r1["t"] = self.t
transaction.commit()
r2 = self.db.open().root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
# In one transaction, just add a key.
b = self.t
b[1] = 1
# bucket 0 has 16 values: [0, 1] + [4, 8 .. 56]
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
transaction.commit()
# In the other transaction, delete bucket 2.
b = copy
for k in range(120, 200, 4):
del b[k]
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 3)
self.assertEqual(state[0][1], 60)
# This shouldn't create a ConflictError.
transaction.commit()
# And the resulting BTree shouldn't have internal damage.
b._check()
# The snaky control flow in _bucket__p_resolveConflict ended up trying
# to decref a NULL pointer if conflict resolution was fed 3 empty
# buckets. http://collector.zope.org/Zope/553
def testThreeEmptyBucketsNoSegfault(self):
self.openDB()
r1 = self.db.open().root()
self.assertEqual(len(self.t), 0)
r1["t"] = b = self.t # an empty tree
transaction.commit()
r2 = self.db.open(synch=False).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
# In one transaction, add and delete a key.
b[2] = 2
del b[2]
transaction.commit()
# In the other transaction, also add and delete a key.
b = copy
b[1] = 1
del b[1]
# If the commit() segfaults, the C code is still wrong for this case.
self.assertRaises(ConflictError, transaction.commit)
transaction.abort()
def testCantResolveBTreeConflict(self):
# Test that a conflict involving two different changes to
# an internal BTree node is unresolvable. An internal node
# only changes when there are enough additions or deletions
# to a child bucket that the bucket is split or removed.
# It's (almost necessarily) a white-box test, and sensitive to
# implementation details.
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Set up database connections to provoke conflict.
self.openDB()
r1 = self.db.open().root()
r1["t"] = self.t
transaction.commit()
r2 = self.db.open(synch=False).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
# Now one transaction should add enough keys to cause a split,
# and another should remove all the keys in one bucket.
for k in range(200, 300, 4):
self.t[k] = k
transaction.commit()
for k in range(0, 60, 4):
del copy[k]
try:
transaction.commit()
except ConflictError, detail:
self.assert_(str(detail).startswith('database conflict error'))
transaction.abort()
else:
self.fail("expected ConflictError")
def testConflictWithOneEmptyBucket(self):
# If one transaction empties a bucket, while another adds an item
# to the bucket, all the changes "look resolvable": bucket conflict
# resolution returns a bucket containing (only) the item added by
# the latter transaction, but changes from the former transaction
# removing the bucket are uncontested: the bucket is removed from
# the BTree despite that resolution thinks it's non-empty! This
# was first reported by Dieter Maurer, to zodb-dev on 22 Mar 2005.
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Set up database connections to provoke conflict.
self.openDB()
r1 = self.db.open().root()
r1["t"] = self.t
transaction.commit()
r2 = self.db.open(synch=False).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
# Now one transaction empties the first bucket, and another adds a
# key to the first bucket.
for k in range(0, 60, 4):
del self.t[k]
transaction.commit()
copy[1] = 1
try:
transaction.commit()
except ConflictError, detail:
self.assert_(str(detail).startswith('database conflict error'))
transaction.abort()
else:
self.fail("expected ConflictError")
# Same thing, except commit the transactions in the opposite order.
b = OOBTree()
for i in range(0, 200, 4):
b[i] = i
r1 = self.db.open().root()
r1["t"] = b
transaction.commit()
r2 = self.db.open(synch=False).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(b._p_serial, copy._p_serial)
# Now one transaction empties the first bucket, and another adds a
# key to the first bucket.
b[1] = 1
transaction.commit()
for k in range(0, 60, 4):
del copy[k]
try:
transaction.commit()
except ConflictError, detail:
self.assert_(str(detail).startswith('database conflict error'))
transaction.abort()
else:
self.fail("expected ConflictError")
def test_suite():
suite = TestSuite()
for k in (
TestIIBTrees, TestIISets, TestIITreeSets, TestIIBuckets,
TestIFBTrees, TestIFSets, TestIFTreeSets, TestIFBuckets,
TestIOBTrees, TestIOSets, TestIOTreeSets, TestIOBuckets,
TestOOBTrees, TestOOSets, TestOOTreeSets, TestOOBuckets,
TestOIBTrees, TestOISets, TestOITreeSets, TestOIBuckets,
NastyConfict):
suite.addTest(makeSuite(k))
return suite
ZODB/src/BTrees/tests/testBTrees.py 0000640 0001750 0000144 00000145676 10306151001 016743 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import random
from unittest import TestCase, TestSuite, TextTestRunner, makeSuite
from BTrees.OOBTree import OOBTree, OOBucket, OOSet, OOTreeSet
from BTrees.IOBTree import IOBTree, IOBucket, IOSet, IOTreeSet
from BTrees.IIBTree import IIBTree, IIBucket, IISet, IITreeSet
from BTrees.IFBTree import IFBTree, IFBucket, IFSet, IFTreeSet
from BTrees.OIBTree import OIBTree, OIBucket, OISet, OITreeSet
from BTrees.check import check
import transaction
from ZODB import DB
from ZODB.MappingStorage import MappingStorage
class Base(TestCase):
""" Tests common to all types: sets, buckets, and BTrees """
db = None
def tearDown(self):
if self.db is not None:
self.db.close()
self.t = None
del self.t
def _getRoot(self):
if self.db is None:
# Unclear: On the next line, the ZODB4 flavor of this routine
# [asses a cache_size argument:
# self.db = DB(MappingStorage(), cache_size=1)
# If that's done here, though, testLoadAndStore() and
# testGhostUnghost() both nail the CPU and seemingly
# never finish.
self.db = DB(MappingStorage())
return self.db.open().root()
def _closeRoot(self, root):
root._p_jar.close()
def testLoadAndStore(self):
for i in 0, 10, 1000:
t = self.t.__class__()
self._populate(t, i)
root = None
root = self._getRoot()
root[i] = t
transaction.commit()
root2 = self._getRoot()
if hasattr(t, 'items'):
self.assertEqual(list(root2[i].items()) , list(t.items()))
else:
self.assertEqual(list(root2[i].keys()) , list(t.keys()))
self._closeRoot(root)
self._closeRoot(root2)
def testGhostUnghost(self):
for i in 0, 10, 1000:
t = self.t.__class__()
self._populate(t, i)
root = self._getRoot()
root[i] = t
transaction.commit()
root2 = self._getRoot()
root2[i]._p_deactivate()
transaction.commit()
if hasattr(t, 'items'):
self.assertEqual(list(root2[i].items()) , list(t.items()))
else:
self.assertEqual(list(root2[i].keys()) , list(t.keys()))
self._closeRoot(root)
self._closeRoot(root2)
def testSimpleExclusiveKeyRange(self):
t = self.t.__class__()
self.assertEqual(list(t.keys()), [])
self.assertEqual(list(t.keys(excludemin=True)), [])
self.assertEqual(list(t.keys(excludemax=True)), [])
self.assertEqual(list(t.keys(excludemin=True, excludemax=True)), [])
self._populate(t, 1)
self.assertEqual(list(t.keys()), [0])
self.assertEqual(list(t.keys(excludemin=True)), [])
self.assertEqual(list(t.keys(excludemax=True)), [])
self.assertEqual(list(t.keys(excludemin=True, excludemax=True)), [])
t.clear()
self._populate(t, 2)
self.assertEqual(list(t.keys()), [0, 1])
self.assertEqual(list(t.keys(excludemin=True)), [1])
self.assertEqual(list(t.keys(excludemax=True)), [0])
self.assertEqual(list(t.keys(excludemin=True, excludemax=True)), [])
t.clear()
self._populate(t, 3)
self.assertEqual(list(t.keys()), [0, 1, 2])
self.assertEqual(list(t.keys(excludemin=True)), [1, 2])
self.assertEqual(list(t.keys(excludemax=True)), [0, 1])
self.assertEqual(list(t.keys(excludemin=True, excludemax=True)), [1])
self.assertEqual(list(t.keys(-1, 3, excludemin=True, excludemax=True)),
[0, 1, 2])
self.assertEqual(list(t.keys(0, 3, excludemin=True, excludemax=True)),
[1, 2])
self.assertEqual(list(t.keys(-1, 2, excludemin=True, excludemax=True)),
[0, 1])
self.assertEqual(list(t.keys(0, 2, excludemin=True, excludemax=True)),
[1])
class MappingBase(Base):
""" Tests common to mappings (buckets, btrees) """
def _populate(self, t, l):
# Make some data
for i in range(l): t[i]=i
def testRepr(self):
# test the repr because buckets have a complex repr implementation
# internally the cutoff from a stack allocated buffer to a heap
# allocated buffer is 10000.
for i in range(1000):
self.t[i] = i
r = repr(self.t)
# Make sure the repr is 10000 bytes long for a bucket.
# But since the test is also run for btrees, skip the length
# check if the repr starts with '<'
if not r.startswith('<'):
self.assert_(len(r) > 10000)
def testGetItemFails(self):
self.assertRaises(KeyError, self._getitemfail)
def _getitemfail(self):
return self.t[1]
def testGetReturnsDefault(self):
self.assertEqual(self.t.get(1) , None)
self.assertEqual(self.t.get(1, 'foo') , 'foo')
def testSetItemGetItemWorks(self):
self.t[1] = 1
a = self.t[1]
self.assertEqual(a , 1, `a`)
def testReplaceWorks(self):
self.t[1] = 1
self.assertEqual(self.t[1] , 1, self.t[1])
self.t[1] = 2
self.assertEqual(self.t[1] , 2, self.t[1])
def testLen(self):
added = {}
r = range(1000)
for x in r:
k = random.choice(r)
self.t[k] = x
added[k] = x
addl = added.keys()
self.assertEqual(len(self.t) , len(addl), len(self.t))
def testHasKeyWorks(self):
self.t[1] = 1
self.assert_(self.t.has_key(1))
self.assert_(1 in self.t)
self.assert_(0 not in self.t)
self.assert_(2 not in self.t)
def testValuesWorks(self):
for x in range(100):
self.t[x] = x*x
v = self.t.values()
for i in range(100):
self.assertEqual(v[i], i*i)
self.assertRaises(IndexError, lambda: v[i+1])
i = 0
for value in self.t.itervalues():
self.assertEqual(value, i*i)
i += 1
def testValuesWorks1(self):
for x in range(100):
self.t[99-x] = x
for x in range(40):
lst = list(self.t.values(0+x,99-x))
lst.sort()
self.assertEqual(lst,range(0+x,99-x+1))
lst = list(self.t.values(max=99-x, min=0+x))
lst.sort()
self.assertEqual(lst,range(0+x,99-x+1))
def testValuesNegativeIndex(self):
L = [-3, 6, -11, 4]
for i in L:
self.t[i] = i
L.sort()
vals = self.t.values()
for i in range(-1, -5, -1):
self.assertEqual(vals[i], L[i])
self.assertRaises(IndexError, lambda: vals[-5])
def testKeysWorks(self):
for x in range(100):
self.t[x] = x
v = self.t.keys()
i = 0
for x in v:
self.assertEqual(x,i)
i = i + 1
self.assertRaises(IndexError, lambda: v[i])
for x in range(40):
lst = self.t.keys(0+x,99-x)
self.assertEqual(list(lst), range(0+x, 99-x+1))
lst = self.t.keys(max=99-x, min=0+x)
self.assertEqual(list(lst), range(0+x, 99-x+1))
self.assertEqual(len(v), 100)
def testKeysNegativeIndex(self):
L = [-3, 6, -11, 4]
for i in L:
self.t[i] = i
L.sort()
keys = self.t.keys()
for i in range(-1, -5, -1):
self.assertEqual(keys[i], L[i])
self.assertRaises(IndexError, lambda: keys[-5])
def testItemsWorks(self):
for x in range(100):
self.t[x] = 2*x
v = self.t.items()
i = 0
for x in v:
self.assertEqual(x[0], i)
self.assertEqual(x[1], 2*i)
i += 1
self.assertRaises(IndexError, lambda: v[i+1])
i = 0
for x in self.t.iteritems():
self.assertEqual(x, (i, 2*i))
i += 1
items = list(self.t.items(min=12, max=20))
self.assertEqual(items, zip(range(12, 21), range(24, 43, 2)))
items = list(self.t.iteritems(min=12, max=20))
self.assertEqual(items, zip(range(12, 21), range(24, 43, 2)))
def testItemsNegativeIndex(self):
L = [-3, 6, -11, 4]
for i in L:
self.t[i] = i
L.sort()
items = self.t.items()
for i in range(-1, -5, -1):
self.assertEqual(items[i], (L[i], L[i]))
self.assertRaises(IndexError, lambda: items[-5])
def testDeleteInvalidKeyRaisesKeyError(self):
self.assertRaises(KeyError, self._deletefail)
def _deletefail(self):
del self.t[1]
def testMaxKeyMinKey(self):
self.t[7] = 6
self.t[3] = 10
self.t[8] = 12
self.t[1] = 100
self.t[5] = 200
self.t[10] = 500
self.t[6] = 99
self.t[4] = 150
del self.t[7]
t = self.t
self.assertEqual(t.maxKey(), 10)
self.assertEqual(t.maxKey(6), 6)
self.assertEqual(t.maxKey(9), 8)
self.assertEqual(t.minKey(), 1)
self.assertEqual(t.minKey(3), 3)
self.assertEqual(t.minKey(9), 10)
try:
t.maxKey(t.minKey() - 1)
except ValueError, err:
self.assertEqual(str(err), "no key satisfies the conditions")
else:
self.fail("expected ValueError")
try:
t.minKey(t.maxKey() + 1)
except ValueError, err:
self.assertEqual(str(err), "no key satisfies the conditions")
else:
self.fail("expected ValueError")
def testClear(self):
r = range(100)
for x in r:
rnd = random.choice(r)
self.t[rnd] = 0
self.t.clear()
diff = lsubtract(list(self.t.keys()), [])
self.assertEqual(diff, [])
def testUpdate(self):
d={}
l=[]
for i in range(10000):
k=random.randrange(-2000, 2001)
d[k]=i
l.append((k, i))
items=d.items()
items.sort()
self.t.update(d)
self.assertEqual(list(self.t.items()), items)
self.t.clear()
self.assertEqual(list(self.t.items()), [])
self.t.update(l)
self.assertEqual(list(self.t.items()), items)
# Before ZODB 3.4.2, update/construction from PersistentMapping failed.
def testUpdateFromPersistentMapping(self):
from persistent.mapping import PersistentMapping
pm = PersistentMapping({1: 2})
self.t.update(pm)
self.assertEqual(list(self.t.items()), [(1, 2)])
# Construction goes thru the same internals as .update().
t = self.t.__class__(pm)
self.assertEqual(list(t.items()), [(1, 2)])
def testEmptyRangeSearches(self):
t = self.t
t.update([(1,1), (5,5), (9,9)])
self.assertEqual(list(t.keys(-6,-4)), [], list(t.keys(-6,-4)))
self.assertEqual(list(t.keys(2,4)), [], list(t.keys(2,4)))
self.assertEqual(list(t.keys(6,8)), [], list(t.keys(6,8)))
self.assertEqual(list(t.keys(10,12)), [], list(t.keys(10,12)))
self.assertEqual(list(t.keys(9, 1)), [], list(t.keys(9, 1)))
# For IITreeSets, this one was returning 31 for len(keys), and
# list(keys) produced a list with 100 elements.
t.clear()
t.update(zip(range(300), range(300)))
keys = t.keys(200, 50)
self.assertEqual(len(keys), 0)
self.assertEqual(list(keys), [])
self.assertEqual(list(t.iterkeys(200, 50)), [])
keys = t.keys(max=50, min=200)
self.assertEqual(len(keys), 0)
self.assertEqual(list(keys), [])
self.assertEqual(list(t.iterkeys(max=50, min=200)), [])
def testSlicing(self):
# Test that slicing of .keys()/.values()/.items() works exactly the
# same way as slicing a Python list with the same contents.
# This tests fixes to several bugs in this area, starting with
# http://collector.zope.org/Zope/419,
# "BTreeItems slice contains 1 too many elements".
t = self.t
for n in range(10):
t.clear()
self.assertEqual(len(t), 0)
keys = []
values = []
items = []
for key in range(n):
value = -2 * key
t[key] = value
keys.append(key)
values.append(value)
items.append((key, value))
self.assertEqual(len(t), n)
kslice = t.keys()
vslice = t.values()
islice = t.items()
self.assertEqual(len(kslice), n)
self.assertEqual(len(vslice), n)
self.assertEqual(len(islice), n)
# Test whole-structure slices.
x = kslice[:]
self.assertEqual(list(x), keys[:])
x = vslice[:]
self.assertEqual(list(x), values[:])
x = islice[:]
self.assertEqual(list(x), items[:])
for lo in range(-2*n, 2*n+1):
# Test one-sided slices.
x = kslice[:lo]
self.assertEqual(list(x), keys[:lo])
x = kslice[lo:]
self.assertEqual(list(x), keys[lo:])
x = vslice[:lo]
self.assertEqual(list(x), values[:lo])
x = vslice[lo:]
self.assertEqual(list(x), values[lo:])
x = islice[:lo]
self.assertEqual(list(x), items[:lo])
x = islice[lo:]
self.assertEqual(list(x), items[lo:])
for hi in range(-2*n, 2*n+1):
# Test two-sided slices.
x = kslice[lo:hi]
self.assertEqual(list(x), keys[lo:hi])
x = vslice[lo:hi]
self.assertEqual(list(x), values[lo:hi])
x = islice[lo:hi]
self.assertEqual(list(x), items[lo:hi])
# The specific test case from Zope collector 419.
t.clear()
for i in xrange(100):
t[i] = 1
tslice = t.items()[20:80]
self.assertEqual(len(tslice), 60)
self.assertEqual(list(tslice), zip(range(20, 80), [1]*60))
def testIterators(self):
t = self.t
for keys in [], [-2], [1, 4], range(-170, 2000, 6):
t.clear()
for k in keys:
t[k] = -3 * k
self.assertEqual(list(t), keys)
x = []
for k in t:
x.append(k)
self.assertEqual(x, keys)
it = iter(t)
self.assert_(it is iter(it))
x = []
try:
while 1:
x.append(it.next())
except StopIteration:
pass
self.assertEqual(x, keys)
self.assertEqual(list(t.iterkeys()), keys)
self.assertEqual(list(t.itervalues()), list(t.values()))
self.assertEqual(list(t.iteritems()), list(t.items()))
def testRangedIterators(self):
t = self.t
for keys in [], [-2], [1, 4], range(-170, 2000, 13):
t.clear()
values = []
for k in keys:
value = -3 * k
t[k] = value
values.append(value)
items = zip(keys, values)
self.assertEqual(list(t.iterkeys()), keys)
self.assertEqual(list(t.itervalues()), values)
self.assertEqual(list(t.iteritems()), items)
if not keys:
continue
min_mid_max = (keys[0], keys[len(keys) >> 1], keys[-1])
for key1 in min_mid_max:
for lo in range(key1 - 1, key1 + 2):
# Test one-sided range iterators.
goodkeys = [k for k in keys if lo <= k]
got = t.iterkeys(lo)
self.assertEqual(goodkeys, list(got))
goodvalues = [t[k] for k in goodkeys]
got = t.itervalues(lo)
self.assertEqual(goodvalues, list(got))
gooditems = zip(goodkeys, goodvalues)
got = t.iteritems(lo)
self.assertEqual(gooditems, list(got))
for key2 in min_mid_max:
for hi in range(key2 - 1, key2 + 2):
goodkeys = [k for k in keys if lo <= k <= hi]
got = t.iterkeys(min=lo, max=hi)
self.assertEqual(goodkeys, list(got))
goodvalues = [t[k] for k in goodkeys]
got = t.itervalues(lo, max=hi)
self.assertEqual(goodvalues, list(got))
gooditems = zip(goodkeys, goodvalues)
got = t.iteritems(max=hi, min=lo)
self.assertEqual(gooditems, list(got))
def testBadUpdateTupleSize(self):
# This one silently ignored the excess in Zope3.
try:
self.t.update([(1, 2, 3)])
except TypeError:
pass
else:
self.fail("update() with 3-tuple didn't complain")
# This one dumped core in Zope3.
try:
self.t.update([(1,)])
except TypeError:
pass
else:
self.fail("update() with 1-tuple didn't complain")
# This one should simply succeed.
self.t.update([(1, 2)])
self.assertEqual(list(self.t.items()), [(1, 2)])
def testSimpleExclusivRanges(self):
def identity(x):
return x
def dup(x):
return [(y, y) for y in x]
for methodname, f in (("keys", identity),
("values", identity),
("items", dup),
("iterkeys", identity),
("itervalues", identity),
("iteritems", dup)):
t = self.t.__class__()
meth = getattr(t, methodname, None)
if meth is None:
continue
self.assertEqual(list(meth()), [])
self.assertEqual(list(meth(excludemin=True)), [])
self.assertEqual(list(meth(excludemax=True)), [])
self.assertEqual(list(meth(excludemin=True, excludemax=True)), [])
self._populate(t, 1)
self.assertEqual(list(meth()), f([0]))
self.assertEqual(list(meth(excludemin=True)), [])
self.assertEqual(list(meth(excludemax=True)), [])
self.assertEqual(list(meth(excludemin=True, excludemax=True)), [])
t.clear()
self._populate(t, 2)
self.assertEqual(list(meth()), f([0, 1]))
self.assertEqual(list(meth(excludemin=True)), f([1]))
self.assertEqual(list(meth(excludemax=True)), f([0]))
self.assertEqual(list(meth(excludemin=True, excludemax=True)), [])
t.clear()
self._populate(t, 3)
self.assertEqual(list(meth()), f([0, 1, 2]))
self.assertEqual(list(meth(excludemin=True)), f([1, 2]))
self.assertEqual(list(meth(excludemax=True)), f([0, 1]))
self.assertEqual(list(meth(excludemin=True, excludemax=True)),
f([1]))
self.assertEqual(list(meth(-1, 3, excludemin=True,
excludemax=True)),
f([0, 1, 2]))
self.assertEqual(list(meth(0, 3, excludemin=True,
excludemax=True)),
f([1, 2]))
self.assertEqual(list(meth(-1, 2, excludemin=True,
excludemax=True)),
f([0, 1]))
self.assertEqual(list(meth(0, 2, excludemin=True,
excludemax=True)),
f([1]))
def testSetdefault(self):
t = self.t
self.assertEqual(t.setdefault(1, 2), 2)
# That should also have associated 1 with 2 in the tree.
self.assert_(1 in t)
self.assertEqual(t[1], 2)
# And trying to change it again should have no effect.
self.assertEqual(t.setdefault(1, 666), 2)
self.assertEqual(t[1], 2)
# Not enough arguments.
self.assertRaises(TypeError, t.setdefault)
self.assertRaises(TypeError, t.setdefault, 1)
# Too many arguments.
self.assertRaises(TypeError, t.setdefault, 1, 2, 3)
def testPop(self):
t = self.t
# Empty container.
# If no default given, raises KeyError.
self.assertRaises(KeyError, t.pop, 1)
# But if default given, returns that instead.
self.assertEqual(t.pop(1, 42), 42)
t[1] = 3
# KeyError when key is not in container and default is not passed.
self.assertRaises(KeyError, t.pop, 5)
self.assertEqual(list(t.items()), [(1, 3)])
# If key is in container, returns the value and deletes the key.
self.assertEqual(t.pop(1), 3)
self.assertEqual(len(t), 0)
# If key is present, return value bypassing default.
t[1] = 3
self.assertEqual(t.pop(1, 7), 3)
self.assertEqual(len(t), 0)
# Pop only one item.
t[1] = 3
t[2] = 4
self.assertEqual(len(t), 2)
self.assertEqual(t.pop(1), 3)
self.assertEqual(len(t), 1)
self.assertEqual(t[2], 4)
self.assertEqual(t.pop(1, 3), 3)
# Too few arguments.
self.assertRaises(TypeError, t.pop)
# Too many arguments.
self.assertRaises(TypeError, t.pop, 1, 2, 3)
class NormalSetTests(Base):
""" Test common to all set types """
def _populate(self, t, l):
# Make some data
t.update(range(l))
def testInsertReturnsValue(self):
t = self.t
self.assertEqual(t.insert(5) , 1)
def testDuplicateInsert(self):
t = self.t
t.insert(5)
self.assertEqual(t.insert(5) , 0)
def testInsert(self):
t = self.t
t.insert(1)
self.assert_(t.has_key(1))
self.assert_(1 in t)
self.assert_(2 not in t)
def testBigInsert(self):
t = self.t
r = xrange(10000)
for x in r:
t.insert(x)
for x in r:
self.assert_(t.has_key(x))
self.assert_(x in t)
def testRemoveSucceeds(self):
t = self.t
r = xrange(10000)
for x in r: t.insert(x)
for x in r: t.remove(x)
def testRemoveFails(self):
self.assertRaises(KeyError, self._removenonexistent)
def _removenonexistent(self):
self.t.remove(1)
def testHasKeyFails(self):
t = self.t
self.assert_(not t.has_key(1))
self.assert_(1 not in t)
def testKeys(self):
t = self.t
r = xrange(1000)
for x in r:
t.insert(x)
diff = lsubtract(t.keys(), r)
self.assertEqual(diff, [])
def testClear(self):
t = self.t
r = xrange(1000)
for x in r: t.insert(x)
t.clear()
diff = lsubtract(t.keys(), [])
self.assertEqual(diff , [], diff)
def testMaxKeyMinKey(self):
t = self.t
t.insert(1)
t.insert(2)
t.insert(3)
t.insert(8)
t.insert(5)
t.insert(10)
t.insert(6)
t.insert(4)
self.assertEqual(t.maxKey() , 10)
self.assertEqual(t.maxKey(6) , 6)
self.assertEqual(t.maxKey(9) , 8)
self.assertEqual(t.minKey() , 1)
self.assertEqual(t.minKey(3) , 3)
self.assertEqual(t.minKey(9) , 10)
self.assert_(t.minKey() in t)
self.assert_(t.minKey()-1 not in t)
self.assert_(t.maxKey() in t)
self.assert_(t.maxKey()+1 not in t)
try:
t.maxKey(t.minKey() - 1)
except ValueError, err:
self.assertEqual(str(err), "no key satisfies the conditions")
else:
self.fail("expected ValueError")
try:
t.minKey(t.maxKey() + 1)
except ValueError, err:
self.assertEqual(str(err), "no key satisfies the conditions")
else:
self.fail("expected ValueError")
def testUpdate(self):
d={}
l=[]
for i in range(10000):
k=random.randrange(-2000, 2001)
d[k]=i
l.append(k)
items = d.keys()
items.sort()
self.t.update(l)
self.assertEqual(list(self.t.keys()), items)
def testEmptyRangeSearches(self):
t = self.t
t.update([1, 5, 9])
self.assertEqual(list(t.keys(-6,-4)), [], list(t.keys(-6,-4)))
self.assertEqual(list(t.keys(2,4)), [], list(t.keys(2,4)))
self.assertEqual(list(t.keys(6,8)), [], list(t.keys(6,8)))
self.assertEqual(list(t.keys(10,12)), [], list(t.keys(10,12)))
self.assertEqual(list(t.keys(9,1)), [], list(t.keys(9,1)))
# For IITreeSets, this one was returning 31 for len(keys), and
# list(keys) produced a list with 100 elements.
t.clear()
t.update(range(300))
keys = t.keys(200, 50)
self.assertEqual(len(keys), 0)
self.assertEqual(list(keys), [])
keys = t.keys(max=50, min=200)
self.assertEqual(len(keys), 0)
self.assertEqual(list(keys), [])
def testSlicing(self):
# Test that slicing of .keys() works exactly the same way as slicing
# a Python list with the same contents.
t = self.t
for n in range(10):
t.clear()
self.assertEqual(len(t), 0)
keys = range(10*n, 11*n)
t.update(keys)
self.assertEqual(len(t), n)
kslice = t.keys()
self.assertEqual(len(kslice), n)
# Test whole-structure slices.
x = kslice[:]
self.assertEqual(list(x), keys[:])
for lo in range(-2*n, 2*n+1):
# Test one-sided slices.
x = kslice[:lo]
self.assertEqual(list(x), keys[:lo])
x = kslice[lo:]
self.assertEqual(list(x), keys[lo:])
for hi in range(-2*n, 2*n+1):
# Test two-sided slices.
x = kslice[lo:hi]
self.assertEqual(list(x), keys[lo:hi])
def testIterator(self):
t = self.t
for keys in [], [-2], [1, 4], range(-170, 2000, 6):
t.clear()
t.update(keys)
self.assertEqual(list(t), keys)
x = []
for k in t:
x.append(k)
self.assertEqual(x, keys)
it = iter(t)
self.assert_(it is iter(it))
x = []
try:
while 1:
x.append(it.next())
except StopIteration:
pass
self.assertEqual(x, keys)
class ExtendedSetTests(NormalSetTests):
def testLen(self):
t = self.t
r = xrange(10000)
for x in r: t.insert(x)
self.assertEqual(len(t) , 10000, len(t))
def testGetItem(self):
t = self.t
r = xrange(10000)
for x in r: t.insert(x)
for x in r:
self.assertEqual(t[x] , x)
class BTreeTests(MappingBase):
""" Tests common to all BTrees """
def tearDown(self):
self.t._check()
check(self.t)
MappingBase.tearDown(self)
def testDeleteNoChildrenWorks(self):
self.t[5] = 6
self.t[2] = 10
self.t[6] = 12
self.t[1] = 100
self.t[3] = 200
self.t[10] = 500
self.t[4] = 99
del self.t[4]
diff = lsubtract(self.t.keys(), [1,2,3,5,6,10])
self.assertEqual(diff , [], diff)
def testDeleteOneChildWorks(self):
self.t[5] = 6
self.t[2] = 10
self.t[6] = 12
self.t[1] = 100
self.t[3] = 200
self.t[10] = 500
self.t[4] = 99
del self.t[3]
diff = lsubtract(self.t.keys(), [1,2,4,5,6,10])
self.assertEqual(diff , [], diff)
def testDeleteTwoChildrenNoInorderSuccessorWorks(self):
self.t[5] = 6
self.t[2] = 10
self.t[6] = 12
self.t[1] = 100
self.t[3] = 200
self.t[10] = 500
self.t[4] = 99
del self.t[2]
diff = lsubtract(self.t.keys(), [1,3,4,5,6,10])
self.assertEqual(diff , [], diff)
def testDeleteTwoChildrenInorderSuccessorWorks(self):
# 7, 3, 8, 1, 5, 10, 6, 4 -- del 3
self.t[7] = 6
self.t[3] = 10
self.t[8] = 12
self.t[1] = 100
self.t[5] = 200
self.t[10] = 500
self.t[6] = 99
self.t[4] = 150
del self.t[3]
diff = lsubtract(self.t.keys(), [1,4,5,6,7,8,10])
self.assertEqual(diff , [], diff)
def testDeleteRootWorks(self):
# 7, 3, 8, 1, 5, 10, 6, 4 -- del 7
self.t[7] = 6
self.t[3] = 10
self.t[8] = 12
self.t[1] = 100
self.t[5] = 200
self.t[10] = 500
self.t[6] = 99
self.t[4] = 150
del self.t[7]
diff = lsubtract(self.t.keys(), [1,3,4,5,6,8,10])
self.assertEqual(diff , [], diff)
def testRandomNonOverlappingInserts(self):
added = {}
r = range(100)
for x in r:
k = random.choice(r)
if not added.has_key(k):
self.t[k] = x
added[k] = 1
addl = added.keys()
addl.sort()
diff = lsubtract(list(self.t.keys()), addl)
self.assertEqual(diff , [], (diff, addl, list(self.t.keys())))
def testRandomOverlappingInserts(self):
added = {}
r = range(100)
for x in r:
k = random.choice(r)
self.t[k] = x
added[k] = 1
addl = added.keys()
addl.sort()
diff = lsubtract(self.t.keys(), addl)
self.assertEqual(diff , [], diff)
def testRandomDeletes(self):
r = range(1000)
added = []
for x in r:
k = random.choice(r)
self.t[k] = x
added.append(k)
deleted = []
for x in r:
k = random.choice(r)
if self.t.has_key(k):
self.assert_(k in self.t)
del self.t[k]
deleted.append(k)
if self.t.has_key(k):
self.fail( "had problems deleting %s" % k )
badones = []
for x in deleted:
if self.t.has_key(x):
badones.append(x)
self.assertEqual(badones , [], (badones, added, deleted))
def testTargetedDeletes(self):
r = range(1000)
for x in r:
k = random.choice(r)
self.t[k] = x
for x in r:
try:
del self.t[x]
except KeyError:
pass
self.assertEqual(realseq(self.t.keys()) , [], realseq(self.t.keys()))
def testPathologicalRightBranching(self):
r = range(1000)
for x in r:
self.t[x] = 1
self.assertEqual(realseq(self.t.keys()) , r, realseq(self.t.keys()))
for x in r:
del self.t[x]
self.assertEqual(realseq(self.t.keys()) , [], realseq(self.t.keys()))
def testPathologicalLeftBranching(self):
r = range(1000)
revr = r[:]
revr.reverse()
for x in revr:
self.t[x] = 1
self.assertEqual(realseq(self.t.keys()) , r, realseq(self.t.keys()))
for x in revr:
del self.t[x]
self.assertEqual(realseq(self.t.keys()) , [], realseq(self.t.keys()))
def testSuccessorChildParentRewriteExerciseCase(self):
add_order = [
85, 73, 165, 273, 215, 142, 233, 67, 86, 166, 235, 225, 255,
73, 175, 171, 285, 162, 108, 28, 283, 258, 232, 199, 260,
298, 275, 44, 261, 291, 4, 181, 285, 289, 216, 212, 129,
243, 97, 48, 48, 159, 22, 285, 92, 110, 27, 55, 202, 294,
113, 251, 193, 290, 55, 58, 239, 71, 4, 75, 129, 91, 111,
271, 101, 289, 194, 218, 77, 142, 94, 100, 115, 101, 226,
17, 94, 56, 18, 163, 93, 199, 286, 213, 126, 240, 245, 190,
195, 204, 100, 199, 161, 292, 202, 48, 165, 6, 173, 40, 218,
271, 228, 7, 166, 173, 138, 93, 22, 140, 41, 234, 17, 249,
215, 12, 292, 246, 272, 260, 140, 58, 2, 91, 246, 189, 116,
72, 259, 34, 120, 263, 168, 298, 118, 18, 28, 299, 192, 252,
112, 60, 277, 273, 286, 15, 263, 141, 241, 172, 255, 52, 89,
127, 119, 255, 184, 213, 44, 116, 231, 173, 298, 178, 196,
89, 184, 289, 98, 216, 115, 35, 132, 278, 238, 20, 241, 128,
179, 159, 107, 206, 194, 31, 260, 122, 56, 144, 118, 283,
183, 215, 214, 87, 33, 205, 183, 212, 221, 216, 296, 40,
108, 45, 188, 139, 38, 256, 276, 114, 270, 112, 214, 191,
147, 111, 299, 107, 101, 43, 84, 127, 67, 205, 251, 38, 91,
297, 26, 165, 187, 19, 6, 73, 4, 176, 195, 90, 71, 30, 82,
139, 210, 8, 41, 253, 127, 190, 102, 280, 26, 233, 32, 257,
194, 263, 203, 190, 111, 218, 199, 29, 81, 207, 18, 180,
157, 172, 192, 135, 163, 275, 74, 296, 298, 265, 105, 191,
282, 277, 83, 188, 144, 259, 6, 173, 81, 107, 292, 231,
129, 65, 161, 113, 103, 136, 255, 285, 289, 1
]
delete_order = [
276, 273, 12, 275, 2, 286, 127, 83, 92, 33, 101, 195,
299, 191, 22, 232, 291, 226, 110, 94, 257, 233, 215, 184,
35, 178, 18, 74, 296, 210, 298, 81, 265, 175, 116, 261,
212, 277, 260, 234, 6, 129, 31, 4, 235, 249, 34, 289, 105,
259, 91, 93, 119, 7, 183, 240, 41, 253, 290, 136, 75, 292,
67, 112, 111, 256, 163, 38, 126, 139, 98, 56, 282, 60, 26,
55, 245, 225, 32, 52, 40, 271, 29, 252, 239, 89, 87, 205,
213, 180, 97, 108, 120, 218, 44, 187, 196, 251, 202, 203,
172, 28, 188, 77, 90, 199, 297, 282, 141, 100, 161, 216,
73, 19, 17, 189, 30, 258
]
for x in add_order:
self.t[x] = 1
for x in delete_order:
try: del self.t[x]
except KeyError:
if self.t.has_key(x): self.assertEqual(1,2,"failed to delete %s" % x)
def testRangeSearchAfterSequentialInsert(self):
r = range(100)
for x in r:
self.t[x] = 0
diff = lsubtract(list(self.t.keys(0, 100)), r)
self.assertEqual(diff , [], diff)
def testRangeSearchAfterRandomInsert(self):
r = range(100)
a = {}
for x in r:
rnd = random.choice(r)
self.t[rnd] = 0
a[rnd] = 0
diff = lsubtract(list(self.t.keys(0, 100)), a.keys())
self.assertEqual(diff , [], diff)
def testPathologicalRangeSearch(self):
t = self.t
# Build a 2-level tree with at least two buckets.
for i in range(200):
t[i] = i
items, dummy = t.__getstate__()
self.assert_(len(items) > 2) # at least two buckets and a key
# All values in the first bucket are < firstkey. All in the
# second bucket are >= firstkey, and firstkey is the first key in
# the second bucket.
firstkey = items[1]
therange = t.keys(-1, firstkey)
self.assertEqual(len(therange), firstkey + 1)
self.assertEqual(list(therange), range(firstkey + 1))
# Now for the tricky part. If we delete firstkey, the second bucket
# loses its smallest key, but firstkey remains in the BTree node.
# If we then do a high-end range search on firstkey, the BTree node
# directs us to look in the second bucket, but there's no longer any
# key <= firstkey in that bucket. The correct answer points to the
# end of the *first* bucket. The algorithm has to be smart enough
# to "go backwards" in the BTree then; if it doesn't, it will
# erroneously claim that the range is empty.
del t[firstkey]
therange = t.keys(min=-1, max=firstkey)
self.assertEqual(len(therange), firstkey)
self.assertEqual(list(therange), range(firstkey))
def testInsertMethod(self):
t = self.t
t[0] = 1
self.assertEqual(t.insert(0, 1) , 0)
self.assertEqual(t.insert(1, 1) , 1)
self.assertEqual(lsubtract(list(t.keys()), [0,1]) , [])
def testDamagedIterator(self):
# A cute one from Steve Alexander. This caused the BTreeItems
# object to go insane, accessing memory beyond the allocated part
# of the bucket. If it fails, the symptom is either a C-level
# assertion error (if the BTree code was compiled without NDEBUG),
# or most likely a segfault (if the BTree code was compiled with
# NDEBUG).
t = self.t.__class__()
self._populate(t, 10)
# In order for this to fail, it's important that k be a "lazy"
# iterator, referring to the BTree by indirect position (index)
# instead of a fully materialized list. Then the position can
# end up pointing into trash memory, if the bucket pointed to
# shrinks.
k = t.keys()
for dummy in range(20):
try:
del t[k[0]]
except RuntimeError, detail:
self.assertEqual(str(detail), "the bucket being iterated "
"changed size")
break
# tests of various type errors
class TypeTest(TestCase):
def testBadTypeRaises(self):
self.assertRaises(TypeError, self._stringraises)
self.assertRaises(TypeError, self._floatraises)
self.assertRaises(TypeError, self._noneraises)
class TestIOBTrees(TypeTest):
def setUp(self):
self.t = IOBTree()
def _stringraises(self):
self.t['c'] = 1
def _floatraises(self):
self.t[2.5] = 1
def _noneraises(self):
self.t[None] = 1
class TestOIBTrees(TypeTest):
def setUp(self):
self.t = OIBTree()
def _stringraises(self):
self.t[1] = 'c'
def _floatraises(self):
self.t[1] = 1.4
def _noneraises(self):
self.t[1] = None
def testEmptyFirstBucketReportedByGuido(self):
b = self.t
for i in xrange(29972): # reduce to 29971 and it works
b[i] = i
for i in xrange(30): # reduce to 29 and it works
del b[i]
b[i+40000] = i
self.assertEqual(b.keys()[0], 30)
class TestIIBTrees(TestCase):
def setUp(self):
self.t = IIBTree()
def testNonIntegerKeyRaises(self):
self.assertRaises(TypeError, self._stringraiseskey)
self.assertRaises(TypeError, self._floatraiseskey)
self.assertRaises(TypeError, self._noneraiseskey)
def testNonIntegerValueRaises(self):
self.assertRaises(TypeError, self._stringraisesvalue)
self.assertRaises(TypeError, self._floatraisesvalue)
self.assertRaises(TypeError, self._noneraisesvalue)
def _stringraiseskey(self):
self.t['c'] = 1
def _floatraiseskey(self):
self.t[2.5] = 1
def _noneraiseskey(self):
self.t[None] = 1
def _stringraisesvalue(self):
self.t[1] = 'c'
def _floatraisesvalue(self):
self.t[1] = 1.4
def _noneraisesvalue(self):
self.t[1] = None
class TestIFBTrees(TestCase):
def setUp(self):
self.t = IFBTree()
def testNonIntegerKeyRaises(self):
self.assertRaises(TypeError, self._stringraiseskey)
self.assertRaises(TypeError, self._floatraiseskey)
self.assertRaises(TypeError, self._noneraiseskey)
def testNonNumericValueRaises(self):
self.assertRaises(TypeError, self._stringraisesvalue)
self.assertRaises(TypeError, self._noneraisesvalue)
self.t[1] = 1
self.t[1] = 1.0
def _stringraiseskey(self):
self.t['c'] = 1
def _floatraiseskey(self):
self.t[2.5] = 1
def _noneraiseskey(self):
self.t[None] = 1
def _stringraisesvalue(self):
self.t[1] = 'c'
def _floatraisesvalue(self):
self.t[1] = 1.4
def _noneraisesvalue(self):
self.t[1] = None
class TestIOSets(TestCase):
def setUp(self):
self.t = IOSet()
def testNonIntegerInsertRaises(self):
self.assertRaises(TypeError,self._insertstringraises)
self.assertRaises(TypeError,self._insertfloatraises)
self.assertRaises(TypeError,self._insertnoneraises)
def _insertstringraises(self):
self.t.insert('a')
def _insertfloatraises(self):
self.t.insert(1.4)
def _insertnoneraises(self):
self.t.insert(None)
class DegenerateBTree(TestCase):
# Build a degenerate tree (set). Boxes are BTree nodes. There are
# 5 leaf buckets, each containing a single int. Keys in the BTree
# nodes don't appear in the buckets. Seven BTree nodes are purely
# indirection nodes (no keys). Buckets aren't all at the same depth:
#
# +------------------------+
# | 4 |
# +------------------------+
# | |
# | v
# | +-+
# | | |
# | +-+
# | |
# v v
# +-------+ +-------------+
# | 2 | | 6 10 |
# +-------+ +-------------+
# | | | | |
# v v v v v
# +-+ +-+ +-+ +-+ +-+
# | | | | | | | | | |
# +-+ +-+ +-+ +-+ +-+
# | | | | |
# v v v v v
# 1 3 +-+ 7 11
# | |
# +-+
# |
# v
# 5
#
# This is nasty for many algorithms. Consider a high-end range search
# for 4. The BTree nodes direct it to the 5 bucket, but the correct
# answer is the 3 bucket, which requires going in a different direction
# at the very top node already. Consider a low-end range search for
# 9. The BTree nodes direct it to the 7 bucket, but the correct answer
# is the 11 bucket. This is also a nasty-case tree for deletions.
def _build_degenerate_tree(self):
# Build the buckets and chain them together.
bucket11 = IISet([11])
bucket7 = IISet()
bucket7.__setstate__(((7,), bucket11))
bucket5 = IISet()
bucket5.__setstate__(((5,), bucket7))
bucket3 = IISet()
bucket3.__setstate__(((3,), bucket5))
bucket1 = IISet()
bucket1.__setstate__(((1,), bucket3))
# Build the deepest layers of indirection nodes.
ts = IITreeSet
tree1 = ts()
tree1.__setstate__(((bucket1,), bucket1))
tree3 = ts()
tree3.__setstate__(((bucket3,), bucket3))
tree5lower = ts()
tree5lower.__setstate__(((bucket5,), bucket5))
tree5 = ts()
tree5.__setstate__(((tree5lower,), bucket5))
tree7 = ts()
tree7.__setstate__(((bucket7,), bucket7))
tree11 = ts()
tree11.__setstate__(((bucket11,), bucket11))
# Paste together the middle layers.
tree13 = ts()
tree13.__setstate__(((tree1, 2, tree3), bucket1))
tree5711lower = ts()
tree5711lower.__setstate__(((tree5, 6, tree7, 10, tree11), bucket5))
tree5711 = ts()
tree5711.__setstate__(((tree5711lower,), bucket5))
# One more.
t = ts()
t.__setstate__(((tree13, 4, tree5711), bucket1))
t._check()
check(t)
return t, [1, 3, 5, 7, 11]
def testBasicOps(self):
t, keys = self._build_degenerate_tree()
self.assertEqual(len(t), len(keys))
self.assertEqual(list(t.keys()), keys)
# has_key actually returns the depth of a bucket.
self.assertEqual(t.has_key(1), 4)
self.assertEqual(t.has_key(3), 4)
self.assertEqual(t.has_key(5), 6)
self.assertEqual(t.has_key(7), 5)
self.assertEqual(t.has_key(11), 5)
for i in 0, 2, 4, 6, 8, 9, 10, 12:
self.assert_(i not in t)
def _checkRanges(self, tree, keys):
self.assertEqual(len(tree), len(keys))
sorted_keys = keys[:]
sorted_keys.sort()
self.assertEqual(list(tree.keys()), sorted_keys)
for k in keys:
self.assert_(k in tree)
if keys:
lokey = sorted_keys[0]
hikey = sorted_keys[-1]
self.assertEqual(lokey, tree.minKey())
self.assertEqual(hikey, tree.maxKey())
else:
lokey = hikey = 42
# Try all range searches.
for lo in range(lokey - 1, hikey + 2):
for hi in range(lo - 1, hikey + 2):
for skipmin in False, True:
for skipmax in False, True:
wantlo, wanthi = lo, hi
if skipmin:
wantlo += 1
if skipmax:
wanthi -= 1
want = [k for k in keys if wantlo <= k <= wanthi]
got = list(tree.keys(lo, hi, skipmin, skipmax))
self.assertEqual(want, got)
def testRanges(self):
t, keys = self._build_degenerate_tree()
self._checkRanges(t, keys)
def testDeletes(self):
# Delete keys in all possible orders, checking each tree along
# the way.
# This is a tough test. Previous failure modes included:
# 1. A variety of assertion failures in _checkRanges.
# 2. Assorted "Invalid firstbucket pointer" failures at
# seemingly random times, coming out of the BTree destructor.
# 3. Under Python 2.3 CVS, some baffling
# RuntimeWarning: tp_compare didn't return -1 or -2 for exception
# warnings, possibly due to memory corruption after a BTree
# goes insane.
t, keys = self._build_degenerate_tree()
for oneperm in permutations(keys):
t, keys = self._build_degenerate_tree()
for key in oneperm:
t.remove(key)
keys.remove(key)
t._check()
check(t)
self._checkRanges(t, keys)
# We removed all the keys, so the tree should be empty now.
self.assertEqual(t.__getstate__(), None)
# A damaged tree may trigger an "invalid firstbucket pointer"
# failure at the time its destructor is invoked. Try to force
# that to happen now, so it doesn't look like a baffling failure
# at some unrelated line.
del t # trigger destructor
class IIBucketTest(MappingBase):
def setUp(self):
self.t = IIBucket()
class IFBucketTest(MappingBase):
def setUp(self):
self.t = IFBucket()
class IOBucketTest(MappingBase):
def setUp(self):
self.t = IOBucket()
class OIBucketTest(MappingBase):
def setUp(self):
self.t = OIBucket()
class OOBucketTest(MappingBase):
def setUp(self):
self.t = OOBucket()
class IITreeSetTest(NormalSetTests):
def setUp(self):
self.t = IITreeSet()
class IFTreeSetTest(NormalSetTests):
def setUp(self):
self.t = IFTreeSet()
class IOTreeSetTest(NormalSetTests):
def setUp(self):
self.t = IOTreeSet()
class OITreeSetTest(NormalSetTests):
def setUp(self):
self.t = OITreeSet()
class OOTreeSetTest(NormalSetTests):
def setUp(self):
self.t = OOTreeSet()
class IISetTest(ExtendedSetTests):
def setUp(self):
self.t = IISet()
# Collector 1843. Error returns were effectively ignored in
# Bucket_rangeSearch(), leading to "delayed" errors, or worse.
def testNonIntKeyRaises(self):
self.t.insert(1)
# This one used to fail to raise the TypeError when it occurred.
self.assertRaises(TypeError, self.t.keys, "")
# This one used to segfault.
self.assertRaises(TypeError, self.t.keys, 0, "")
class IFSetTest(ExtendedSetTests):
def setUp(self):
self.t = IFSet()
class IOSetTest(ExtendedSetTests):
def setUp(self):
self.t = IOSet()
class OISetTest(ExtendedSetTests):
def setUp(self):
self.t = OISet()
class OOSetTest(ExtendedSetTests):
def setUp(self):
self.t = OOSet()
class IIBTreeTest(BTreeTests):
def setUp(self):
self.t = IIBTree()
class IFBTreeTest(BTreeTests):
def setUp(self):
self.t = IFBTree()
class IOBTreeTest(BTreeTests):
def setUp(self):
self.t = IOBTree()
class OIBTreeTest(BTreeTests):
def setUp(self):
self.t = OIBTree()
class OOBTreeTest(BTreeTests):
def setUp(self):
self.t = OOBTree()
# cmp error propagation tests
class DoesntLikeBeingCompared:
def __cmp__(self,other):
raise ValueError('incomparable')
class TestCmpError(TestCase):
def testFoo(self):
t = OOBTree()
t['hello world'] = None
try:
t[DoesntLikeBeingCompared()] = None
except ValueError,e:
self.assertEqual(str(e), 'incomparable')
else:
self.fail('incomarable objects should not be allowed into '
'the tree')
def test_suite():
s = TestSuite()
for klass in (
IIBucketTest, IIBTreeTest, IISetTest, IITreeSetTest,
IFBucketTest, IFBTreeTest, IFSetTest, IFTreeSetTest,
IOBucketTest, IOBTreeTest, IOSetTest, IOTreeSetTest,
OOBucketTest, OOBTreeTest, OOSetTest, OOTreeSetTest,
OIBucketTest, OIBTreeTest, OISetTest, OITreeSetTest,
# Note: there is no TestOOBTrees. The next three are
# checking for assorted TypeErrors, and when both keys
# and values oare objects (OO), there's nothing to test.
TestIIBTrees, TestIFBTrees, TestIOBTrees, TestOIBTrees,
TestIOSets,
DegenerateBTree,
TestCmpError):
s.addTest(makeSuite(klass))
return s
## utility functions
def lsubtract(l1, l2):
l1 = list(l1)
l2 = list(l2)
l = filter(lambda x, l1=l1: x not in l1, l2)
l = l + filter(lambda x, l2=l2: x not in l2, l1)
return l
def realseq(itemsob):
return [x for x in itemsob]
def permutations(x):
# Return a list of all permutations of list x.
n = len(x)
if n <= 1:
return [x]
result = []
x0 = x[0]
for i in range(n):
# Build the (n-1)! permutations with x[i] in the first position.
xcopy = x[:]
first, xcopy[i] = xcopy[i], x0
result.extend([[first] + p for p in permutations(xcopy[1:])])
return result
def main():
TextTestRunner().run(test_suite())
if __name__ == '__main__':
main()
ZODB/src/BTrees/tests/test_compare.py 0000640 0001750 0000144 00000004543 10057366465 017360 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Test errors during comparison of BTree keys."""
import unittest
from BTrees.OOBTree import OOBucket as Bucket, OOSet as Set
import transaction
from ZODB.MappingStorage import MappingStorage
from ZODB.DB import DB
class CompareTest(unittest.TestCase):
s = "A string with hi-bit-set characters: \700\701"
u = u"A unicode string"
def setUp(self):
# These defaults only make sense if the default encoding
# prevents s from being promoted to Unicode.
self.assertRaises(UnicodeError, unicode, self.s)
# An object needs to be added to the database to
self.db = DB(MappingStorage())
root = self.db.open().root()
self.bucket = root["bucket"] = Bucket()
self.set = root["set"] = Set()
transaction.commit()
def tearDown(self):
self.assert_(self.bucket._p_changed != 2)
self.assert_(self.set._p_changed != 2)
transaction.abort()
def assertUE(self, callable, *args):
self.assertRaises(UnicodeError, callable, *args)
def testBucketGet(self):
self.bucket[self.s] = 1
self.assertUE(self.bucket.get, self.u)
def testSetGet(self):
self.set.insert(self.s)
self.assertUE(self.set.remove, self.u)
def testBucketSet(self):
self.bucket[self.s] = 1
self.assertUE(self.bucket.__setitem__, self.u, 1)
def testSetSet(self):
self.set.insert(self.s)
self.assertUE(self.set.insert, self.u)
def testBucketMinKey(self):
self.bucket[self.s] = 1
self.assertUE(self.bucket.minKey, self.u)
def testSetMinKey(self):
self.set.insert(self.s)
self.assertUE(self.set.minKey, self.u)
def test_suite():
return unittest.makeSuite(CompareTest)
ZODB/src/BTrees/BTreeItemsTemplate.c 0000640 0001750 0000144 00000047706 10057366465 017032 0 ustar ctheune users /*****************************************************************************
Copyright (c) 2001, 2002 Zope Corporation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
#define BTREEITEMSTEMPLATE_C "$Id: BTreeItemsTemplate.c 25186 2004-06-02 15:07:33Z jim $\n"
/* A BTreeItems struct is returned from calling .items(), .keys() or
* .values() on a BTree-based data structure, and is also the result of
* taking slices of those. It represents a contiguous slice of a BTree.
*
* The start of the slice is in firstbucket, at offset first. The end of
* the slice is in lastbucket, at offset last. Both endpoints are inclusive.
* It must possible to get from firstbucket to lastbucket via following
* bucket 'next' pointers zero or more times. firstbucket, first, lastbucket,
* and last are readonly after initialization. An empty slice is represented
* by firstbucket == lastbucket == currentbucket == NULL.
*
* 'kind' determines whether this slice represents 'k'eys alone, 'v'alues
* alone, or 'i'items (key+value pairs). 'kind' is also readonly after
* initialization.
*
* The combination of currentbucket, currentoffset and pseudoindex acts as
* a search finger. Offset currentoffset in bucket currentbucket is at index
* pseudoindex, where pseudoindex==0 corresponds to offset first in bucket
* firstbucket, and pseudoindex==-1 corresponds to offset last in bucket
* lastbucket. The function BTreeItems_seek() can be used to set this combo
* correctly for any in-bounds index, and uses this combo on input to avoid
* needing to search from the start (or end) on each call. Calling
* BTreeItems_seek() with consecutive larger positions is very efficent.
* Calling it with consecutive smaller positions is more efficient than if
* a search finger weren't being used at all, but is still quadratic time
* in the number of buckets in the slice.
*/
typedef struct {
PyObject_HEAD
Bucket *firstbucket; /* First bucket */
Bucket *currentbucket; /* Current bucket (search finger) */
Bucket *lastbucket; /* Last bucket */
int currentoffset; /* Offset in currentbucket */
int pseudoindex; /* search finger index */
int first; /* Start offset in firstbucket */
int last; /* End offset in lastbucket */
char kind; /* 'k', 'v', 'i' */
} BTreeItems;
#define ITEMS(O)((BTreeItems*)(O))
static PyObject *
newBTreeItems(char kind,
Bucket *lowbucket, int lowoffset,
Bucket *highbucket, int highoffset);
static void
BTreeItems_dealloc(BTreeItems *self)
{
Py_XDECREF(self->firstbucket);
Py_XDECREF(self->lastbucket);
Py_XDECREF(self->currentbucket);
PyObject_DEL(self);
}
static int
BTreeItems_length_or_nonzero(BTreeItems *self, int nonzero)
{
int r;
Bucket *b, *next;
b = self->firstbucket;
if (b == NULL)
return 0;
r = self->last + 1 - self->first;
if (nonzero && r > 0)
/* Short-circuit if all we care about is nonempty */
return 1;
if (b == self->lastbucket)
return r;
Py_INCREF(b);
PER_USE_OR_RETURN(b, -1);
while ((next = b->next)) {
r += b->len;
if (nonzero && r > 0)
/* Short-circuit if all we care about is nonempty */
break;
if (next == self->lastbucket)
break; /* we already counted the last bucket */
Py_INCREF(next);
PER_UNUSE(b);
Py_DECREF(b);
b = next;
PER_USE_OR_RETURN(b, -1);
}
PER_UNUSE(b);
Py_DECREF(b);
return r >= 0 ? r : 0;
}
static int
BTreeItems_length( BTreeItems *self)
{
return BTreeItems_length_or_nonzero(self, 0);
}
/*
** BTreeItems_seek
**
** Find the ith position in the BTreeItems.
**
** Arguments: self The BTree
** i the index to seek to, in 0 .. len(self)-1, or in
** -len(self) .. -1, as for indexing a Python sequence.
**
**
** Returns 0 if successful, -1 on failure to seek (like out-of-bounds).
** Upon successful return, index i is at offset self->currentoffset in bucket
** self->currentbucket.
*/
static int
BTreeItems_seek(BTreeItems *self, int i)
{
int delta, pseudoindex, currentoffset;
Bucket *b, *currentbucket;
int error;
pseudoindex = self->pseudoindex;
currentoffset = self->currentoffset;
currentbucket = self->currentbucket;
if (currentbucket == NULL) goto no_match;
delta = i - pseudoindex;
while (delta > 0) { /* move right */
int max;
/* Want to move right delta positions; the most we can move right in
* this bucket is currentbucket->len - currentoffset - 1 positions.
*/
PER_USE_OR_RETURN(currentbucket, -1);
max = currentbucket->len - currentoffset - 1;
b = currentbucket->next;
PER_UNUSE(currentbucket);
if (delta <= max) {
currentoffset += delta;
pseudoindex += delta;
if (currentbucket == self->lastbucket
&& currentoffset > self->last) goto no_match;
break;
}
/* Move to start of next bucket. */
if (currentbucket == self->lastbucket || b == NULL) goto no_match;
currentbucket = b;
pseudoindex += max + 1;
delta -= max + 1;
currentoffset = 0;
}
while (delta < 0) { /* move left */
int status;
/* Want to move left -delta positions; the most we can move left in
* this bucket is currentoffset positions.
*/
if ((-delta) <= currentoffset) {
currentoffset += delta;
pseudoindex += delta;
if (currentbucket == self->firstbucket
&& currentoffset < self->first) goto no_match;
break;
}
/* Move to end of previous bucket. */
if (currentbucket == self->firstbucket) goto no_match;
status = PreviousBucket(¤tbucket, self->firstbucket);
if (status == 0)
goto no_match;
else if (status < 0)
return -1;
pseudoindex -= currentoffset + 1;
delta += currentoffset + 1;
PER_USE_OR_RETURN(currentbucket, -1);
currentoffset = currentbucket->len - 1;
PER_UNUSE(currentbucket);
}
assert(pseudoindex == i);
/* Alas, the user may have mutated the bucket since the last time we
* were called, and if they deleted stuff, we may be pointing into
* trash memory now.
*/
PER_USE_OR_RETURN(currentbucket, -1);
error = currentoffset < 0 || currentoffset >= currentbucket->len;
PER_UNUSE(currentbucket);
if (error) {
PyErr_SetString(PyExc_RuntimeError,
"the bucket being iterated changed size");
return -1;
}
Py_INCREF(currentbucket);
Py_DECREF(self->currentbucket);
self->currentbucket = currentbucket;
self->currentoffset = currentoffset;
self->pseudoindex = pseudoindex;
return 0;
no_match:
IndexError(i);
return -1;
}
/* Return the right kind ('k','v','i') of entry from bucket b at offset i.
* b must be activated. Returns NULL on error.
*/
static PyObject *
getBucketEntry(Bucket *b, int i, char kind)
{
PyObject *result = NULL;
assert(b);
assert(0 <= i && i < b->len);
switch (kind) {
case 'k':
COPY_KEY_TO_OBJECT(result, b->keys[i]);
break;
case 'v':
COPY_VALUE_TO_OBJECT(result, b->values[i]);
break;
case 'i': {
PyObject *key;
PyObject *value;;
COPY_KEY_TO_OBJECT(key, b->keys[i]);
if (!key) break;
COPY_VALUE_TO_OBJECT(value, b->values[i]);
if (!value) {
Py_DECREF(key);
break;
}
result = PyTuple_New(2);
if (result) {
PyTuple_SET_ITEM(result, 0, key);
PyTuple_SET_ITEM(result, 1, value);
}
else {
Py_DECREF(key);
Py_DECREF(value);
}
break;
}
default:
PyErr_SetString(PyExc_AssertionError,
"getBucketEntry: unknown kind");
break;
}
return result;
}
/*
** BTreeItems_item
**
** Arguments: self a BTreeItems structure
** i Which item to inspect
**
** Returns: the BTreeItems_item_BTree of self->kind, i
** (ie pulls the ith item out)
*/
static PyObject *
BTreeItems_item(BTreeItems *self, int i)
{
PyObject *result;
if (BTreeItems_seek(self, i) < 0) return NULL;
PER_USE_OR_RETURN(self->currentbucket, NULL);
result = getBucketEntry(self->currentbucket, self->currentoffset,
self->kind);
PER_UNUSE(self->currentbucket);
return result;
}
/*
** BTreeItems_slice
**
** Creates a new BTreeItems structure representing the slice
** between the low and high range
**
** Arguments: self The old BTreeItems structure
** ilow The start index
** ihigh The end index
**
** Returns: BTreeItems item
*/
static PyObject *
BTreeItems_slice(BTreeItems *self, int ilow, int ihigh)
{
Bucket *lowbucket;
Bucket *highbucket;
int lowoffset;
int highoffset;
int length = -1; /* len(self), but computed only if needed */
/* Complications:
* A Python slice never raises IndexError, but BTreeItems_seek does.
* Python did only part of index normalization before calling this:
* ilow may be < 0 now, and ihigh may be arbitrarily large. It's
* our responsibility to clip them.
* A Python slice is exclusive of the high index, but a BTreeItems
* struct is inclusive on both ends.
*/
/* First adjust ilow and ihigh to be legit endpoints in the Python
* sense (ilow inclusive, ihigh exclusive). This block duplicates the
* logic from Python's list_slice function (slicing for builtin lists).
*/
if (ilow < 0)
ilow = 0;
else {
if (length < 0)
length = BTreeItems_length(self);
if (ilow > length)
ilow = length;
}
if (ihigh < ilow)
ihigh = ilow;
else {
if (length < 0)
length = BTreeItems_length(self);
if (ihigh > length)
ihigh = length;
}
assert(0 <= ilow && ilow <= ihigh);
assert(length < 0 || ihigh <= length);
/* Now adjust for that our struct is inclusive on both ends. This is
* easy *except* when the slice is empty: there's no good way to spell
* that in an inclusive-on-both-ends scheme. For example, if the
* slice is btree.items([:0]), ilow == ihigh == 0 at this point, and if
* we were to subtract 1 from ihigh that would get interpreted by
* BTreeItems_seek as meaning the *entire* set of items. Setting ilow==1
* and ihigh==0 doesn't work either, as BTreeItems_seek raises IndexError
* if we attempt to seek to ilow==1 when the underlying sequence is empty.
* It seems simplest to deal with empty slices as a special case here.
*/
if (ilow == ihigh) {
/* empty slice */
lowbucket = highbucket = NULL;
lowoffset = 1;
highoffset = 0;
}
else {
assert(ilow < ihigh);
--ihigh; /* exclusive -> inclusive */
if (BTreeItems_seek(self, ilow) < 0) return NULL;
lowbucket = self->currentbucket;
lowoffset = self->currentoffset;
if (BTreeItems_seek(self, ihigh) < 0) return NULL;
highbucket = self->currentbucket;
highoffset = self->currentoffset;
}
return newBTreeItems(self->kind,
lowbucket, lowoffset, highbucket, highoffset);
}
static PySequenceMethods BTreeItems_as_sequence = {
(inquiry) BTreeItems_length,
(binaryfunc)0,
(intargfunc)0,
(intargfunc) BTreeItems_item,
(intintargfunc) BTreeItems_slice,
};
/* Number Method items (just for nb_nonzero!) */
static int
BTreeItems_nonzero(BTreeItems *self)
{
return BTreeItems_length_or_nonzero(self, 1);
}
static PyNumberMethods BTreeItems_as_number_for_nonzero = {
0,0,0,0,0,0,0,0,0,0,
(inquiry)BTreeItems_nonzero};
static PyTypeObject BTreeItemsType = {
PyObject_HEAD_INIT(NULL)
0, /*ob_size*/
MOD_NAME_PREFIX "BTreeItems", /*tp_name*/
sizeof(BTreeItems), /*tp_basicsize*/
0, /*tp_itemsize*/
/* methods */
(destructor) BTreeItems_dealloc, /*tp_dealloc*/
(printfunc)0, /*tp_print*/
(getattrfunc)0, /*obsolete tp_getattr*/
(setattrfunc)0, /*obsolete tp_setattr*/
(cmpfunc)0, /*tp_compare*/
(reprfunc)0, /*tp_repr*/
&BTreeItems_as_number_for_nonzero, /*tp_as_number*/
&BTreeItems_as_sequence, /*tp_as_sequence*/
0, /*tp_as_mapping*/
(hashfunc)0, /*tp_hash*/
(ternaryfunc)0, /*tp_call*/
(reprfunc)0, /*tp_str*/
0, /*tp_getattro*/
0, /*tp_setattro*/
/* Space for future expansion */
0L,0L,
"Sequence type used to iterate over BTree items." /* Documentation string */
};
/* Returns a new BTreeItems object representing the contiguous slice from
* offset lowoffset in bucket lowbucket through offset highoffset in bucket
* highbucket, inclusive. Pass lowbucket == NULL for an empty slice.
* The currentbucket is set to lowbucket, currentoffset ot lowoffset, and
* pseudoindex to 0. kind is 'k', 'v' or 'i' (see BTreeItems struct docs).
*/
static PyObject *
newBTreeItems(char kind,
Bucket *lowbucket, int lowoffset,
Bucket *highbucket, int highoffset)
{
BTreeItems *self;
UNLESS (self = PyObject_NEW(BTreeItems, &BTreeItemsType)) return NULL;
self->kind=kind;
self->first=lowoffset;
self->last=highoffset;
if (! lowbucket || ! highbucket
|| (lowbucket == highbucket && lowoffset > highoffset))
{
self->firstbucket = 0;
self->lastbucket = 0;
self->currentbucket = 0;
}
else
{
Py_INCREF(lowbucket);
self->firstbucket = lowbucket;
Py_INCREF(highbucket);
self->lastbucket = highbucket;
Py_INCREF(lowbucket);
self->currentbucket = lowbucket;
}
self->currentoffset = lowoffset;
self->pseudoindex = 0;
return OBJECT(self);
}
static int
nextBTreeItems(SetIteration *i)
{
if (i->position >= 0)
{
if (i->position)
{
DECREF_KEY(i->key);
DECREF_VALUE(i->value);
}
if (BTreeItems_seek(ITEMS(i->set), i->position) >= 0)
{
Bucket *currentbucket;
currentbucket = BUCKET(ITEMS(i->set)->currentbucket);
UNLESS(PER_USE(currentbucket))
{
/* Mark iteration terminated, so that finiSetIteration doesn't
* try to redundantly decref the key and value
*/
i->position = -1;
return -1;
}
COPY_KEY(i->key, currentbucket->keys[ITEMS(i->set)->currentoffset]);
INCREF_KEY(i->key);
COPY_VALUE(i->value,
currentbucket->values[ITEMS(i->set)->currentoffset]);
INCREF_VALUE(i->value);
i->position ++;
PER_UNUSE(currentbucket);
}
else
{
i->position = -1;
PyErr_Clear();
}
}
return 0;
}
static int
nextTreeSetItems(SetIteration *i)
{
if (i->position >= 0)
{
if (i->position)
{
DECREF_KEY(i->key);
}
if (BTreeItems_seek(ITEMS(i->set), i->position) >= 0)
{
Bucket *currentbucket;
currentbucket = BUCKET(ITEMS(i->set)->currentbucket);
UNLESS(PER_USE(currentbucket))
{
/* Mark iteration terminated, so that finiSetIteration doesn't
* try to redundantly decref the key and value
*/
i->position = -1;
return -1;
}
COPY_KEY(i->key, currentbucket->keys[ITEMS(i->set)->currentoffset]);
INCREF_KEY(i->key);
i->position ++;
PER_UNUSE(currentbucket);
}
else
{
i->position = -1;
PyErr_Clear();
}
}
return 0;
}
/* Support for the iteration protocol new in Python 2.2. */
static PyTypeObject BTreeIter_Type;
/* The type of iterator objects, returned by e.g. iter(IIBTree()). */
typedef struct {
PyObject_HEAD
/* We use a BTreeItems object because it's convenient and flexible.
* We abuse it two ways:
* 1. We set currentbucket to NULL when the iteration is finished.
* 2. We don't bother keeping pseudoindex in synch.
*/
BTreeItems *pitems;
} BTreeIter;
/* Return a new iterator object, to traverse the keys and/or values
* represented by pitems. pitems must not be NULL. Returns NULL if error.
*/
static BTreeIter *
BTreeIter_new(BTreeItems *pitems)
{
BTreeIter *result;
assert(pitems != NULL);
result = PyObject_New(BTreeIter, &BTreeIter_Type);
if (result) {
Py_INCREF(pitems);
result->pitems = pitems;
}
return result;
}
/* The iterator's tp_dealloc slot. */
static void
BTreeIter_dealloc(BTreeIter *bi)
{
Py_DECREF(bi->pitems);
PyObject_Del(bi);
}
/* The implementation of the iterator's tp_iternext slot. Returns "the next"
* item; returns NULL if error; returns NULL without setting an error if the
* iteration is exhausted (that's the way to terminate the iteration protocol).
*/
static PyObject *
BTreeIter_next(BTreeIter *bi, PyObject *args)
{
PyObject *result = NULL; /* until proven innocent */
BTreeItems *items = bi->pitems;
int i = items->currentoffset;
Bucket *bucket = items->currentbucket;
if (bucket == NULL) /* iteration termination is sticky */
return NULL;
PER_USE_OR_RETURN(bucket, NULL);
if (i >= bucket->len) {
/* We never leave this routine normally with i >= len: somebody
* else mutated the current bucket.
*/
PyErr_SetString(PyExc_RuntimeError,
"the bucket being iterated changed size");
/* Arrange for that this error is sticky too. */
items->currentoffset = INT_MAX;
goto Done;
}
/* Build the result object, from bucket at offset i. */
result = getBucketEntry(bucket, i, items->kind);
/* Advance position for next call. */
if (bucket == items->lastbucket && i >= items->last) {
/* Next call should terminate the iteration. */
Py_DECREF(items->currentbucket);
items->currentbucket = NULL;
}
else {
++i;
if (i >= bucket->len) {
Py_XINCREF(bucket->next);
items->currentbucket = bucket->next;
Py_DECREF(bucket);
i = 0;
}
items->currentoffset = i;
}
Done:
PER_UNUSE(bucket);
return result;
}
static PyObject *
BTreeIter_getiter(PyObject *it)
{
Py_INCREF(it);
return it;
}
static PyTypeObject BTreeIter_Type = {
PyObject_HEAD_INIT(NULL)
0, /* ob_size */
MOD_NAME_PREFIX "-iterator", /* tp_name */
sizeof(BTreeIter), /* tp_basicsize */
0, /* tp_itemsize */
/* methods */
(destructor)BTreeIter_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_compare */
0, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
0, /*PyObject_GenericGetAttr,*/ /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT, /* tp_flags */
0, /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
(getiterfunc)BTreeIter_getiter, /* tp_iter */
(iternextfunc)BTreeIter_next, /* tp_iternext */
0, /* tp_methods */
0, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
};
ZODB/src/BTrees/OIBTree.py 0000640 0001750 0000144 00000001307 10057366465 014755 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# hack to overcome dynamic-linking headache.
from _OIBTree import *
ZODB/src/BTrees/sorters.c 0000640 0001750 0000144 00000034725 10057366465 015031 0 ustar ctheune users /*****************************************************************************
Copyright (c) 2002 Zope Corporation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
/* Revision information: $Id: sorters.c 25186 2004-06-02 15:07:33Z jim $ */
/* The only routine here intended to be used outside the file is
size_t sort_int4_nodups(int *p, size_t n)
Sort the array of n ints pointed at by p, in place, and also remove
duplicates. Return the number of unique elements remaining, which occupy
a contiguous and monotonically increasing slice of the array starting at p.
Example: If the input array is [3, 1, 2, 3, 1, 5, 2], sort_int4_nodups
returns 4, and the first 4 elements of the array are changed to
[1, 2, 3, 5]. The content of the remaining array positions is not defined.
Notes:
+ This is specific to 4-byte signed ints, with endianness natural to the
platform.
+ 4*n bytes of available heap memory are required for best speed.
*/
#include
#include
#include
#include
#include
/* The type of array elements to be sorted. Most of the routines don't
care about the type, and will work fine for any scalar C type (provided
they're recompiled with element_type appropriately redefined). However,
the radix sort has to know everything about the type's internal
representation.
*/
typedef int element_type;
/* The radixsort is faster than the quicksort for large arrays, but radixsort
has high fixed overhead, making it a poor choice for small arrays. The
crossover point isn't critical, and is sensitive to things like compiler
and machine cache structure, so don't worry much about this.
*/
#define QUICKSORT_BEATS_RADIXSORT 800U
/* In turn, the quicksort backs off to an insertion sort for very small
slices. MAX_INSERTION is the largest slice quicksort leaves entirely to
insertion. Because this version of quicksort uses a median-of-3 rule for
selecting a pivot, MAX_INSERTION must be at least 2 (so that quicksort
has at least 3 values to look at in a slice). Again, the exact value here
isn't critical.
*/
#define MAX_INSERTION 25U
#if MAX_INSERTION < 2U
# error "MAX_INSERTION must be >= 2"
#endif
/* LSB-first radix sort of the n elements in 'in'.
'work' is work storage at least as large as 'in'. Depending on how many
swaps are done internally, the final result may come back in 'in' or 'work';
and that pointer is returned.
radixsort_int4 is specific to signed 4-byte ints, with natural machine
endianness.
*/
static element_type*
radixsort_int4(element_type *in, element_type *work, size_t n)
{
/* count[i][j] is the number of input elements that have byte value j
in byte position i, where byte position 0 is the LSB. Note that
holding i fixed, the sum of count[i][j] over all j in range(256)
is n.
*/
size_t count[4][256];
size_t i;
int offset, offsetinc;
/* Which byte position are we working on now? 0=LSB, 1, 2, ... */
int bytenum;
assert(sizeof(element_type) == 4);
assert(in);
assert(work);
/* Compute all of count in one pass. */
memset(count, 0, sizeof(count));
for (i = 0; i < n; ++i) {
element_type const x = in[i];
++count[0][(x ) & 0xff];
++count[1][(x >> 8) & 0xff];
++count[2][(x >> 16) & 0xff];
++count[3][(x >> 24) & 0xff];
}
/* For p an element_type* cast to char*, offset is how much farther we
have to go to get to the LSB of the element; this is 0 for little-
endian boxes and sizeof(element_type)-1 for big-endian.
offsetinc is 1 or -1, respectively, telling us which direction to go
from p+offset to get to the element's more-significant bytes.
*/
{
int one = 1;
if (*(char*)&one) {
/* Little endian. */
offset = 0;
offsetinc = 1;
}
else {
/* Big endian. */
offset = sizeof(element_type) - 1;
offsetinc = -1;
}
}
/* The radix sort. */
for (bytenum = 0;
bytenum < sizeof(element_type);
++bytenum, offset += offsetinc) {
/* Do a stable distribution sort on byte position bytenum,
from in to work. index[i] tells us the work index at which
to store the next in element with byte value i. pinbyte
points to the correct byte in the input array.
*/
size_t index[256];
unsigned char* pinbyte;
size_t total = 0;
size_t *pcount = count[bytenum];
/* Compute the correct output starting index for each possible
byte value.
*/
if (bytenum < sizeof(element_type) - 1) {
for (i = 0; i < 256; ++i) {
const size_t icount = pcount[i];
index[i] = total;
total += icount;
if (icount == n)
break;
}
if (i < 256) {
/* All bytes in the current position have value
i, so there's nothing to do on this pass.
*/
continue;
}
}
else {
/* The MSB of signed ints needs to be distributed
differently than the other bytes, in order
0x80, 0x81, ... 0xff, 0x00, 0x01, ... 0x7f
*/
for (i = 128; i < 256; ++i) {
const size_t icount = pcount[i];
index[i] = total;
total += icount;
if (icount == n)
break;
}
if (i < 256)
continue;
for (i = 0; i < 128; ++i) {
const size_t icount = pcount[i];
index[i] = total;
total += icount;
if (icount == n)
break;
}
if (i < 128)
continue;
}
assert(total == n);
/* Distribute the elements according to byte value. Note that
this is where most of the time is spent.
Note: The loop is unrolled 4x by hand, for speed. This
may be a pessimization someday, but was a significant win
on my MSVC 6.0 timing tests.
*/
pinbyte = (unsigned char *)in + offset;
i = 0;
/* Reduce number of elements to copy to a multiple of 4. */
while ((n - i) & 0x3) {
unsigned char byte = *pinbyte;
work[index[byte]++] = in[i];
++i;
pinbyte += sizeof(element_type);
}
for (; i < n; i += 4, pinbyte += 4 * sizeof(element_type)) {
unsigned char byte1 = *(pinbyte );
unsigned char byte2 = *(pinbyte + sizeof(element_type));
unsigned char byte3 = *(pinbyte + 2 * sizeof(element_type));
unsigned char byte4 = *(pinbyte + 3 * sizeof(element_type));
element_type in1 = in[i ];
element_type in2 = in[i+1];
element_type in3 = in[i+2];
element_type in4 = in[i+3];
work[index[byte1]++] = in1;
work[index[byte2]++] = in2;
work[index[byte3]++] = in3;
work[index[byte4]++] = in4;
}
/* Swap in and work (just a pointer swap). */
{
element_type *temp = in;
in = work;
work = temp;
}
}
return in;
}
/* Remove duplicates from sorted array in, storing exactly one of each distinct
element value into sorted array out. It's OK (and expected!) for in == out,
but otherwise the n elements beginning at in must not overlap with the n
beginning at out.
Return the number of elements in out.
*/
static size_t
uniq(element_type *out, element_type *in, size_t n)
{
size_t i;
element_type lastelt;
element_type *pout;
assert(out);
assert(in);
if (n == 0)
return 0;
/* i <- first index in 'in' that contains a duplicate.
in[0], in[1], ... in[i-1] are unique, but in[i-1] == in[i].
Set i to n if everything is unique.
*/
for (i = 1; i < n; ++i) {
if (in[i-1] == in[i])
break;
}
/* in[:i] is unique; copy to out[:i] if needed. */
assert(i > 0);
if (in != out)
memcpy(out, in, i * sizeof(element_type));
pout = out + i;
lastelt = in[i-1]; /* safe even when i == n */
for (++i; i < n; ++i) {
element_type elt = in[i];
if (elt != lastelt)
*pout++ = lastelt = elt;
}
return pout - out;
}
#if 0
/* insertionsort is no longer referenced directly, but I'd like to keep
* the code here just in case.
*/
/* Straight insertion sort of the n elements starting at 'in'. */
static void
insertionsort(element_type *in, size_t n)
{
element_type *p, *q;
element_type minimum; /* smallest seen so far */
element_type *plimit = in + n;
assert(in);
if (n < 2)
return;
minimum = *in;
for (p = in+1; p < plimit; ++p) {
/* *in <= *(in+1) <= ... <= *(p-1). Slide *p into place. */
element_type thiselt = *p;
if (thiselt < minimum) {
/* This is a new minimum. This saves p-in compares
when it happens, but should happen so rarely that
it's not worth checking for its own sake: the
point is that the far more popular 'else' branch can
exploit that thiselt is *not* the smallest so far.
*/
memmove(in+1, in, (p - in) * sizeof(*in));
*in = minimum = thiselt;
}
else {
/* thiselt >= minimum, so the loop will find a q
with *q <= thiselt. This saves testing q >= in
on each trip. It's such a simple loop that saving
a per-trip test is a major speed win.
*/
for (q = p-1; *q > thiselt; --q)
*(q+1) = *q;
*(q+1) = thiselt;
}
}
}
#endif
/* The maximum number of elements in the pending-work stack quicksort
maintains. The maximum stack depth is approximately log2(n), so
arrays of size up to approximately MAX_INSERTION * 2**STACKSIZE can be
sorted. The memory burden for the stack is small, so better safe than
sorry.
*/
#define STACKSIZE 60
/* A _stacknode remembers a contiguous slice of an array that needs to sorted.
lo must be <= hi, and, unlike Python array slices, this includes both ends.
*/
struct _stacknode {
element_type *lo;
element_type *hi;
};
static void
quicksort(element_type *plo, size_t n)
{
element_type *phi;
/* Swap two array elements. */
element_type _temp;
#define SWAP(P, Q) (_temp = *(P), *(P) = *(Q), *(Q) = _temp)
/* Stack of pending array slices to be sorted. */
struct _stacknode stack[STACKSIZE];
struct _stacknode *stackfree = stack; /* available stack slot */
/* Push an array slice on the pending-work stack. */
#define PUSH(PLO, PHI) \
do { \
assert(stackfree - stack < STACKSIZE); \
assert((PLO) <= (PHI)); \
stackfree->lo = (PLO); \
stackfree->hi = (PHI); \
++stackfree; \
} while(0)
assert(plo);
phi = plo + n - 1;
for (;;) {
element_type pivot;
element_type *pi, *pj;
assert(plo <= phi);
n = phi - plo + 1;
if (n <= MAX_INSERTION) {
/* Do a small insertion sort. Contra Knuth, we do
this now instead of waiting until the end, because
this little slice is likely still in cache now.
*/
element_type *p, *q;
element_type minimum = *plo;
for (p = plo+1; p <= phi; ++p) {
/* *plo <= *(plo+1) <= ... <= *(p-1).
Slide *p into place. */
element_type thiselt = *p;
if (thiselt < minimum) {
/* New minimum. */
memmove(plo+1,
plo,
(p - plo) * sizeof(*p));
*plo = minimum = thiselt;
}
else {
/* thiselt >= minimum, so the loop will
find a q with *q <= thiselt.
*/
for (q = p-1; *q > thiselt; --q)
*(q+1) = *q;
*(q+1) = thiselt;
}
}
/* Pop another slice off the stack. */
if (stack == stackfree)
break; /* no more slices -- we're done */
--stackfree;
plo = stackfree->lo;
phi = stackfree->hi;
continue;
}
/* Parition the slice.
For pivot, take the median of the leftmost, rightmost, and
middle elements. First sort those three; then the median
is the middle one. For technical reasons, the middle
element is swapped to plo+1 first (see Knuth Vol 3 Ed 2
section 5.2.2 exercise 55 -- reverse-sorted arrays can
take quadratic time otherwise!).
*/
{
element_type *plop1 = plo + 1;
element_type *pmid = plo + (n >> 1);
assert(plo < pmid && pmid < phi);
SWAP(plop1, pmid);
/* Sort plo, plop1, phi. */
/* Smaller of rightmost two -> middle. */
if (*plop1 > *phi)
SWAP(plop1, phi);
/* Smallest of all -> left; if plo is already the
smallest, the sort is complete.
*/
if (*plo > *plop1) {
SWAP(plo, plop1);
/* Largest of all -> right. */
if (*plop1 > *phi)
SWAP(plop1, phi);
}
pivot = *plop1;
pi = plop1;
}
assert(*plo <= pivot);
assert(*pi == pivot);
assert(*phi >= pivot);
pj = phi;
/* Partition wrt pivot. This is the time-critical part, and
nearly every decision in the routine aims at making this
loop as fast as possible -- even small points like
arranging that all loop tests can be done correctly at the
bottoms of loops instead of the tops, and that pointers can
be derefenced directly as-is (without fiddly +1 or -1).
The aim is to make the C here so simple that a compiler
has a good shot at doing as well as hand-crafted assembler.
*/
for (;;) {
/* Invariants:
1. pi < pj.
2. All elements at plo, plo+1 .. pi are <= pivot.
3. All elements at pj, pj+1 .. phi are >= pivot.
4. There is an element >= pivot to the right of pi.
5. There is an element <= pivot to the left of pj.
Note that #4 and #5 save us from needing to check
that the pointers stay in bounds.
*/
assert(pi < pj);
do { ++pi; } while (*pi < pivot);
assert(pi <= pj);
do { --pj; } while (*pj > pivot);
assert(pj >= pi - 1);
if (pi < pj)
SWAP(pi, pj);
else
break;
}
assert(plo+1 < pi && pi <= phi);
assert(plo < pj && pj < phi);
assert(*pi >= pivot);
assert( (pi == pj && *pj == pivot) ||
(pj + 1 == pi && *pj <= pivot) );
/* Swap pivot into its final position, pj. */
assert(plo[1] == pivot);
plo[1] = *pj;
*pj = pivot;
/* Subfiles are from plo to pj-1 inclusive, and pj+1 to phi
inclusive. Push the larger one, and loop back to do the
smaller one directly.
*/
if (pj - plo >= phi - pj) {
PUSH(plo, pj-1);
plo = pj+1;
}
else {
PUSH(pj+1, phi);
phi = pj-1;
}
}
#undef PUSH
#undef SWAP
}
/* Sort p and remove duplicates, as fast as we can. */
static size_t
sort_int4_nodups(int *p, size_t n)
{
size_t nunique;
element_type *work;
assert(sizeof(int) == sizeof(element_type));
assert(p);
/* Use quicksort if the array is small, OR if malloc can't find
enough temp memory for radixsort.
*/
work = NULL;
if (n > QUICKSORT_BEATS_RADIXSORT)
work = (element_type *)malloc(n * sizeof(element_type));
if (work) {
element_type *out = radixsort_int4(p, work, n);
nunique = uniq(p, out, n);
free(work);
}
else {
quicksort(p, n);
nunique = uniq(p, p, n);
}
return nunique;
}
ZODB/src/BTrees/BTreeModuleTemplate.c 0000640 0001750 0000144 00000036306 10276710420 017153 0 ustar ctheune users /*****************************************************************************
Copyright (c) 2001, 2002 Zope Corporation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
#include "Python.h"
/* include structmember.h for offsetof */
#include "structmember.h"
#ifdef PERSISTENT
#include "persistent/cPersistence.h"
#else
#define PER_USE_OR_RETURN(self, NULL)
#define PER_ALLOW_DEACTIVATION(self)
#define PER_PREVENT_DEACTIVATION(self)
#define PER_DEL(self)
#define PER_USE(O) 1
#define PER_ACCESSED(O) 1
#endif
/* So sue me. This pair gets used all over the place, so much so that it
* interferes with understanding non-persistence parts of algorithms.
* PER_UNUSE can be used after a successul PER_USE or PER_USE_OR_RETURN.
* It allows the object to become ghostified, and tells the persistence
* machinery that the object's fields were used recently.
*/
#define PER_UNUSE(OBJ) do { \
PER_ALLOW_DEACTIVATION(OBJ); \
PER_ACCESSED(OBJ); \
} while (0)
/*
The tp_name slots of the various BTree types contain the fully
qualified names of the types, e.g. zodb.btrees.OOBTree.OOBTree.
The full name is usd to support pickling and because it is not
possible to modify the __module__ slot of a type dynamically. (This
may be a bug in Python 2.2).
*/
#define MODULE_NAME "BTrees._" MOD_NAME_PREFIX "BTree."
static PyObject *sort_str, *reverse_str, *__setstate___str,
*_bucket_type_str;
static PyObject *ConflictError = NULL;
static void PyVar_Assign(PyObject **v, PyObject *e) { Py_XDECREF(*v); *v=e;}
#define ASSIGN(V,E) PyVar_Assign(&(V),(E))
#define UNLESS(E) if (!(E))
#define OBJECT(O) ((PyObject*)(O))
#define MIN_BUCKET_ALLOC 16
#define MAX_BTREE_SIZE(B) DEFAULT_MAX_BTREE_SIZE
#define MAX_BUCKET_SIZE(B) DEFAULT_MAX_BUCKET_SIZE
#define SameType_Check(O1, O2) ((O1)->ob_type==(O2)->ob_type)
#define ASSERT(C, S, R) if (! (C)) { \
PyErr_SetString(PyExc_AssertionError, (S)); return (R); }
/* Various kinds of BTree and Bucket structs are instances of
* "sized containers", and have a common initial layout:
* The stuff needed for all Python objects, or all Persistent objects.
* int size: The maximum number of things that could be contained
* without growing the container.
* int len: The number of things currently contained.
*
* Invariant: 0 <= len <= size.
*
* A sized container typically goes on to declare one or more pointers
* to contiguous arrays with 'size' elements each, the initial 'len' of
* which are currently in use.
*/
#ifdef PERSISTENT
#define sizedcontainer_HEAD \
cPersistent_HEAD \
int size; \
int len;
#else
#define sizedcontainer_HEAD \
PyObject_HEAD \
int size; \
int len;
#endif
/* Nothing is actually of type Sized, but (pointers to) BTree nodes and
* Buckets can be cast to Sized* in contexts that only need to examine
* the members common to all sized containers.
*/
typedef struct Sized_s {
sizedcontainer_HEAD
} Sized;
#define SIZED(O) ((Sized*)(O))
/* A Bucket wraps contiguous vectors of keys and values. Keys are unique,
* and stored in sorted order. The 'values' pointer may be NULL if the
* Bucket is used to implement a set. Buckets serving as leafs of BTrees
* are chained together via 'next', so that the entire BTree contents
* can be traversed in sorted order quickly and easily.
*/
typedef struct Bucket_s {
sizedcontainer_HEAD
struct Bucket_s *next; /* the bucket with the next-larger keys */
KEY_TYPE *keys; /* 'len' keys, in increasing order */
VALUE_TYPE *values; /* 'len' corresponding values; NULL if a set */
} Bucket;
#define BUCKET(O) ((Bucket*)(O))
/* A BTree is complicated. See Maintainer.txt.
*/
typedef struct BTreeItem_s {
KEY_TYPE key;
Sized *child; /* points to another BTree, or to a Bucket of some sort */
} BTreeItem;
typedef struct BTree_s {
sizedcontainer_HEAD
/* firstbucket points to the bucket containing the smallest key in
* the BTree. This is found by traversing leftmost child pointers
* (data[0].child) until reaching a Bucket.
*/
Bucket *firstbucket;
/* The BTree points to 'len' children, via the "child" fields of the data
* array. There are len-1 keys in the 'key' fields, stored in increasing
* order. data[0].key is unused. For i in 0 .. len-1, all keys reachable
* from data[i].child are >= data[i].key and < data[i+1].key, at the
* endpoints pretending that data[0].key is minus infinity and
* data[len].key is positive infinity.
*/
BTreeItem *data;
} BTree;
static PyTypeObject BTreeType;
static PyTypeObject BucketType;
#define BTREE(O) ((BTree*)(O))
/* Use BTREE_SEARCH to find which child pointer to follow.
* RESULT An int lvalue to hold the index i such that SELF->data[i].child
* is the correct node to search next.
* SELF A pointer to a BTree node.
* KEY The key you're looking for, of type KEY_TYPE.
* ONERROR What to do if key comparison raises an exception; for example,
* perhaps 'return NULL'.
*
* See Maintainer.txt for discussion: this is optimized in subtle ways.
* It's recommended that you call this at the start of a routine, waiting
* to check for self->len == 0 after.
*/
#define BTREE_SEARCH(RESULT, SELF, KEY, ONERROR) { \
int _lo = 0; \
int _hi = (SELF)->len; \
int _i, _cmp; \
for (_i = _hi >> 1; _i > _lo; _i = (_lo + _hi) >> 1) { \
TEST_KEY_SET_OR(_cmp, (SELF)->data[_i].key, (KEY)) \
ONERROR; \
if (_cmp < 0) _lo = _i; \
else if (_cmp > 0) _hi = _i; \
else /* equal */ break; \
} \
(RESULT) = _i; \
}
/* SetIteration structs are used in the internal set iteration protocol.
* When you want to iterate over a set or bucket or BTree (even an
* individual key!),
* 1. Declare a new iterator:
* SetIteration si = {0,0,0};
* Using "{0,0,0}" or "{0,0}" appear most common. Only one {0} is
* necssary. At least one must be given so that finiSetIteration() works
* correctly even if you don't get around to calling initSetIteration().
* 2. Initialize it via
* initSetIteration(&si, PyObject *s, useValues)
* It's an error if that returns an int < 0. In case of error on the
* init call, calling finiSetIteration(&si) is optional. But if the
* init call succeeds, you must eventually call finiSetIteration(),
* and whether or not subsequent calls to si.next() fail.
* 3. Get the first element:
* if (si.next(&si) < 0) { there was an error }
* If the set isn't empty, this sets si.position to an int >= 0,
* si.key to the element's key (of type KEY_TYPE), and maybe si.value to
* the element's value (of type VALUE_TYPE). si.value is defined
* iff si.usesValue is true.
* 4. Process all the elements:
* while (si.position >= 0) {
* do something with si.key and/or si.value;
* if (si.next(&si) < 0) { there was an error; }
* }
* 5. Finalize the SetIterator:
* finiSetIteration(&si);
* This is mandatory! si may contain references to iterator objects,
* keys and values, and they must be cleaned up else they'll leak. If
* this were C++ we'd hide that in the destructor, but in C you have to
* do it by hand.
*/
typedef struct SetIteration_s
{
PyObject *set; /* the set, bucket, BTree, ..., being iterated */
int position; /* initialized to 0; set to -1 by next() when done */
int usesValue; /* true iff 'set' has values & we iterate them */
KEY_TYPE key; /* next() sets to next key */
VALUE_TYPE value; /* next() may set to next value */
int (*next)(struct SetIteration_s*); /* function to get next key+value */
} SetIteration;
/* Finish the set iteration protocol. This MUST be called by everyone
* who starts a set iteration, unless the initial call to initSetIteration
* failed; in that case, and only that case, calling finiSetIteration is
* optional.
*/
static void
finiSetIteration(SetIteration *i)
{
assert(i != NULL);
if (i->set == NULL)
return;
Py_DECREF(i->set);
i->set = NULL; /* so it doesn't hurt to call this again */
if (i->position > 0) {
/* next() was called at least once, but didn't finish iterating
* (else position would be negative). So the cached key and
* value need to be cleaned up.
*/
DECREF_KEY(i->key);
if (i->usesValue) {
DECREF_VALUE(i->value);
}
}
i->position = -1; /* stop any stray next calls from doing harm */
}
static PyObject *
IndexError(int i)
{
PyObject *v;
v = PyInt_FromLong(i);
if (!v) {
v = Py_None;
Py_INCREF(v);
}
PyErr_SetObject(PyExc_IndexError, v);
Py_DECREF(v);
return NULL;
}
/* Search for the bucket immediately preceding *current, in the bucket chain
* starting at first. current, *current and first must not be NULL.
*
* Return:
* 1 *current holds the correct bucket; this is a borrowed reference
* 0 no such bucket exists; *current unaltered
* -1 error; *current unaltered
*/
static int
PreviousBucket(Bucket **current, Bucket *first)
{
Bucket *trailing = NULL; /* first travels; trailing follows it */
int result = 0;
assert(current && *current && first);
if (first == *current)
return 0;
do {
trailing = first;
PER_USE_OR_RETURN(first, -1);
first = first->next;
PER_UNUSE(trailing);
if (first == *current) {
*current = trailing;
result = 1;
break;
}
} while (first);
return result;
}
static void *
BTree_Malloc(size_t sz)
{
void *r;
ASSERT(sz > 0, "non-positive size malloc", NULL);
r = malloc(sz);
if (r)
return r;
PyErr_NoMemory();
return NULL;
}
static void *
BTree_Realloc(void *p, size_t sz)
{
void *r;
ASSERT(sz > 0, "non-positive size realloc", NULL);
if (p)
r = realloc(p, sz);
else
r = malloc(sz);
UNLESS (r)
PyErr_NoMemory();
return r;
}
/* Shared keyword-argument list for BTree/Bucket
* (iter)?(keys|values|items)
*/
static char *search_keywords[] = {"min", "max",
"excludemin", "excludemax",
0};
#include "BTreeItemsTemplate.c"
#include "BucketTemplate.c"
#include "SetTemplate.c"
#include "BTreeTemplate.c"
#include "TreeSetTemplate.c"
#include "SetOpTemplate.c"
#include "MergeTemplate.c"
static struct PyMethodDef module_methods[] = {
{"difference", (PyCFunction) difference_m, METH_VARARGS,
"difference(o1, o2) -- "
"compute the difference between o1 and o2"
},
{"union", (PyCFunction) union_m, METH_VARARGS,
"union(o1, o2) -- compute the union of o1 and o2\n"
},
{"intersection", (PyCFunction) intersection_m, METH_VARARGS,
"intersection(o1, o2) -- "
"compute the intersection of o1 and o2"
},
#ifdef MERGE
{"weightedUnion", (PyCFunction) wunion_m, METH_VARARGS,
"weightedUnion(o1, o2 [, w1, w2]) -- compute the union of o1 and o2\n"
"\nw1 and w2 are weights."
},
{"weightedIntersection", (PyCFunction) wintersection_m, METH_VARARGS,
"weightedIntersection(o1, o2 [, w1, w2]) -- "
"compute the intersection of o1 and o2\n"
"\nw1 and w2 are weights."
},
#endif
#ifdef MULTI_INT_UNION
{"multiunion", (PyCFunction) multiunion_m, METH_VARARGS,
"multiunion(seq) -- compute union of a sequence of integer sets.\n"
"\n"
"Each element of seq must be an integer set, or convertible to one\n"
"via the set iteration protocol. The union returned is an IISet."
},
#endif
{NULL, NULL} /* sentinel */
};
static char BTree_module_documentation[] =
"\n"
MASTER_ID
BTREEITEMSTEMPLATE_C
"$Id: BTreeModuleTemplate.c 37871 2005-08-11 17:55:28Z fdrake $\n"
BTREETEMPLATE_C
BUCKETTEMPLATE_C
KEYMACROS_H
MERGETEMPLATE_C
SETOPTEMPLATE_C
SETTEMPLATE_C
TREESETTEMPLATE_C
VALUEMACROS_H
BTREEITEMSTEMPLATE_C
;
int
init_persist_type(PyTypeObject *type)
{
type->ob_type = &PyType_Type;
type->tp_base = cPersistenceCAPI->pertype;
if (PyType_Ready(type) < 0)
return 0;
return 1;
}
void
INITMODULE (void)
{
PyObject *m, *d, *c;
sort_str = PyString_InternFromString("sort");
if (!sort_str)
return;
reverse_str = PyString_InternFromString("reverse");
if (!reverse_str)
return;
__setstate___str = PyString_InternFromString("__setstate__");
if (!__setstate___str)
return;
_bucket_type_str = PyString_InternFromString("_bucket_type");
if (!_bucket_type_str)
return;
/* Grab the ConflictError class */
m = PyImport_ImportModule("ZODB.POSException");
if (m != NULL) {
c = PyObject_GetAttrString(m, "BTreesConflictError");
if (c != NULL)
ConflictError = c;
Py_DECREF(m);
}
if (ConflictError == NULL) {
Py_INCREF(PyExc_ValueError);
ConflictError=PyExc_ValueError;
}
/* Initialize the PyPersist_C_API and the type objects. */
cPersistenceCAPI = PyCObject_Import("persistent.cPersistence", "CAPI");
if (cPersistenceCAPI == NULL)
return;
BTreeItemsType.ob_type = &PyType_Type;
BTreeIter_Type.ob_type = &PyType_Type;
BTreeIter_Type.tp_getattro = PyObject_GenericGetAttr;
BucketType.tp_new = PyType_GenericNew;
SetType.tp_new = PyType_GenericNew;
BTreeType.tp_new = PyType_GenericNew;
TreeSetType.tp_new = PyType_GenericNew;
if (!init_persist_type(&BucketType))
return;
if (!init_persist_type(&BTreeType))
return;
if (!init_persist_type(&SetType))
return;
if (!init_persist_type(&TreeSetType))
return;
if (PyDict_SetItem(BTreeType.tp_dict, _bucket_type_str,
(PyObject *)&BucketType) < 0) {
fprintf(stderr, "btree failed\n");
return;
}
if (PyDict_SetItem(TreeSetType.tp_dict, _bucket_type_str,
(PyObject *)&SetType) < 0) {
fprintf(stderr, "bucket failed\n");
return;
}
/* Create the module and add the functions */
m = Py_InitModule4("_" MOD_NAME_PREFIX "BTree",
module_methods, BTree_module_documentation,
(PyObject *)NULL, PYTHON_API_VERSION);
/* Add some symbolic constants to the module */
d = PyModule_GetDict(m);
if (PyDict_SetItemString(d, MOD_NAME_PREFIX "Bucket",
(PyObject *)&BucketType) < 0)
return;
if (PyDict_SetItemString(d, MOD_NAME_PREFIX "BTree",
(PyObject *)&BTreeType) < 0)
return;
if (PyDict_SetItemString(d, MOD_NAME_PREFIX "Set",
(PyObject *)&SetType) < 0)
return;
if (PyDict_SetItemString(d, MOD_NAME_PREFIX "TreeSet",
(PyObject *)&TreeSetType) < 0)
return;
if (PyDict_SetItemString(d, MOD_NAME_PREFIX "TreeIterator",
(PyObject *)&BTreeIter_Type) < 0)
return;
}
ZODB/src/BTrees/intvaluemacros.h 0000640 0001750 0000144 00000001536 10051217333 016342 0 ustar ctheune users
#define VALUEMACROS_H "$Id: intvaluemacros.h 24682 2004-05-14 19:32:43Z tim_one $\n"
#define VALUE_TYPE int
#undef VALUE_TYPE_IS_PYOBJECT
#define TEST_VALUE(K, T) (((K) < (T)) ? -1 : (((K) > (T)) ? 1: 0))
#define VALUE_SAME(VALUE, TARGET) ( (VALUE) == (TARGET) )
#define DECLARE_VALUE(NAME) VALUE_TYPE NAME
#define VALUE_PARSE "i"
#define DECREF_VALUE(k)
#define INCREF_VALUE(k)
#define COPY_VALUE(V, E) (V=(E))
#define COPY_VALUE_TO_OBJECT(O, K) O=PyInt_FromLong(K)
#define COPY_VALUE_FROM_ARG(TARGET, ARG, STATUS) \
if (PyInt_Check(ARG)) TARGET=PyInt_AsLong(ARG); else { \
PyErr_SetString(PyExc_TypeError, "expected integer value"); \
(STATUS)=0; (TARGET)=0; }
#define NORMALIZE_VALUE(V, MIN) ((MIN) > 0) ? ((V)/=(MIN)) : 0
#define MERGE_DEFAULT 1
#define MERGE(O1, w1, O2, w2) ((O1)*(w1)+(O2)*(w2))
#define MERGE_WEIGHT(O, w) ((O)*(w))
ZODB/src/BTrees/OOBTree.py 0000640 0001750 0000144 00000001307 10057366465 014763 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# hack to overcome dynamic-linking headache.
from _OOBTree import *
ZODB/src/BTrees/floatvaluemacros.h 0000640 0001750 0000144 00000001676 10156161407 016670 0 ustar ctheune users
#define VALUEMACROS_H "$Id: floatvaluemacros.h 28617 2004-12-09 23:43:03Z tim_one $\n"
#define VALUE_TYPE float
#undef VALUE_TYPE_IS_PYOBJECT
#define TEST_VALUE(K, T) (((K) < (T)) ? -1 : (((K) > (T)) ? 1: 0))
#define VALUE_SAME(VALUE, TARGET) ( (VALUE) == (TARGET) )
#define DECLARE_VALUE(NAME) VALUE_TYPE NAME
#define VALUE_PARSE "f"
#define DECREF_VALUE(k)
#define INCREF_VALUE(k)
#define COPY_VALUE(V, E) (V=(E))
#define COPY_VALUE_TO_OBJECT(O, K) O=PyFloat_FromDouble(K)
#define COPY_VALUE_FROM_ARG(TARGET, ARG, STATUS) \
if (PyFloat_Check(ARG)) TARGET = (float)PyFloat_AsDouble(ARG); \
else if (PyInt_Check(ARG)) TARGET = (float)PyInt_AsLong(ARG); \
else { \
PyErr_SetString(PyExc_TypeError, "expected float or int value"); \
(STATUS)=0; (TARGET)=0; }
#define NORMALIZE_VALUE(V, MIN) ((MIN) > 0) ? ((V)/=(MIN)) : 0
#define MERGE_DEFAULT 1.0f
#define MERGE(O1, w1, O2, w2) ((O1)*(w1)+(O2)*(w2))
#define MERGE_WEIGHT(O, w) ((O)*(w))
ZODB/src/BTrees/intkeymacros.h 0000640 0001750 0000144 00000001161 10051217333 016010 0 ustar ctheune users
#define KEYMACROS_H "$Id: intkeymacros.h 24682 2004-05-14 19:32:43Z tim_one $\n"
#define KEY_TYPE int
#undef KEY_TYPE_IS_PYOBJECT
#define KEY_CHECK PyInt_Check
#define TEST_KEY_SET_OR(V, K, T) if ( ( (V) = (((K) < (T)) ? -1 : (((K) > (T)) ? 1: 0)) ) , 0 )
#define DECREF_KEY(KEY)
#define INCREF_KEY(k)
#define COPY_KEY(KEY, E) (KEY=(E))
#define COPY_KEY_TO_OBJECT(O, K) O=PyInt_FromLong(K)
#define COPY_KEY_FROM_ARG(TARGET, ARG, STATUS) \
if (PyInt_Check(ARG)) TARGET=PyInt_AS_LONG(ARG); else { \
PyErr_SetString(PyExc_TypeError, "expected integer key"); \
(STATUS)=0; (TARGET)=0; }
#define MULTI_INT_UNION 1
ZODB/src/BTrees/_IFBTree.c 0000640 0001750 0000144 00000002112 10156136553 014661 0 ustar ctheune users /*############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
############################################################################*/
#define MASTER_ID "$Id: _IFBTree.c 28613 2004-12-09 21:02:35Z tim_one $\n"
/* IIBTree - int key, int value BTree
Implements a collection using int type keys
and int type values
*/
/* Setup template macros */
#define PERSISTENT
#define MOD_NAME_PREFIX "IF"
#define INITMODULE init_IFBTree
#define DEFAULT_MAX_BUCKET_SIZE 120
#define DEFAULT_MAX_BTREE_SIZE 500
#include "intkeymacros.h"
#include "floatvaluemacros.h"
#include "BTreeModuleTemplate.c"
ZODB/src/BTrees/Development.txt 0000640 0001750 0000144 00000040741 10375443702 016173 0 ustar ctheune users =====================
Developer Information
=====================
This document provides information for developers who maintain or extend
`BTrees`.
Macros
======
`BTrees` are defined using a "template", roughly akin to a C++ template. To
create a new family of `BTrees`, create a source file that defines macros used
to handle differences in key and value types:
Configuration Macros
--------------------
``MASTER_ID``
A string to hold an RCS/CVS Id key to be included in compiled binaries.
``MOD_NAME_PREFIX``
A string (like "IO" or "OO") that provides the prefix used for the module.
This gets used to generate type names and the internal module name string.
``DEFAULT_MAX_BUCKET_SIZE``
An int giving the maximum bucket size (number of key/value pairs). When a
bucket gets larger than this due to an insertion *into a BTREE*, it
splits. Inserting into a bucket directly doesn't split, and functions
that produce a bucket output (e.g., ``union()``) also have no bound on how
large a bucket may get. Someday this will be tunable on `BTree`.
instances.
``DEFAULT_MAX_BTREE_SIZE``
An ``int`` giving the maximum size (number of children) of an internal
btree node. Someday this will be tunable on ``BTree`` instances.
Macros for Keys
---------------
``KEY_TYPE``
The C type declaration for keys (e.g., ``int`` or ``PyObject*``).
``KEY_TYPE_IS_PYOBJECT``
Define if ``KEY_TYPE`` is a ``PyObject*`, else ``undef``.
``KEY_CHECK(K)``
Tests whether the ``PyObject* K`` can be converted to the (``C``) key type
(``KEY_TYPE``). The macro should return a boolean (zero for false,
non-zero for true). When it returns false, its caller should probably set
a ``TypeError`` exception.
``TEST_KEY_SET_OR(V, K, T)``
Like Python's ``cmp()``. Compares K(ey) to T(arget), where ``K``
and ``T`` are ``C`` values of type `KEY_TYPE`. ``V`` is assigned an `int`
value depending on the outcome::
< 0 if K < T
== 0 if K == T
> 0 if K > T
This macro acts like an ``if``, where the following statement is executed
only if a Python exception has been raised because the values could not be
compared.
``DECREF_KEY(K)``
``K`` is a value of ``KEY_TYPE``. If ``KEY_TYPE`` is a flavor of
``PyObject*``, write this to do ``Py_DECREF(K)``. Else (e.g.,
``KEY_TYPE`` is ``int``) make it a nop.
`INCREF_KEY(K)`
``K`` is a value of `KEY_TYPE`. If `KEY_TYPE` is a flavor of
``PyObject*``, write this to do ``Py_INCREF(K)``. Else (e.g., `KEY_TYPE`
is ``int``) make it a nop.
``COPY_KEY(K, E)``
Like ``K=E``. Copy a key from ``E`` to ``K``, both of ``KEY_TYPE``. Note
that this doesn't ``decref K`` or ``incref E`` when ``KEY_TYPE`` is a
``PyObject*``; the caller is responsible for keeping refcounts straight.
``COPY_KEY_TO_OBJECT(O, K)``
Roughly like ``O=K``. ``O`` is a ``PyObject*``, and the macro must build
a Python object form of ``K``, assign it to ``O``, and ensure that ``O``
owns the reference to its new value. It may do this by creating a new
Python object based on ``K`` (e.g., ``PyInt_FromLong(K)`` when
``KEY_TYPE`` is ``int``), or simply by doing ``Py_INCREF(K)`` if
``KEY_TYPE`` is a ``PyObject*``.
``COPY_KEY_FROM_ARG(TARGET, ARG, STATUS)``
Copy an argument to the target without creating a new reference to
``ARG``. ``ARG`` is a ``PyObject*``, and ``TARGET`` is of type
``KEY_TYPE``. If this can't be done (for example, ``KEY_CHECK(ARG)``
returns false), set a Python error and set status to ``0``. If there is
no error, leave status alone.
Macros for Values
-----------------
``VALUE_TYPE``
The C type declaration for values (e.g., ``int`` or ``PyObject*``).
``VALUE_TYPE_IS_PYOBJECT``
Define if ``VALUE_TYPE`` is a ``PyObject*``, else ``undef``.
``TEST_VALUE(X, Y)``
Like Python's ``cmp()``. Compares ``X`` to ``Y``, where ``X`` & ``Y`` are
``C`` values of type ``VALUE_TYPE``. The macro returns an ``int``, with
value::
< 0 if X < Y
== 0 if X == Y
> 0 if X > Y
Bug: There is no provision for determining whether the comparison attempt
failed (set a Python exception).
``DECREF_VALUE(K)``
Like ``DECREF_KEY``, except applied to values of ``VALUE_TYPE``.
``INCREF_VALUE(K)``
Like ``INCREF_KEY``, except applied to values of ``VALUE_TYPE``.
``COPY_VALUE(K, E)``
Like ``COPY_KEY``, except applied to values of ``VALUE_TYPE``.
``COPY_VALUE_TO_OBJECT(O, K)``
Like ``COPY_KEY_TO_OBJECT``, except applied to values of ``VALUE_TYPE``.
``COPY_VALUE_FROM_ARG(TARGET, ARG, STATUS)``
Like ``COPY_KEY_FROM_ARG``, except applied to values of ``VALUE_TYPE``.
``NORMALIZE_VALUE(V, MIN)``
Normalize the value, ``V``, using the parameter ``MIN``. This is almost
certainly a YAGNI. It is a no-op for most types. For integers, ``V`` is
replaced by ``V/MIN`` only if ``MIN > 0``.
Macros for Set Operations
-------------------------
``MERGE_DEFAULT``
A value of ``VALUE_TYPE`` specifying the value to associate with set
elements when sets are merged with mappings via weighed union or weighted
intersection.
``MERGE(O1, w1, O2, w2)``
Performs a weighted merge of two values, ``O1`` and ``O2``, using weights
``w1`` and ``w2``. The result must be of ``VALUE_TYPE``. Note that
weighted unions and weighted intersections are not enabled if this macro
is left undefined.
``MERGE_WEIGHT(O, w)``
Computes a weighted value for ``O``. The result must be of
``VALUE_TYPE``. This is used for "filling out" weighted unions, i.e. to
compute a weighted value for keys that appear in only one of the input
mappings. If left undefined, ``MERGE_WEIGHT`` defaults to::
#define MERGE_WEIGHT(O, w) (O)
``MULTI_INT_UNION``
The value doesn't matter. If defined, `SetOpTemplate.c` compiles code for
a ``multiunion()`` function (compute a union of many input sets at high
speed). This currently makes sense only for structures with integer keys.
BTree Clues
===========
More or less random bits of helpful info.
+ In papers and textbooks, this flavor of BTree is usually called a B+-Tree,
where "+" is a superscript.
+ All keys and all values live in the bucket leaf nodes. Keys in interior
(BTree) nodes merely serve to guide a search efficiently toward the correct
leaf.
+ When a key is deleted, it's physically removed from the bucket it's in, but
this doesn't propagate back up the tree: since keys in interior nodes only
serve to guide searches, it's OK-- and saves time --to leave "stale" keys in
interior nodes.
+ No attempt is made to rebalance the tree after a deletion, unless a bucket
thereby becomes entirely empty. "Classic BTrees" do rebalance, keeping all
buckets at least half full (provided there are enough keys in the entire
tree to fill half a bucket). The tradeoffs are murky. Pathological cases
in the presence of deletion do exist. Pathologies include trees tending
toward only one key per bucket, and buckets at differing depths (all buckets
are at the same depth in a classic BTree).
+ ``DEFAULT_MAX_BUCKET_SIZE`` and ``DEFAULT_MAX_BTREE_SIZE`` are chosen mostly
to "even out" pickle sizes in storage. That's why, e.g., an `IIBTree` has
larger values than an `OOBTree`: pickles store ints more efficiently than
they can store arbitrary Python objects.
+ In a non-empty BTree, every bucket node contains at least one key, and every
BTree node contains at least one child and a non-NULL firstbucket pointer.
However, a BTree node may not contain any keys.
+ An empty BTree consists solely of a BTree node with ``len==0`` and
``firstbucket==NULL``.
+ Although a BTree can become unbalanced under a mix of inserts and deletes
(meaning both that there's nothing stronger that can be said about buckets
than that they're not empty, and that buckets can appear at different
depths), a BTree node always has children of the same kind: they're all
buckets, or they're all BTree nodes.
The ``BTREE_SEARCH`` Macro
==========================
For notational ease, consider a fixed BTree node ``x``, and let
::
K(i) mean x->data.key[i]
C(i) mean all the keys reachable from x->data.child[i]
For each ``i`` in ``0`` to ``x->len-1`` inclusive,
::
K(i) <= C(i) < K(i+1)
is a BTree node invariant, where we pretend that ``K(0)`` holds a key smaller
than any possible key, and ``K(x->len)`` holds a key larger than any possible
key. (Note that ``K(x->len)`` doesn't actually exist, and ``K(0)`` is never
used although space for it exists in non-empty BTree nodes.)
When searching for a key ``k``, then, the child pointer we want to follow is
the one at index ``i`` such that ``K(i) <= k < K(i+1)``. There can be at most
one such ``i``, since the ``K(i)`` are strictly increasing. And there is at
least one such ``i`` provided the tree isn't empty (so that ``0 < len``). For
the moment, assume the tree isn't empty (we'll get back to that later).
The macro's chief loop invariant is
::
K(lo) < k < K(hi)
This holds trivially at the start, since ``lo`` is set to ``0``, and ``hi`` to
``x->len``, and we pretend ``K(0)`` is minus infinity and ``K(len)`` is plus
infinity. Inside the loop, if ``K(i) < k`` we set ``lo`` to ``i``, and if
``K(i) > k`` we set ``hi`` to ``i``. These obviously preserve the invariant.
If ``K(i) == k``, the loop breaks and sets the result to ``i``, and since
``K(i) == k`` in that case ``i`` is obviously the correct result.
Other cases depend on how ``i = floor((lo + hi)/2)`` works, exactly. Suppose
``lo + d = hi`` for some ``d >= 0``. Then ``i = floor((lo + lo + d)/2) =
floor(lo + d/2) = lo + floor(d/2)``. So:
a. ``[d == 0] (lo == i == hi)`` if and only if ``(lo == hi)``.
b. ``[d == 1] (lo == i < hi)`` if and only if ``(lo+1 == hi)``.
c. ``[d > 1] (lo < i < hi)`` if and only if ``(lo+1 < hi)``.
If the node is empty ``(x->len == 0)``, then ``lo==i==hi==0`` at the start,
and the loop exits immediately (the first ``i > lo`` test fails), without
entering the body.
Else ``lo < hi`` at the start, and the invariant ``K(lo) < k < K(hi)`` holds.
If ``lo+1 < hi``, we're in case (c): ``i`` is strictly between ``lo`` and
``hi``, so the loop body is entered, and regardless of whether the body sets
the new ``lo`` or the new ``hi`` to ``i``, the new ``lo`` is strictly less
than the new ``hi``, and the difference between the new ``lo`` and new ``hi``
is strictly less than the difference between the old ``lo`` and old ``hi``.
So long as the new ``lo + 1`` remains < the new ``hi``, we stay in this case.
We can't stay in this case forever, though: because ``hi-lo`` decreases on
each trip but remains > ``0``, ``lo+1 == hi`` must eventually become true.
(In fact, it becomes true quickly, in about ``log2(x->len)`` trips; the point
is more that ``lo`` doesn't equal ``hi`` when the loop ends, it has to end
with ``lo+1==hi`` and ``i==lo``).
Then we're in case (b): ``i==lo==hi-1`` then, and the loop exits. The
invariant still holds, with ``lo==i`` and ``hi==lo+1==i+1``::
K(i) < k < K(i+1)
so ``i`` is again the correct answer.
Optimization points:
--------------------
+ Division by 2 is done via shift rather via "/2". These are signed ints, and
almost all C compilers treat signed int division as truncating, and shifting
is not the same as truncation for signed int division. The compiler has no
way to know these values aren't negative, so has to generate longer-winded
code for "/2". But we know these values aren't negative, and exploit it.
+ The order of _cmp comparisons matters. We're in an interior BTree node, and
are looking at only a tiny fraction of all the keys that exist. So finding
the key exactly in this node is unlikely, and checking ``_cmp == 0`` is a
waste of time to the same extent. It doesn't matter whether we check for
``_cmp < 0`` or ``_cmp > 0`` first, so long as we do both before worrying
about equality.
+ At the start of a routine, it's better to run this macro even if ``x->len``
is ``0`` (check for that afterwards). We just called a function and so
probably drained the pipeline. If the first thing we do then is read up
``self->len`` and check it against ``0``, we just sit there waiting for the
data to get read up, and then another immediate test-and-branch, and for a
very unlikely case (BTree nodes are rarely empty). It's better to get into
the loop right away so the normal case makes progress ASAP.
The ``BUCKET_SEARCH`` Macro
===========================
This has a different job than ``BTREE_SEARCH``: the key ``0`` slot is
legitimate in a bucket, and we want to find the index at which the key
belongs. If the key is larger than the bucket's largest key, a new slot at
index len is where it belongs, else it belongs at the smallest ``i`` with
``keys[i]`` >= the key we're looking for. We also need to know whether or not
the key is present (``BTREE_SEARCH`` didn't care; it only wanted to find the
next node to search).
The mechanics of the search are quite similar, though. The primary
loop invariant changes to (say we're searching for key ``k``)::
K(lo-1) < k < K(hi)
where ``K(i)`` means ``keys[i]``, and we pretend ``K(-1)`` is minus infinity
and ``K(len)`` is plus infinity.
If the bucket is empty, ``lo=hi=i=0`` at the start, the loop body is never
entered, and the macro sets ``INDEX`` to 0 and ``ABSENT`` to true. That's why
``_cmp`` is initialized to 1 (``_cmp`` becomes ``ABSENT``).
Else the bucket is not empty, lok``, ``hi`` is set to ``i``, preserving that ``K[hi] = K[i] > k``.
If the loop exits after either of those, ``_cmp != 0``, so ``ABSENT`` becomes
true.
If ``K[i]=k``, the loop breaks, so that ``INDEX`` becomes ``i``, and
``ABSENT`` becomes false (``_cmp=0`` in this case).
The same case analysis for ``BTREE_SEARCH`` on ``lo`` and ``hi`` holds here:
a. ``(lo == i == hi)`` if and only if ``(lo == hi)``.
b. ``(lo == i < hi)`` if and only if ``(lo+1 == hi)``.
c. ``(lo < i < hi)`` if and only if ``(lo+1 < hi)``.
So long as ``lo+1 < hi``, we're in case (c), and either break with equality
(in which case the right results are obviously computed) or narrow the range.
If equality doesn't obtain, the range eventually narrows to cases (a) or (b).
To go from (c) to (a), we must have ``lo+2==hi`` at the start, and
``K[i]=K[lo+1]
key``), because when it pays it narrows the range more (we get a little
boost from setting ``lo=i+1`` in this case; the other case sets ``hi=i``,
which isn't as much of a narrowing).
ZODB/src/BTrees/SetOpTemplate.c 0000640 0001750 0000144 00000036355 10156136553 016050 0 ustar ctheune users /*****************************************************************************
Copyright (c) 2001, 2002 Zope Corporation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
/****************************************************************************
Set operations
****************************************************************************/
#define SETOPTEMPLATE_C "$Id: SetOpTemplate.c 28613 2004-12-09 21:02:35Z tim_one $\n"
#ifdef KEY_CHECK
static int
nextKeyAsSet(SetIteration *i)
{
if (i->position >= 0) {
if (i->position) {
DECREF_KEY(i->key);
i->position = -1;
}
else
i->position = 1;
}
return 0;
}
#endif
/* initSetIteration
*
* Start the set iteration protocol. See the comments at struct SetIteration.
*
* Arguments
* i The address of a SetIteration control struct.
* s The address of the set, bucket, BTree, ..., to be iterated.
* useValues Boolean; if true, and s has values (is a mapping), copy
* them into i->value each time i->next() is called; else
* ignore s's values even if s is a mapping.
*
* Return
* 0 on success; -1 and an exception set if error.
* i.usesValue is set to 1 (true) if s has values and useValues was
* true; else usesValue is set to 0 (false).
* i.set gets a new reference to s, or to some other object used to
* iterate over s.
* i.position is set to 0.
* i.next is set to an appropriate iteration function.
* i.key and i.value are left alone.
*
* Internal
* i.position < 0 means iteration terminated.
* i.position = 0 means iteration hasn't yet begun (next() hasn't
* been called yet).
* In all other cases, i.key, and possibly i.value, own references.
* These must be cleaned up, either by next() routines, or by
* finiSetIteration.
* next() routines must ensure the above. They should return without
* doing anything when i.position < 0.
* It's the responsibility of {init, fini}setIteration to clean up
* the reference in i.set, and to ensure that no stale references
* live in i.key or i.value if iteration terminates abnormally.
* A SetIteration struct has been cleaned up iff i.set is NULL.
*/
static int
initSetIteration(SetIteration *i, PyObject *s, int useValues)
{
i->set = NULL;
i->position = -1; /* set to 0 only on normal return */
i->usesValue = 0; /* assume it's a set or that values aren't iterated */
if (PyObject_IsInstance(s, (PyObject *)&BucketType))
{
i->set = s;
Py_INCREF(s);
if (useValues)
{
i->usesValue = 1;
i->next = nextBucket;
}
else
i->next = nextSet;
}
else if (PyObject_IsInstance(s, (PyObject *)&SetType))
{
i->set = s;
Py_INCREF(s);
i->next = nextSet;
}
else if (PyObject_IsInstance(s, (PyObject *)&BTreeType))
{
i->set = BTree_rangeSearch(BTREE(s), NULL, NULL, 'i');
UNLESS(i->set) return -1;
if (useValues)
{
i->usesValue = 1;
i->next = nextBTreeItems;
}
else
i->next = nextTreeSetItems;
}
else if (PyObject_IsInstance(s, (PyObject *)&TreeSetType))
{
i->set = BTree_rangeSearch(BTREE(s), NULL, NULL, 'k');
UNLESS(i->set) return -1;
i->next = nextTreeSetItems;
}
#ifdef KEY_CHECK
else if (KEY_CHECK(s))
{
int copied = 1;
COPY_KEY_FROM_ARG(i->key, s, copied);
UNLESS (copied) return -1;
INCREF_KEY(i->key);
i->set = s;
Py_INCREF(s);
i->next = nextKeyAsSet;
}
#endif
else
{
PyErr_SetString(PyExc_TypeError, "invalid argument");
return -1;
}
i->position = 0;
return 0;
}
#ifndef MERGE_WEIGHT
#define MERGE_WEIGHT(O, w) (O)
#endif
static int
copyRemaining(Bucket *r, SetIteration *i, int merge,
/* See comment # 42 */
#ifdef MERGE
VALUE_TYPE w)
#else
int w)
#endif
{
while (i->position >= 0)
{
if(r->len >= r->size && Bucket_grow(r, -1, ! merge) < 0) return -1;
COPY_KEY(r->keys[r->len], i->key);
INCREF_KEY(r->keys[r->len]);
if (merge)
{
COPY_VALUE(r->values[r->len], MERGE_WEIGHT(i->value, w));
INCREF_VALUE(r->values[r->len]);
}
r->len++;
if (i->next(i) < 0) return -1;
}
return 0;
}
/* This is the workhorse for all set merge operations: the weighted and
* unweighted flavors of union and intersection, and set difference. The
* algorithm is conceptually simple but the code is complicated due to all
* the options.
*
* s1, s2
* The input collections to be merged.
*
* usevalues1, usevalues2
* Booleans. In the output, should values from s1 (or s2) be used? This
* only makes sense when an operation intends to support mapping outputs;
* these should both be false for operations that want pure set outputs.
*
* w1, w2
* If usevalues1(2) are true, these are the weights to apply to the
* input values.
*
* c1
* Boolean. Should keys that appear in c1 but not c2 appear in the output?
* c12
* Boolean. Should keys that appear in both inputs appear in the output?
* c2
* Boolean. Should keys that appear in c2 but not c1 appear in the output?
*
* Returns NULL if error, else a Set or Bucket, depending on whether a set or
* mapping was requested.
*/
static PyObject *
set_operation(PyObject *s1, PyObject *s2,
int usevalues1, int usevalues2,
/* Comment # 42
The following ifdef works around a template/type problem
Weights are passed as integers. In particular, the weight passed by
difference is one. This works find in the int value and float value
cases but makes no sense in the object value case. In the object
value case, we don't do merging, so we don't use the weights, so it
doesn't matter what they are.
*/
#ifdef MERGE
VALUE_TYPE w1, VALUE_TYPE w2,
#else
int w1, int w2,
#endif
int c1, int c12, int c2)
{
Bucket *r=0;
SetIteration i1 = {0,0,0}, i2 = {0,0,0};
int cmp, merge;
if (initSetIteration(&i1, s1, usevalues1) < 0) goto err;
if (initSetIteration(&i2, s2, usevalues2) < 0) goto err;
merge = i1.usesValue | i2.usesValue;
if (merge)
{
#ifndef MERGE
if (c12 && i1.usesValue && i2.usesValue) goto invalid_set_operation;
#endif
if (! i1.usesValue&& i2.usesValue)
{
SetIteration t;
int i;
/* See comment # 42 above */
#ifdef MERGE
VALUE_TYPE v;
#else
int v;
#endif
t=i1; i1=i2; i2=t;
i=c1; c1=c2; c2=i;
v=w1; w1=w2; w2=v;
}
#ifdef MERGE_DEFAULT
i1.value=MERGE_DEFAULT;
i2.value=MERGE_DEFAULT;
#else
if (i1.usesValue)
{
if (! i2.usesValue && c2) goto invalid_set_operation;
}
else
{
if (c1 || c12) goto invalid_set_operation;
}
#endif
UNLESS(r=BUCKET(PyObject_CallObject(OBJECT(&BucketType), NULL)))
goto err;
}
else
{
UNLESS(r=BUCKET(PyObject_CallObject(OBJECT(&SetType), NULL)))
goto err;
}
if (i1.next(&i1) < 0) goto err;
if (i2.next(&i2) < 0) goto err;
while (i1.position >= 0 && i2.position >= 0)
{
TEST_KEY_SET_OR(cmp, i1.key, i2.key) goto err;
if(cmp < 0)
{
if(c1)
{
if(r->len >= r->size && Bucket_grow(r, -1, ! merge) < 0) goto err;
COPY_KEY(r->keys[r->len], i1.key);
INCREF_KEY(r->keys[r->len]);
if (merge)
{
COPY_VALUE(r->values[r->len], MERGE_WEIGHT(i1.value, w1));
INCREF_VALUE(r->values[r->len]);
}
r->len++;
}
if (i1.next(&i1) < 0) goto err;
}
else if(cmp==0)
{
if(c12)
{
if(r->len >= r->size && Bucket_grow(r, -1, ! merge) < 0) goto err;
COPY_KEY(r->keys[r->len], i1.key);
INCREF_KEY(r->keys[r->len]);
if (merge)
{
#ifdef MERGE
r->values[r->len] = MERGE(i1.value, w1, i2.value, w2);
#else
COPY_VALUE(r->values[r->len], i1.value);
INCREF_VALUE(r->values[r->len]);
#endif
}
r->len++;
}
if (i1.next(&i1) < 0) goto err;
if (i2.next(&i2) < 0) goto err;
}
else
{
if(c2)
{
if(r->len >= r->size && Bucket_grow(r, -1, ! merge) < 0) goto err;
COPY_KEY(r->keys[r->len], i2.key);
INCREF_KEY(r->keys[r->len]);
if (merge)
{
COPY_VALUE(r->values[r->len], MERGE_WEIGHT(i2.value, w2));
INCREF_VALUE(r->values[r->len]);
}
r->len++;
}
if (i2.next(&i2) < 0) goto err;
}
}
if(c1 && copyRemaining(r, &i1, merge, w1) < 0) goto err;
if(c2 && copyRemaining(r, &i2, merge, w2) < 0) goto err;
finiSetIteration(&i1);
finiSetIteration(&i2);
return OBJECT(r);
#ifndef MERGE_DEFAULT
invalid_set_operation:
PyErr_SetString(PyExc_TypeError, "invalid set operation");
#endif
err:
finiSetIteration(&i1);
finiSetIteration(&i2);
Py_XDECREF(r);
return NULL;
}
static PyObject *
difference_m(PyObject *ignored, PyObject *args)
{
PyObject *o1, *o2;
UNLESS(PyArg_ParseTuple(args, "OO", &o1, &o2)) return NULL;
if (o1 == Py_None || o2 == Py_None)
{
/* difference(None, X) -> None; difference(X, None) -> X */
Py_INCREF(o1);
return o1;
}
return set_operation(o1, o2, 1, 0, /* preserve values from o1, ignore o2's */
1, 0, /* o1's values multiplied by 1 */
1, 0, 0); /* take only keys unique to o1 */
}
static PyObject *
union_m(PyObject *ignored, PyObject *args)
{
PyObject *o1, *o2;
UNLESS(PyArg_ParseTuple(args, "OO", &o1, &o2)) return NULL;
if (o1 == Py_None)
{
Py_INCREF(o2);
return o2;
}
else if (o2 == Py_None)
{
Py_INCREF(o1);
return o1;
}
return set_operation(o1, o2, 0, 0, /* ignore values in both */
1, 1, /* the weights are irrelevant */
1, 1, 1); /* take all keys */
}
static PyObject *
intersection_m(PyObject *ignored, PyObject *args)
{
PyObject *o1, *o2;
UNLESS(PyArg_ParseTuple(args, "OO", &o1, &o2)) return NULL;
if (o1 == Py_None)
{
Py_INCREF(o2);
return o2;
}
else if (o2 == Py_None)
{
Py_INCREF(o1);
return o1;
}
return set_operation(o1, o2, 0, 0, /* ignore values in both */
1, 1, /* the weights are irrelevant */
0, 1, 0); /* take only keys common to both */
}
#ifdef MERGE
static PyObject *
wunion_m(PyObject *ignored, PyObject *args)
{
PyObject *o1, *o2;
VALUE_TYPE w1 = 1, w2 = 1;
UNLESS(PyArg_ParseTuple(args, "OO|" VALUE_PARSE VALUE_PARSE,
&o1, &o2, &w1, &w2)
) return NULL;
if (o1 == Py_None)
return Py_BuildValue(VALUE_PARSE "O", (o2 == Py_None ? 0 : w2), o2);
else if (o2 == Py_None)
return Py_BuildValue(VALUE_PARSE "O", w1, o1);
o1 = set_operation(o1, o2, 1, 1, w1, w2, 1, 1, 1);
if (o1)
ASSIGN(o1, Py_BuildValue(VALUE_PARSE "O", (VALUE_TYPE)1, o1));
return o1;
}
static PyObject *
wintersection_m(PyObject *ignored, PyObject *args)
{
PyObject *o1, *o2;
VALUE_TYPE w1 = 1, w2 = 1;
UNLESS(PyArg_ParseTuple(args, "OO|" VALUE_PARSE VALUE_PARSE,
&o1, &o2, &w1, &w2)
) return NULL;
if (o1 == Py_None)
return Py_BuildValue(VALUE_PARSE "O", (o2 == Py_None ? 0 : w2), o2);
else if (o2 == Py_None)
return Py_BuildValue(VALUE_PARSE "O", w1, o1);
o1 = set_operation(o1, o2, 1, 1, w1, w2, 0, 1, 0);
if (o1)
ASSIGN(o1, Py_BuildValue(VALUE_PARSE "O",
((o1->ob_type == (PyTypeObject*)(&SetType)) ? w2+w1 : 1),
o1));
return o1;
}
#endif
#ifdef MULTI_INT_UNION
#include "sorters.c"
/* Input is a sequence of integer sets (or convertible to sets by the
set iteration protocol). Output is the union of the sets. The point
is to run much faster than doing pairs of unions.
*/
static PyObject *
multiunion_m(PyObject *ignored, PyObject *args)
{
PyObject *seq; /* input sequence */
int n; /* length of input sequence */
PyObject *set = NULL; /* an element of the input sequence */
Bucket *result; /* result set */
SetIteration setiter = {0};
int i;
UNLESS(PyArg_ParseTuple(args, "O", &seq))
return NULL;
n = PyObject_Length(seq);
if (n < 0)
return NULL;
/* Construct an empty result set. */
result = BUCKET(PyObject_CallObject(OBJECT(&SetType), NULL));
if (result == NULL)
return NULL;
/* For each set in the input sequence, append its elements to the result
set. At this point, we ignore the possibility of duplicates. */
for (i = 0; i < n; ++i) {
set = PySequence_GetItem(seq, i);
if (set == NULL)
goto Error;
/* If set is a bucket, do a straight resize + memcpy. */
if (set->ob_type == (PyTypeObject*)&SetType ||
set->ob_type == (PyTypeObject*)&BucketType)
{
Bucket *b = BUCKET(set);
int status = 0;
UNLESS (PER_USE(b)) goto Error;
if (b->len)
status = bucket_append(result, b, 0, b->len, 0, i < n-1);
PER_UNUSE(b);
if (status < 0) goto Error;
}
else {
/* No cheap way: iterate over set's elements one at a time. */
if (initSetIteration(&setiter, set, 0) < 0) goto Error;
if (setiter.next(&setiter) < 0) goto Error;
while (setiter.position >= 0) {
if (result->len >= result->size && Bucket_grow(result, -1, 1) < 0)
goto Error;
COPY_KEY(result->keys[result->len], setiter.key);
++result->len;
/* We know the key is an int, so no need to incref it. */
if (setiter.next(&setiter) < 0) goto Error;
}
finiSetIteration(&setiter);
}
Py_DECREF(set);
set = NULL;
}
/* Combine, sort, remove duplicates, and reset the result's len.
If the set shrinks (which happens if and only if there are
duplicates), no point to realloc'ing the set smaller, as we
expect the result set to be short-lived.
*/
if (result->len > 0) {
size_t newlen; /* number of elements in final result set */
newlen = sort_int4_nodups(result->keys, (size_t)result->len);
result->len = (int)newlen;
}
return (PyObject *)result;
Error:
Py_DECREF(result);
Py_XDECREF(set);
finiSetIteration(&setiter);
return NULL;
}
#endif
ZODB/src/BTrees/MergeTemplate.c 0000640 0001750 0000144 00000026237 10220643624 016045 0 ustar ctheune users /*****************************************************************************
Copyright (c) 2001, 2002 Zope Corporation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
#define MERGETEMPLATE_C "$Id: MergeTemplate.c 29676 2005-03-24 23:04:52Z tim_one $\n"
/****************************************************************************
Set operations
****************************************************************************/
static int
merge_output(Bucket *r, SetIteration *i, int mapping)
{
if (r->len >= r->size && Bucket_grow(r, -1, !mapping) < 0)
return -1;
COPY_KEY(r->keys[r->len], i->key);
INCREF_KEY(r->keys[r->len]);
if (mapping) {
COPY_VALUE(r->values[r->len], i->value);
INCREF_VALUE(r->values[r->len]);
}
r->len++;
return 0;
}
/* The "reason" argument is a little integer giving "a reason" for the
* error. In the Zope3 codebase, these are mapped to explanatory strings
* via zodb/btrees/interfaces.py.
*/
static PyObject *
merge_error(int p1, int p2, int p3, int reason)
{
PyObject *r;
UNLESS (r=Py_BuildValue("iiii", p1, p2, p3, reason)) r=Py_None;
if (ConflictError == NULL) {
ConflictError = PyExc_ValueError;
Py_INCREF(ConflictError);
}
PyErr_SetObject(ConflictError, r);
if (r != Py_None)
{
Py_DECREF(r);
}
return NULL;
}
/* It's hard to explain "the rules" for bucket_merge, in large part because
* any automatic conflict-resolution scheme is going to be incorrect for
* some endcases of *some* app. The scheme here is pretty conservative,
* and should be OK for most apps. It's easier to explain what the code
* allows than what it forbids:
*
* Leaving things alone: it's OK if both s2 and s3 leave a piece of s1
* alone (don't delete the key, and don't change the value).
*
* Key deletion: a transaction (s2 or s3) can delete a key (from s1), but
* only if the other transaction (of s2 and s3) doesn't delete the same key.
* However, it's not OK for s2 and s3 to, between them, end up deleting all
* the keys. This is a higher-level constraint, due to that the caller of
* bucket_merge() doesn't have enough info to unlink the resulting empty
* bucket from its BTree correctly. It's also not OK if s2 or s3 are empty,
* because the transaction that emptied the bucket unlinked the bucket from
* the tree, and nothing we do here can get it linked back in again.
*
* Key insertion: s2 or s3 can add a new key, provided the other transaction
* doesn't insert the same key. It's not OK even if they insert the same
* pair.
*
* Mapping value modification: s2 or s3 can modify the value associated
* with a key in s1, provided the other transaction doesn't make a
* modification of the same key to a different value. It's OK if s2 and s3
* both give the same new value to the key while it's hard to be precise about
* why, this doesn't seem consistent with that it's *not* OK for both to add
* a new key mapping to the same value).
*/
static PyObject *
bucket_merge(Bucket *s1, Bucket *s2, Bucket *s3)
{
Bucket *r=0;
PyObject *s;
SetIteration i1 = {0,0,0}, i2 = {0,0,0}, i3 = {0,0,0};
int cmp12, cmp13, cmp23, mapping, set;
/* If either "after" bucket is empty, punt. */
if (s2->len == 0 || s3->len == 0)
{
merge_error(-1, -1, -1, 12);
goto err;
}
if (initSetIteration(&i1, OBJECT(s1), 1) < 0)
goto err;
if (initSetIteration(&i2, OBJECT(s2), 1) < 0)
goto err;
if (initSetIteration(&i3, OBJECT(s3), 1) < 0)
goto err;
mapping = i1.usesValue | i2.usesValue | i3.usesValue;
set = !mapping;
if (mapping)
r = (Bucket *)PyObject_CallObject((PyObject *)&BucketType, NULL);
else
r = (Bucket *)PyObject_CallObject((PyObject *)&SetType, NULL);
if (r == NULL)
goto err;
if (i1.next(&i1) < 0)
goto err;
if (i2.next(&i2) < 0)
goto err;
if (i3.next(&i3) < 0)
goto err;
/* Consult zodb/btrees/interfaces.py for the meaning of the last
* argument passed to merge_error().
*/
/* TODO: This isn't passing on errors raised by value comparisons. */
while (i1.position >= 0 && i2.position >= 0 && i3.position >= 0)
{
TEST_KEY_SET_OR(cmp12, i1.key, i2.key) goto err;
TEST_KEY_SET_OR(cmp13, i1.key, i3.key) goto err;
if (cmp12==0)
{
if (cmp13==0)
{
if (set || (TEST_VALUE(i1.value, i2.value) == 0))
{ /* change in i3 value or all same */
if (merge_output(r, &i3, mapping) < 0) goto err;
}
else if (set || (TEST_VALUE(i1.value, i3.value) == 0))
{ /* change in i2 value */
if (merge_output(r, &i2, mapping) < 0) goto err;
}
else
{ /* conflicting value changes in i2 and i3 */
merge_error(i1.position, i2.position, i3.position, 1);
goto err;
}
if (i1.next(&i1) < 0) goto err;
if (i2.next(&i2) < 0) goto err;
if (i3.next(&i3) < 0) goto err;
}
else if (cmp13 > 0)
{ /* insert i3 */
if (merge_output(r, &i3, mapping) < 0) goto err;
if (i3.next(&i3) < 0) goto err;
}
else if (set || (TEST_VALUE(i1.value, i2.value) == 0))
{ /* deleted in i3 */
if (i1.next(&i1) < 0) goto err;
if (i2.next(&i2) < 0) goto err;
}
else
{ /* conflicting del in i3 and change in i2 */
merge_error(i1.position, i2.position, i3.position, 2);
goto err;
}
}
else if (cmp13 == 0)
{
if (cmp12 > 0)
{ /* insert i2 */
if (merge_output(r, &i2, mapping) < 0) goto err;
if (i2.next(&i2) < 0) goto err;
}
else if (set || (TEST_VALUE(i1.value, i3.value) == 0))
{ /* deleted in i2 */
if (i1.next(&i1) < 0) goto err;
if (i3.next(&i3) < 0) goto err;
}
else
{ /* conflicting del in i2 and change in i3 */
merge_error(i1.position, i2.position, i3.position, 3);
goto err;
}
}
else
{ /* Both keys changed */
TEST_KEY_SET_OR(cmp23, i2.key, i3.key) goto err;
if (cmp23==0)
{ /* dueling inserts or deletes */
merge_error(i1.position, i2.position, i3.position, 4);
goto err;
}
if (cmp12 > 0)
{ /* insert i2 */
if (cmp23 > 0)
{ /* insert i3 first */
if (merge_output(r, &i3, mapping) < 0) goto err;
if (i3.next(&i3) < 0) goto err;
}
else
{ /* insert i2 first */
if (merge_output(r, &i2, mapping) < 0) goto err;
if (i2.next(&i2) < 0) goto err;
}
}
else if (cmp13 > 0)
{ /* Insert i3 */
if (merge_output(r, &i3, mapping) < 0) goto err;
if (i3.next(&i3) < 0) goto err;
}
else
{ /* 1<2 and 1<3: both deleted 1.key */
merge_error(i1.position, i2.position, i3.position, 5);
goto err;
}
}
}
while (i2.position >= 0 && i3.position >= 0)
{ /* New inserts */
TEST_KEY_SET_OR(cmp23, i2.key, i3.key) goto err;
if (cmp23==0)
{ /* dueling inserts */
merge_error(i1.position, i2.position, i3.position, 6);
goto err;
}
if (cmp23 > 0)
{ /* insert i3 */
if (merge_output(r, &i3, mapping) < 0) goto err;
if (i3.next(&i3) < 0) goto err;
}
else
{ /* insert i2 */
if (merge_output(r, &i2, mapping) < 0) goto err;
if (i2.next(&i2) < 0) goto err;
}
}
while (i1.position >= 0 && i2.position >= 0)
{ /* remainder of i1 deleted in i3 */
TEST_KEY_SET_OR(cmp12, i1.key, i2.key) goto err;
if (cmp12 > 0)
{ /* insert i2 */
if (merge_output(r, &i2, mapping) < 0) goto err;
if (i2.next(&i2) < 0) goto err;
}
else if (cmp12==0 && (set || (TEST_VALUE(i1.value, i2.value) == 0)))
{ /* delete i3 */
if (i1.next(&i1) < 0) goto err;
if (i2.next(&i2) < 0) goto err;
}
else
{ /* Dueling deletes or delete and change */
merge_error(i1.position, i2.position, i3.position, 7);
goto err;
}
}
while (i1.position >= 0 && i3.position >= 0)
{ /* remainder of i1 deleted in i2 */
TEST_KEY_SET_OR(cmp13, i1.key, i3.key) goto err;
if (cmp13 > 0)
{ /* insert i3 */
if (merge_output(r, &i3, mapping) < 0) goto err;
if (i3.next(&i3) < 0) goto err;
}
else if (cmp13==0 && (set || (TEST_VALUE(i1.value, i3.value) == 0)))
{ /* delete i2 */
if (i1.next(&i1) < 0) goto err;
if (i3.next(&i3) < 0) goto err;
}
else
{ /* Dueling deletes or delete and change */
merge_error(i1.position, i2.position, i3.position, 8);
goto err;
}
}
if (i1.position >= 0)
{ /* Dueling deletes */
merge_error(i1.position, i2.position, i3.position, 9);
goto err;
}
while (i2.position >= 0)
{ /* Inserting i2 at end */
if (merge_output(r, &i2, mapping) < 0) goto err;
if (i2.next(&i2) < 0) goto err;
}
while (i3.position >= 0)
{ /* Inserting i3 at end */
if (merge_output(r, &i3, mapping) < 0) goto err;
if (i3.next(&i3) < 0) goto err;
}
/* If the output bucket is empty, conflict resolution doesn't have
* enough info to unlink it from its containing BTree correctly.
*/
if (r->len == 0)
{
merge_error(-1, -1, -1, 10);
goto err;
}
finiSetIteration(&i1);
finiSetIteration(&i2);
finiSetIteration(&i3);
if (s1->next)
{
Py_INCREF(s1->next);
r->next = s1->next;
}
s = bucket_getstate(r);
Py_DECREF(r);
return s;
err:
finiSetIteration(&i1);
finiSetIteration(&i2);
finiSetIteration(&i3);
Py_XDECREF(r);
return NULL;
}
ZODB/src/BTrees/BTreeTemplate.c 0000640 0001750 0000144 00000171575 10306151001 016001 0 ustar ctheune users /*****************************************************************************
Copyright (c) 2001, 2002 Zope Corporation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
#define BTREETEMPLATE_C "$Id: BTreeTemplate.c 38270 2005-09-02 22:28:17Z tim_one $\n"
/* Sanity-check a BTree. This is a private helper for BTree_check. Return:
* -1 Error. If it's an internal inconsistency in the BTree,
* AssertionError is set.
* 0 No problem found.
*
* nextbucket is the bucket "one beyond the end" of the BTree; the last bucket
* directly reachable from following right child pointers *should* be linked
* to nextbucket (and this is checked).
*/
static int
BTree_check_inner(BTree *self, Bucket *nextbucket)
{
int i;
Bucket *bucketafter;
Sized *child;
char *errormsg = "internal error"; /* someone should have overriden */
Sized *activated_child = NULL;
int result = -1; /* until proved innocent */
#define CHECK(CONDITION, ERRORMSG) \
if (!(CONDITION)) { \
errormsg = (ERRORMSG); \
goto Error; \
}
PER_USE_OR_RETURN(self, -1);
CHECK(self->len >= 0, "BTree len < 0");
CHECK(self->len <= self->size, "BTree len > size");
if (self->len == 0) {
/* Empty BTree. */
CHECK(self->firstbucket == NULL,
"Empty BTree has non-NULL firstbucket");
result = 0;
goto Done;
}
/* Non-empty BTree. */
CHECK(self->firstbucket != NULL, "Non-empty BTree has NULL firstbucket");
/* Obscure: The first bucket is pointed to at least by self->firstbucket
* and data[0].child of whichever BTree node it's a child of. However,
* if persistence is enabled then the latter BTree node may be a ghost
* at this point, and so its pointers "don't count": we can only rely
* on self's pointers being intact.
*/
#ifdef PERSISTENT
CHECK(self->firstbucket->ob_refcnt >= 1,
"Non-empty BTree firstbucket has refcount < 1");
#else
CHECK(self->firstbucket->ob_refcnt >= 2,
"Non-empty BTree firstbucket has refcount < 2");
#endif
for (i = 0; i < self->len; ++i) {
CHECK(self->data[i].child != NULL, "BTree has NULL child");
}
if (SameType_Check(self, self->data[0].child)) {
/* Our children are also BTrees. */
child = self->data[0].child;
UNLESS (PER_USE(child)) goto Done;
activated_child = child;
CHECK(self->firstbucket == BTREE(child)->firstbucket,
"BTree has firstbucket different than "
"its first child's firstbucket");
PER_ALLOW_DEACTIVATION(child);
activated_child = NULL;
for (i = 0; i < self->len; ++i) {
child = self->data[i].child;
CHECK(SameType_Check(self, child),
"BTree children have different types");
if (i == self->len - 1)
bucketafter = nextbucket;
else {
BTree *child2 = BTREE(self->data[i+1].child);
UNLESS (PER_USE(child2)) goto Done;
bucketafter = child2->firstbucket;
PER_ALLOW_DEACTIVATION(child2);
}
if (BTree_check_inner(BTREE(child), bucketafter) < 0) goto Done;
}
}
else {
/* Our children are buckets. */
CHECK(self->firstbucket == BUCKET(self->data[0].child),
"Bottom-level BTree node has inconsistent firstbucket belief");
for (i = 0; i < self->len; ++i) {
child = self->data[i].child;
UNLESS (PER_USE(child)) goto Done;
activated_child = child;
CHECK(!SameType_Check(self, child),
"BTree children have different types");
CHECK(child->len >= 1, "Bucket length < 1"); /* no empty buckets! */
CHECK(child->len <= child->size, "Bucket len > size");
#ifdef PERSISTENT
CHECK(child->ob_refcnt >= 1, "Bucket has refcount < 1");
#else
CHECK(child->ob_refcnt >= 2, "Bucket has refcount < 2");
#endif
if (i == self->len - 1)
bucketafter = nextbucket;
else
bucketafter = BUCKET(self->data[i+1].child);
CHECK(BUCKET(child)->next == bucketafter,
"Bucket next pointer is damaged");
PER_ALLOW_DEACTIVATION(child);
activated_child = NULL;
}
}
result = 0;
goto Done;
Error:
PyErr_SetString(PyExc_AssertionError, errormsg);
result = -1;
Done:
/* No point updating access time -- this isn't a "real" use. */
PER_ALLOW_DEACTIVATION(self);
if (activated_child) {
PER_ALLOW_DEACTIVATION(activated_child);
}
return result;
#undef CHECK
}
/* Sanity-check a BTree. This is the ._check() method. Return:
* NULL Error. If it's an internal inconsistency in the BTree,
* AssertionError is set.
* Py_None No problem found.
*/
static PyObject*
BTree_check(BTree *self)
{
PyObject *result = NULL;
int i = BTree_check_inner(self, NULL);
if (i >= 0) {
result = Py_None;
Py_INCREF(result);
}
return result;
}
/*
** _BTree_get
**
** Search a BTree.
**
** Arguments
** self a pointer to a BTree
** keyarg the key to search for, as a Python object
** has_key true/false; when false, try to return the associated
** value; when true, return a boolean
** Return
** When has_key false:
** If key exists, its associated value.
** If key doesn't exist, NULL and KeyError is set.
** When has_key true:
** A Python int is returned in any case.
** If key exists, the depth of the bucket in which it was found.
** If key doesn't exist, 0.
*/
static PyObject *
_BTree_get(BTree *self, PyObject *keyarg, int has_key)
{
KEY_TYPE key;
PyObject *result = NULL; /* guilty until proved innocent */
int copied = 1;
COPY_KEY_FROM_ARG(key, keyarg, copied);
UNLESS (copied) return NULL;
PER_USE_OR_RETURN(self, NULL);
if (self->len == 0) {
/* empty BTree */
if (has_key)
result = PyInt_FromLong(0);
else
PyErr_SetObject(PyExc_KeyError, keyarg);
}
else {
for (;;) {
int i;
Sized *child;
BTREE_SEARCH(i, self, key, goto Done);
child = self->data[i].child;
has_key += has_key != 0; /* bump depth counter, maybe */
if (SameType_Check(self, child)) {
PER_UNUSE(self);
self = BTREE(child);
PER_USE_OR_RETURN(self, NULL);
}
else {
result = _bucket_get(BUCKET(child), keyarg, has_key);
break;
}
}
}
Done:
PER_UNUSE(self);
return result;
}
static PyObject *
BTree_get(BTree *self, PyObject *key)
{
return _BTree_get(self, key, 0);
}
/* Create a new bucket for the BTree or TreeSet using the class attribute
_bucket_type, which is normally initialized to BucketType or SetType
as appropriate.
*/
static Sized *
BTree_newBucket(BTree *self)
{
PyObject *factory;
Sized *result;
/* _bucket_type_str defined in BTreeModuleTemplate.c */
factory = PyObject_GetAttr((PyObject *)self->ob_type, _bucket_type_str);
if (factory == NULL)
return NULL;
/* TODO: Should we check that the factory actually returns something
of the appropriate type? How? The C code here is going to
depend on any custom bucket type having the same layout at the
C level.
*/
result = SIZED(PyObject_CallObject(factory, NULL));
Py_DECREF(factory);
return result;
}
/*
* Move data from the current BTree, from index onward, to the newly created
* BTree 'next'. self and next must both be activated. If index is OOB (< 0
* or >= self->len), use self->len / 2 as the index (i.e., split at the
* midpoint). self must have at least 2 children on entry, and index must
* be such that self and next each have at least one child at exit. self's
* accessed time is updated.
*
* Return:
* -1 error
* 0 OK
*/
static int
BTree_split(BTree *self, int index, BTree *next)
{
int next_size;
Sized *child;
if (index < 0 || index >= self->len)
index = self->len / 2;
next_size = self->len - index;
ASSERT(index > 0, "split creates empty tree", -1);
ASSERT(next_size > 0, "split creates empty tree", -1);
next->data = BTree_Malloc(sizeof(BTreeItem) * next_size);
if (!next->data)
return -1;
memcpy(next->data, self->data + index, sizeof(BTreeItem) * next_size);
next->size = next_size; /* but don't set len until we succeed */
/* Set next's firstbucket. self->firstbucket is still correct. */
child = next->data[0].child;
if (SameType_Check(self, child)) {
PER_USE_OR_RETURN(child, -1);
next->firstbucket = BTREE(child)->firstbucket;
PER_UNUSE(child);
}
else
next->firstbucket = BUCKET(child);
Py_INCREF(next->firstbucket);
next->len = next_size;
self->len = index;
return PER_CHANGED(self) >= 0 ? 0 : -1;
}
/* Fwd decl -- BTree_grow and BTree_split_root reference each other. */
static int BTree_grow(BTree *self, int index, int noval);
/* Split the root. This is a little special because the root isn't a child
* of anything else, and the root needs to retain its object identity. So
* this routine moves the root's data into a new child, and splits the
* latter. This leaves the root with two children.
*
* Return:
* 0 OK
* -1 error
*
* CAUTION: The caller must call PER_CHANGED on self.
*/
static int
BTree_split_root(BTree *self, int noval)
{
BTree *child;
BTreeItem *d;
/* Create a child BTree, and a new data vector for self. */
child = BTREE(PyObject_CallObject(OBJECT(self->ob_type), NULL));
if (!child) return -1;
d = BTree_Malloc(sizeof(BTreeItem) * 2);
if (!d) {
Py_DECREF(child);
return -1;
}
/* Move our data to new BTree. */
child->size = self->size;
child->len = self->len;
child->data = self->data;
child->firstbucket = self->firstbucket;
Py_INCREF(child->firstbucket);
/* Point self to child and split the child. */
self->data = d;
self->len = 1;
self->size = 2;
self->data[0].child = SIZED(child); /* transfers reference ownership */
return BTree_grow(self, 0, noval);
}
/*
** BTree_grow
**
** Grow a BTree
**
** Arguments: self The BTree
** index self->data[index].child needs to be split. index
** must be 0 if self is empty (len == 0), and a new
** empty bucket is created then.
** noval Boolean; is this a set (true) or mapping (false)?
**
** Returns: 0 on success
** -1 on failure
**
** CAUTION: If self is empty on entry, this routine adds an empty bucket.
** That isn't a legitimate BTree; if the caller doesn't put something in
** in the bucket (say, because of a later error), the BTree must be cleared
** to get rid of the empty bucket.
*/
static int
BTree_grow(BTree *self, int index, int noval)
{
int i;
Sized *v, *e = 0;
BTreeItem *d;
if (self->len == self->size) {
if (self->size) {
d = BTree_Realloc(self->data, sizeof(BTreeItem) * self->size * 2);
if (d == NULL)
return -1;
self->data = d;
self->size *= 2;
}
else {
d = BTree_Malloc(sizeof(BTreeItem) * 2);
if (d == NULL)
return -1;
self->data = d;
self->size = 2;
}
}
if (self->len) {
d = self->data + index;
v = d->child;
/* Create a new object of the same type as the target value */
e = (Sized *)PyObject_CallObject((PyObject *)v->ob_type, NULL);
if (e == NULL)
return -1;
UNLESS(PER_USE(v)) {
Py_DECREF(e);
return -1;
}
/* Now split between the original (v) and the new (e) at the midpoint*/
if (SameType_Check(self, v))
i = BTree_split((BTree *)v, -1, (BTree *)e);
else
i = bucket_split((Bucket *)v, -1, (Bucket *)e);
PER_ALLOW_DEACTIVATION(v);
if (i < 0) {
Py_DECREF(e);
assert(PyErr_Occurred());
return -1;
}
index++;
d++;
if (self->len > index) /* Shift up the old values one array slot */
memmove(d+1, d, sizeof(BTreeItem)*(self->len-index));
if (SameType_Check(self, v)) {
COPY_KEY(d->key, BTREE(e)->data->key);
/* We take the unused reference from e, so there's no
reason to INCREF!
*/
/* INCREF_KEY(self->data[1].key); */
}
else {
COPY_KEY(d->key, BUCKET(e)->keys[0]);
INCREF_KEY(d->key);
}
d->child = e;
self->len++;
if (self->len >= MAX_BTREE_SIZE(self) * 2) /* the root is huge */
return BTree_split_root(self, noval);
}
else {
/* The BTree is empty. Create an empty bucket. See CAUTION in
* the comments preceding.
*/
assert(index == 0);
d = self->data;
d->child = BTree_newBucket(self);
if (d->child == NULL)
return -1;
self->len = 1;
Py_INCREF(d->child);
self->firstbucket = (Bucket *)d->child;
}
return 0;
}
/* Return the rightmost bucket reachable from following child pointers
* from self. The caller gets a new reference to this bucket. Note that
* bucket 'next' pointers are not followed: if self is an interior node
* of a BTree, this returns the rightmost bucket in that node's subtree.
* In case of error, returns NULL.
*
* self must not be a ghost; this isn't checked. The result may be a ghost.
*
* Pragmatics: Note that the rightmost bucket's last key is the largest
* key in self's subtree.
*/
static Bucket *
BTree_lastBucket(BTree *self)
{
Sized *pchild;
Bucket *result;
UNLESS (self->data && self->len) {
IndexError(-1); /* is this the best action to take? */
return NULL;
}
pchild = self->data[self->len - 1].child;
if (SameType_Check(self, pchild)) {
self = BTREE(pchild);
PER_USE_OR_RETURN(self, NULL);
result = BTree_lastBucket(self);
PER_UNUSE(self);
}
else {
Py_INCREF(pchild);
result = BUCKET(pchild);
}
return result;
}
static int
BTree_deleteNextBucket(BTree *self)
{
Bucket *b;
UNLESS (PER_USE(self)) return -1;
b = BTree_lastBucket(self);
if (b == NULL)
goto err;
if (Bucket_deleteNextBucket(b) < 0)
goto err;
Py_DECREF(b);
PER_UNUSE(self);
return 0;
err:
Py_XDECREF(b);
PER_ALLOW_DEACTIVATION(self);
return -1;
}
/*
** _BTree_clear
**
** Clears out all of the values in the BTree (firstbucket, keys, and children);
** leaving self an empty BTree.
**
** Arguments: self The BTree
**
** Returns: 0 on success
** -1 on failure
**
** Internal: Deallocation order is important. The danger is that a long
** list of buckets may get freed "at once" via decref'ing the first bucket,
** in which case a chain of consequenct Py_DECREF calls may blow the stack.
** Luckily, every bucket has a refcount of at least two, one due to being a
** BTree node's child, and another either because it's not the first bucket in
** the chain (so the preceding bucket points to it), or because firstbucket
** points to it. By clearing in the natural depth-first, left-to-right
** order, the BTree->bucket child pointers prevent Py_DECREF(bucket->next)
** calls from freeing bucket->next, and the maximum stack depth is equal
** to the height of the tree.
**/
static int
_BTree_clear(BTree *self)
{
const int len = self->len;
if (self->firstbucket) {
/* Obscure: The first bucket is pointed to at least by
* self->firstbucket and data[0].child of whichever BTree node it's
* a child of. However, if persistence is enabled then the latter
* BTree node may be a ghost at this point, and so its pointers "don't
* count": we can only rely on self's pointers being intact.
*/
#ifdef PERSISTENT
ASSERT(self->firstbucket->ob_refcnt > 0,
"Invalid firstbucket pointer", -1);
#else
ASSERT(self->firstbucket->ob_refcnt > 1,
"Invalid firstbucket pointer", -1);
#endif
Py_DECREF(self->firstbucket);
self->firstbucket = NULL;
}
if (self->data) {
int i;
if (len > 0) { /* 0 is special because key 0 is trash */
Py_DECREF(self->data[0].child);
}
for (i = 1; i < len; i++) {
#ifdef KEY_TYPE_IS_PYOBJECT
DECREF_KEY(self->data[i].key);
#endif
Py_DECREF(self->data[i].child);
}
free(self->data);
self->data = NULL;
}
self->len = self->size = 0;
return 0;
}
/*
Set (value != 0) or delete (value=0) a tree item.
If unique is non-zero, then only change if the key is
new.
If noval is non-zero, then don't set a value (the tree
is a set).
Return:
-1 error
0 successful, and number of entries didn't change
>0 successful, and number of entries did change
Internal
There are two distinct return values > 0:
1 Successful, number of entries changed, but firstbucket did not go away.
2 Successful, number of entries changed, firstbucket did go away.
This can only happen on a delete (value == NULL). The caller may
need to change its own firstbucket pointer, and in any case *someone*
needs to adjust the 'next' pointer of the bucket immediately preceding
the bucket that went away (it needs to point to the bucket immediately
following the bucket that went away).
*/
static int
_BTree_set(BTree *self, PyObject *keyarg, PyObject *value,
int unique, int noval)
{
int changed = 0; /* did I mutate? */
int min; /* index of child I searched */
BTreeItem *d; /* self->data[min] */
int childlength; /* len(self->data[min].child) */
int status; /* our return value; and return value from callee */
int self_was_empty; /* was self empty at entry? */
KEY_TYPE key;
int copied = 1;
COPY_KEY_FROM_ARG(key, keyarg, copied);
if (!copied) return -1;
PER_USE_OR_RETURN(self, -1);
self_was_empty = self->len == 0;
if (self_was_empty) {
/* We're empty. Make room. */
if (value) {
if (BTree_grow(self, 0, noval) < 0)
goto Error;
}
else {
/* Can't delete a key from an empty BTree. */
PyErr_SetObject(PyExc_KeyError, keyarg);
goto Error;
}
}
/* Find the right child to search, and hand the work off to it. */
BTREE_SEARCH(min, self, key, goto Error);
d = self->data + min;
if (SameType_Check(self, d->child))
status = _BTree_set(BTREE(d->child), keyarg, value, unique, noval);
else {
int bucket_changed = 0;
status = _bucket_set(BUCKET(d->child), keyarg,
value, unique, noval, &bucket_changed);
#ifdef PERSISTENT
/* If a BTree contains only a single bucket, BTree.__getstate__()
* includes the bucket's entire state, and the bucket doesn't get
* an oid of its own. So if we have a single oid-less bucket that
* changed, it's *our* oid that should be marked as changed -- the
* bucket doesn't have one.
*/
if (bucket_changed
&& self->len == 1
&& self->data[0].child->oid == NULL)
{
changed = 1;
}
#endif
}
if (status == 0) goto Done;
if (status < 0) goto Error;
assert(status == 1 || status == 2);
/* The child changed size. Get its new size. Note that since the tree
* rooted at the child changed size, so did the tree rooted at self:
* our status must be >= 1 too.
*/
UNLESS(PER_USE(d->child)) goto Error;
childlength = d->child->len;
PER_UNUSE(d->child);
if (value) {
/* A bucket got bigger -- if it's "too big", split it. */
int toobig;
assert(status == 1); /* can be 2 only on deletes */
if (SameType_Check(self, d->child))
toobig = childlength > MAX_BTREE_SIZE(d->child);
else
toobig = childlength > MAX_BUCKET_SIZE(d->child);
if (toobig) {
if (BTree_grow(self, min, noval) < 0) goto Error;
changed = 1; /* BTree_grow mutated self */
}
goto Done; /* and status still == 1 */
}
/* A bucket got smaller. This is much harder, and despite that we
* don't try to rebalance the tree.
*/
if (status == 2) { /* this is the last reference to child status */
/* Two problems to solve: May have to adjust our own firstbucket,
* and the bucket that went away needs to get unlinked.
*/
if (min) {
/* This wasn't our firstbucket, so no need to adjust ours (note
* that it can't be the firstbucket of any node above us either).
* Tell "the tree to the left" to do the unlinking.
*/
if (BTree_deleteNextBucket(BTREE(d[-1].child)) < 0) goto Error;
status = 1; /* we solved the child's firstbucket problem */
}
else {
/* This was our firstbucket. Update to new firstbucket value. */
Bucket *nextbucket;
UNLESS(PER_USE(d->child)) goto Error;
nextbucket = BTREE(d->child)->firstbucket;
PER_UNUSE(d->child);
Py_XINCREF(nextbucket);
Py_DECREF(self->firstbucket);
self->firstbucket = nextbucket;
changed = 1;
/* The caller has to do the unlinking -- we can't. Also, since
* it was our firstbucket, it may also be theirs.
*/
assert(status == 2);
}
}
/* If the child isn't empty, we're done! We did all that was possible for
* us to do with the firstbucket problems the child gave us, and since the
* child isn't empty don't create any new firstbucket problems of our own.
*/
if (childlength) goto Done;
/* The child became empty: we need to remove it from self->data.
* But first, if we're a bottom-level node, we've got more bucket-fiddling
* to set up.
*/
if (!SameType_Check(self, d->child)) {
/* We're about to delete a bucket. */
if (min) {
/* It's not our first bucket, so we can tell the previous
* bucket to adjust its reference to it. It can't be anyone
* else's first bucket either, so the caller needn't do anything.
*/
if (Bucket_deleteNextBucket(BUCKET(d[-1].child)) < 0) goto Error;
/* status should be 1, and already is: if it were 2, the
* block above would have set it to 1 in its min != 0 branch.
*/
assert(status == 1);
}
else {
Bucket *nextbucket;
/* It's our first bucket. We can't unlink it directly. */
/* 'changed' will be set true by the deletion code following. */
UNLESS(PER_USE(d->child)) goto Error;
nextbucket = BUCKET(d->child)->next;
PER_UNUSE(d->child);
Py_XINCREF(nextbucket);
Py_DECREF(self->firstbucket);
self->firstbucket = nextbucket;
status = 2; /* we're giving our caller a new firstbucket problem */
}
}
/* Remove the child from self->data. */
Py_DECREF(d->child);
#ifdef KEY_TYPE_IS_PYOBJECT
if (min) {
DECREF_KEY(d->key);
}
else if (self->len > 1) {
/* We're deleting the first child of a BTree with more than one
* child. The key at d+1 is about to be shifted into slot 0,
* and hence never to be referenced again (the key in slot 0 is
* trash).
*/
DECREF_KEY((d+1)->key);
}
/* Else min==0 and len==1: we're emptying the BTree entirely, and
* there is no key in need of decrefing.
*/
#endif
--self->len;
if (min < self->len)
memmove(d, d+1, (self->len - min) * sizeof(BTreeItem));
changed = 1;
Done:
#ifdef PERSISTENT
if (changed) {
if (PER_CHANGED(self) < 0) goto Error;
}
#endif
PER_UNUSE(self);
return status;
Error:
assert(PyErr_Occurred());
if (self_was_empty) {
/* BTree_grow may have left the BTree in an invalid state. Make
* sure the tree is a legitimate empty tree.
*/
_BTree_clear(self);
}
PER_UNUSE(self);
return -1;
}
/*
** BTree_setitem
**
** wrapper for _BTree_set
**
** Arguments: self The BTree
** key The key to insert
** v The value to insert
**
** Returns -1 on failure
** 0 on success
*/
static int
BTree_setitem(BTree *self, PyObject *key, PyObject *v)
{
if (_BTree_set(self, key, v, 0, 0) < 0)
return -1;
return 0;
}
#ifdef PERSISTENT
static PyObject *
BTree__p_deactivate(BTree *self, PyObject *args, PyObject *keywords)
{
int ghostify = 1;
PyObject *force = NULL;
if (args && PyTuple_GET_SIZE(args) > 0) {
PyErr_SetString(PyExc_TypeError,
"_p_deactivate takes not positional arguments");
return NULL;
}
if (keywords) {
int size = PyDict_Size(keywords);
force = PyDict_GetItemString(keywords, "force");
if (force)
size--;
if (size) {
PyErr_SetString(PyExc_TypeError,
"_p_deactivate only accepts keyword arg force");
return NULL;
}
}
if (self->jar && self->oid) {
ghostify = self->state == cPersistent_UPTODATE_STATE;
if (!ghostify && force) {
if (PyObject_IsTrue(force))
ghostify = 1;
if (PyErr_Occurred())
return NULL;
}
if (ghostify) {
if (_BTree_clear(self) < 0)
return NULL;
PER_GHOSTIFY(self);
}
}
Py_INCREF(Py_None);
return Py_None;
}
#endif
static PyObject *
BTree_clear(BTree *self)
{
UNLESS (PER_USE(self)) return NULL;
if (self->len)
{
if (_BTree_clear(self) < 0)
goto err;
if (PER_CHANGED(self) < 0)
goto err;
}
PER_UNUSE(self);
Py_INCREF(Py_None);
return Py_None;
err:
PER_UNUSE(self);
return NULL;
}
/*
* Return:
*
* For an empty BTree (self->len == 0), None.
*
* For a BTree with one child (self->len == 1), and that child is a bucket,
* and that bucket has a NULL oid, a one-tuple containing a one-tuple
* containing the bucket's state:
*
* (
* (
* child[0].__getstate__(),
* ),
* )
*
* Else a two-tuple. The first element is a tuple interleaving the BTree's
* keys and direct children, of size 2*self->len - 1 (key[0] is unused and
* is not saved). The second element is the firstbucket:
*
* (
* (child[0], key[1], child[1], key[2], child[2], ...,
* key[len-1], child[len-1]),
* self->firstbucket
* )
*
* In the above, key[i] means self->data[i].key, and similarly for child[i].
*/
static PyObject *
BTree_getstate(BTree *self)
{
PyObject *r = NULL;
PyObject *o;
int i, l;
UNLESS (PER_USE(self)) return NULL;
if (self->len) {
r = PyTuple_New(self->len * 2 - 1);
if (r == NULL)
goto err;
if (self->len == 1
&& self->data->child->ob_type != self->ob_type
#ifdef PERSISTENT
&& BUCKET(self->data->child)->oid == NULL
#endif
) {
/* We have just one bucket. Save its data directly. */
o = bucket_getstate((Bucket *)self->data->child);
if (o == NULL)
goto err;
PyTuple_SET_ITEM(r, 0, o);
ASSIGN(r, Py_BuildValue("(O)", r));
}
else {
for (i=0, l=0; i < self->len; i++) {
if (i) {
COPY_KEY_TO_OBJECT(o, self->data[i].key);
PyTuple_SET_ITEM(r, l, o);
l++;
}
o = (PyObject *)self->data[i].child;
Py_INCREF(o);
PyTuple_SET_ITEM(r,l,o);
l++;
}
ASSIGN(r, Py_BuildValue("OO", r, self->firstbucket));
}
}
else {
r = Py_None;
Py_INCREF(r);
}
PER_UNUSE(self);
return r;
err:
PER_UNUSE(self);
Py_XDECREF(r);
return NULL;
}
static int
_BTree_setstate(BTree *self, PyObject *state, int noval)
{
PyObject *items, *firstbucket = NULL;
BTreeItem *d;
int len, l, i, copied=1;
if (_BTree_clear(self) < 0)
return -1;
/* The state of a BTree can be one of the following:
None -- an empty BTree
A one-tuple -- a single bucket btree
A two-tuple -- a BTree with more than one bucket
See comments for BTree_getstate() for the details.
*/
if (state == Py_None)
return 0;
if (!PyArg_ParseTuple(state, "O|O:__setstate__", &items, &firstbucket))
return -1;
len = PyTuple_Size(items);
if (len < 0)
return -1;
len = (len + 1) / 2;
assert(len > 0); /* If the BTree is empty, it's state is None. */
assert(self->size == 0); /* We called _BTree_clear(). */
self->data = BTree_Malloc(sizeof(BTreeItem) * len);
if (self->data == NULL)
return -1;
self->size = len;
for (i = 0, d = self->data, l = 0; i < len; i++, d++) {
PyObject *v;
if (i) { /* skip the first key slot */
COPY_KEY_FROM_ARG(d->key, PyTuple_GET_ITEM(items, l), copied);
l++;
if (!copied)
return -1;
INCREF_KEY(d->key);
}
v = PyTuple_GET_ITEM(items, l);
if (PyTuple_Check(v)) {
/* Handle the special case in __getstate__() for a BTree
with a single bucket. */
d->child = BTree_newBucket(self);
if (!d->child)
return -1;
if (noval) {
if (_set_setstate(BUCKET(d->child), v) < 0)
return -1;
}
else {
if (_bucket_setstate(BUCKET(d->child), v) < 0)
return -1;
}
}
else {
d->child = (Sized *)v;
Py_INCREF(v);
}
l++;
}
if (!firstbucket)
firstbucket = (PyObject *)self->data->child;
if (!PyObject_IsInstance(firstbucket, (PyObject *)
(noval ? &SetType : &BucketType))) {
PyErr_SetString(PyExc_TypeError,
"No firstbucket in non-empty BTree");
return -1;
}
self->firstbucket = BUCKET(firstbucket);
Py_INCREF(firstbucket);
#ifndef PERSISTENT
/* firstbucket is also the child of some BTree node, but that node may
* be a ghost if persistence is enabled.
*/
assert(self->firstbucket->ob_refcnt > 1);
#endif
self->len = len;
return 0;
}
static PyObject *
BTree_setstate(BTree *self, PyObject *arg)
{
int r;
PER_PREVENT_DEACTIVATION(self);
r = _BTree_setstate(self, arg, 0);
PER_UNUSE(self);
if (r < 0)
return NULL;
Py_INCREF(Py_None);
return Py_None;
}
#ifdef PERSISTENT
/* Recognize the special cases of a BTree that's empty or contains a single
* bucket. In the former case, return a borrowed reference to Py_None.
* In this single-bucket case, the bucket state is embedded directly in the
* BTree state, like so:
*
* (
* (
* thebucket.__getstate__(),
* ),
* )
*
* When this obtains, return a borrowed reference to thebucket.__getstate__().
* Else return NULL with an exception set. The exception should always be
* ConflictError then, but may be TypeError if the state makes no sense at all
* for a BTree (corrupted or hostile state).
*/
PyObject *
get_bucket_state(PyObject *t)
{
if (t == Py_None)
return Py_None; /* an empty BTree */
if (! PyTuple_Check(t)) {
PyErr_SetString(PyExc_TypeError,
"_p_resolveConflict: expected tuple or None for state");
return NULL;
}
if (PyTuple_GET_SIZE(t) == 2) {
/* A non-degenerate BTree. */
return merge_error(-1, -1, -1, 11);
}
/* We're in the one-bucket case. */
if (PyTuple_GET_SIZE(t) != 1) {
PyErr_SetString(PyExc_TypeError,
"_p_resolveConflict: expected 1- or 2-tuple for state");
return NULL;
}
t = PyTuple_GET_ITEM(t, 0);
if (! PyTuple_Check(t) || PyTuple_GET_SIZE(t) != 1) {
PyErr_SetString(PyExc_TypeError,
"_p_resolveConflict: expected 1-tuple containing "
"bucket state");
return NULL;
}
t = PyTuple_GET_ITEM(t, 0);
if (! PyTuple_Check(t)) {
PyErr_SetString(PyExc_TypeError,
"_p_resolveConflict: expected tuple for bucket state");
return NULL;
}
return t;
}
/* Tricky. The only kind of BTree conflict we can actually potentially
* resolve is the special case of a BTree containing a single bucket,
* in which case this becomes a fancy way of calling the bucket conflict
* resolution code.
*/
static PyObject *
BTree__p_resolveConflict(BTree *self, PyObject *args)
{
PyObject *s[3];
PyObject *x, *y, *z;
if (!PyArg_ParseTuple(args, "OOO", &x, &y, &z))
return NULL;
s[0] = get_bucket_state(x);
if (s[0] == NULL)
return NULL;
s[1] = get_bucket_state(y);
if (s[1] == NULL)
return NULL;
s[2] = get_bucket_state(z);
if (s[2] == NULL)
return NULL;
if (PyObject_IsInstance((PyObject *)self, (PyObject *)&BTreeType))
x = _bucket__p_resolveConflict(OBJECT(&BucketType), s);
else
x = _bucket__p_resolveConflict(OBJECT(&SetType), s);
if (x == NULL)
return NULL;
return Py_BuildValue("((N))", x);
}
#endif
/*
BTree_findRangeEnd -- Find one end, expressed as a bucket and
position, for a range search.
If low, return bucket and index of the smallest item >= key,
otherwise return bucket and index of the largest item <= key.
If exclude_equal, exact matches aren't acceptable; if one is found,
move right if low, or left if !low (this is for range searches exclusive
of an endpoint).
Return:
-1 Error; offset and bucket unchanged
0 Not found; offset and bucket unchanged
1 Correct bucket and offset stored; the caller owns a new reference
to the bucket.
Internal:
We do binary searches in BTree nodes downward, at each step following
C(i) where K(i) <= key < K(i+1). As always, K(i) <= C(i) < K(i+1) too.
(See Maintainer.txt for the meaning of that notation.) That eventually
leads to a bucket where we do Bucket_findRangeEnd. That usually works,
but there are two cases where it can fail to find the correct answer:
1. On a low search, we find a bucket with keys >= K(i), but that doesn't
imply there are keys in the bucket >= key. For example, suppose
a bucket has keys in 1..100, its successor's keys are in 200..300, and
we're doing a low search on 150. We'll end up in the first bucket,
but there are no keys >= 150 in it. K(i+1) > key, though, and all
the keys in C(i+1) >= K(i+1) > key, so the first key in the next
bucket (if any) is the correct result. This is easy to find by
following the bucket 'next' pointer.
2. On a high search, again that the keys in the bucket are >= K(i)
doesn't imply that any key in the bucket is <= key, but it's harder
for this to fail (and an earlier version of this routine didn't
catch it): if K(i) itself is in the bucket, it works (then
K(i) <= key is *a* key in the bucket that's in the desired range).
But when keys get deleted from buckets, they aren't also deleted from
BTree nodes, so there's no guarantee that K(i) is in the bucket.
For example, delete the smallest key S from some bucket, and S
remains in the interior BTree nodes. Do a high search for S, and
the BTree nodes direct the search to the bucket S used to be in,
but all keys remaining in that bucket are > S. The largest key in
the *preceding* bucket (if any) is < K(i), though, and K(i) <= key,
so the largest key in the preceding bucket is < key and so is the
proper result.
This is harder to get at efficiently, as buckets are linked only in
the increasing direction. While we're searching downward,
deepest_smaller is set to the node deepest in the tree where
we *could* have gone to the left of C(i). The rightmost bucket in
deepest_smaller's subtree is the bucket preceding the bucket we find
at first. This is clumsy to get at, but efficient.
*/
static int
BTree_findRangeEnd(BTree *self, PyObject *keyarg, int low, int exclude_equal,
Bucket **bucket, int *offset) {
Sized *deepest_smaller = NULL; /* last possibility to move left */
int deepest_smaller_is_btree = 0; /* Boolean; if false, it's a bucket */
Bucket *pbucket;
int self_got_rebound = 0; /* Boolean; when true, deactivate self */
int result = -1; /* Until proven innocent */
int i;
KEY_TYPE key;
int copied = 1;
COPY_KEY_FROM_ARG(key, keyarg, copied);
UNLESS (copied) return -1;
/* We don't need to: PER_USE_OR_RETURN(self, -1);
because the caller does. */
UNLESS (self->data && self->len) return 0;
/* Search downward until hitting a bucket, stored in pbucket. */
for (;;) {
Sized *pchild;
int pchild_is_btree;
BTREE_SEARCH(i, self, key, goto Done);
pchild = self->data[i].child;
pchild_is_btree = SameType_Check(self, pchild);
if (i) {
deepest_smaller = self->data[i-1].child;
deepest_smaller_is_btree = pchild_is_btree;
}
if (pchild_is_btree) {
if (self_got_rebound) {
PER_UNUSE(self);
}
self = BTREE(pchild);
self_got_rebound = 1;
PER_USE_OR_RETURN(self, -1);
}
else {
pbucket = BUCKET(pchild);
break;
}
}
/* Search the bucket for a suitable key. */
i = Bucket_findRangeEnd(pbucket, keyarg, low, exclude_equal, offset);
if (i < 0)
goto Done;
if (i > 0) {
Py_INCREF(pbucket);
*bucket = pbucket;
result = 1;
goto Done;
}
/* This may be one of the two difficult cases detailed in the comments. */
if (low) {
Bucket *next;
UNLESS(PER_USE(pbucket)) goto Done;
next = pbucket->next;
if (next) {
result = 1;
Py_INCREF(next);
*bucket = next;
*offset = 0;
}
else
result = 0;
PER_UNUSE(pbucket);
}
/* High-end search: if it's possible to go left, do so. */
else if (deepest_smaller) {
if (deepest_smaller_is_btree) {
UNLESS(PER_USE(deepest_smaller)) goto Done;
/* We own the reference this returns. */
pbucket = BTree_lastBucket(BTREE(deepest_smaller));
PER_UNUSE(deepest_smaller);
if (pbucket == NULL) goto Done; /* error */
}
else {
pbucket = BUCKET(deepest_smaller);
Py_INCREF(pbucket);
}
UNLESS(PER_USE(pbucket)) goto Done;
result = 1;
*bucket = pbucket; /* transfer ownership to caller */
*offset = pbucket->len - 1;
PER_UNUSE(pbucket);
}
else
result = 0; /* simply not found */
Done:
if (self_got_rebound) {
PER_UNUSE(self);
}
return result;
}
static PyObject *
BTree_maxminKey(BTree *self, PyObject *args, int min)
{
PyObject *key=0;
Bucket *bucket = NULL;
int offset, rc;
int empty_tree = 1;
UNLESS (PyArg_ParseTuple(args, "|O", &key)) return NULL;
UNLESS (PER_USE(self)) return NULL;
UNLESS (self->data && self->len) goto empty;
/* Find the range */
if (key)
{
if ((rc = BTree_findRangeEnd(self, key, min, 0, &bucket, &offset)) <= 0)
{
if (rc < 0) goto err;
empty_tree = 0;
goto empty;
}
PER_UNUSE(self);
UNLESS (PER_USE(bucket))
{
Py_DECREF(bucket);
return NULL;
}
}
else if (min)
{
bucket = self->firstbucket;
PER_UNUSE(self);
PER_USE_OR_RETURN(bucket, NULL);
Py_INCREF(bucket);
offset = 0;
}
else
{
bucket = BTree_lastBucket(self);
PER_UNUSE(self);
UNLESS (PER_USE(bucket))
{
Py_DECREF(bucket);
return NULL;
}
assert(bucket->len);
offset = bucket->len - 1;
}
COPY_KEY_TO_OBJECT(key, bucket->keys[offset]);
PER_UNUSE(bucket);
Py_DECREF(bucket);
return key;
empty:
PyErr_SetString(PyExc_ValueError,
empty_tree ? "empty tree" :
"no key satisfies the conditions");
err:
PER_UNUSE(self);
if (bucket)
{
PER_UNUSE(bucket);
Py_DECREF(bucket);
}
return NULL;
}
static PyObject *
BTree_minKey(BTree *self, PyObject *args)
{
return BTree_maxminKey(self, args, 1);
}
static PyObject *
BTree_maxKey(BTree *self, PyObject *args)
{
return BTree_maxminKey(self, args, 0);
}
/*
** BTree_rangeSearch
**
** Generates a BTreeItems object based on the two indexes passed in,
** being the range between them.
**
*/
static PyObject *
BTree_rangeSearch(BTree *self, PyObject *args, PyObject *kw, char type)
{
PyObject *min = Py_None;
PyObject *max = Py_None;
int excludemin = 0;
int excludemax = 0;
int rc;
Bucket *lowbucket = NULL;
Bucket *highbucket = NULL;
int lowoffset;
int highoffset;
PyObject *result;
if (args) {
if (! PyArg_ParseTupleAndKeywords(args, kw, "|OOii", search_keywords,
&min,
&max,
&excludemin,
&excludemax))
return NULL;
}
UNLESS (PER_USE(self)) return NULL;
UNLESS (self->data && self->len) goto empty;
/* Find the low range */
if (min != Py_None) {
if ((rc = BTree_findRangeEnd(self, min, 1, excludemin,
&lowbucket, &lowoffset)) <= 0) {
if (rc < 0) goto err;
goto empty;
}
}
else {
lowbucket = self->firstbucket;
lowoffset = 0;
if (excludemin) {
int bucketlen;
UNLESS (PER_USE(lowbucket)) goto err;
bucketlen = lowbucket->len;
PER_UNUSE(lowbucket);
if (bucketlen > 1)
lowoffset = 1;
else if (self->len < 2)
goto empty;
else { /* move to first item in next bucket */
Bucket *next;
UNLESS (PER_USE(lowbucket)) goto err;
next = lowbucket->next;
PER_UNUSE(lowbucket);
assert(next != NULL);
lowbucket = next;
/* and lowoffset is still 0 */
assert(lowoffset == 0);
}
}
Py_INCREF(lowbucket);
}
/* Find the high range */
if (max != Py_None) {
if ((rc = BTree_findRangeEnd(self, max, 0, excludemax,
&highbucket, &highoffset)) <= 0) {
Py_DECREF(lowbucket);
if (rc < 0) goto err;
goto empty;
}
}
else {
int bucketlen;
highbucket = BTree_lastBucket(self);
assert(highbucket != NULL); /* we know self isn't empty */
UNLESS (PER_USE(highbucket)) goto err_and_decref_buckets;
bucketlen = highbucket->len;
PER_UNUSE(highbucket);
highoffset = bucketlen - 1;
if (excludemax) {
if (highoffset > 0)
--highoffset;
else if (self->len < 2)
goto empty_and_decref_buckets;
else { /* move to last item of preceding bucket */
int status;
assert(highbucket != self->firstbucket);
Py_DECREF(highbucket);
status = PreviousBucket(&highbucket, self->firstbucket);
if (status < 0) {
Py_DECREF(lowbucket);
goto err;
}
assert(status > 0);
Py_INCREF(highbucket);
UNLESS (PER_USE(highbucket)) goto err_and_decref_buckets;
highoffset = highbucket->len - 1;
PER_UNUSE(highbucket);
}
}
assert(highoffset >= 0);
}
/* It's still possible that the range is empty, even if min < max. For
* example, if min=3 and max=4, and 3 and 4 aren't in the BTree, but 2 and
* 5 are, then the low position points to the 5 now and the high position
* points to the 2 now. They're not necessarily even in the same bucket,
* so there's no trick we can play with pointer compares to get out
* cheap in general.
*/
if (lowbucket == highbucket && lowoffset > highoffset)
goto empty_and_decref_buckets; /* definitely empty */
/* The buckets differ, or they're the same and the offsets show a non-
* empty range.
*/
if (min != Py_None && max != Py_None && /* both args user-supplied */
lowbucket != highbucket) /* and different buckets */ {
KEY_TYPE first;
KEY_TYPE last;
int cmp;
/* Have to check the hard way: see how the endpoints compare. */
UNLESS (PER_USE(lowbucket)) goto err_and_decref_buckets;
COPY_KEY(first, lowbucket->keys[lowoffset]);
PER_UNUSE(lowbucket);
UNLESS (PER_USE(highbucket)) goto err_and_decref_buckets;
COPY_KEY(last, highbucket->keys[highoffset]);
PER_UNUSE(highbucket);
TEST_KEY_SET_OR(cmp, first, last) goto err_and_decref_buckets;
if (cmp > 0) goto empty_and_decref_buckets;
}
PER_UNUSE(self);
result = newBTreeItems(type, lowbucket, lowoffset, highbucket, highoffset);
Py_DECREF(lowbucket);
Py_DECREF(highbucket);
return result;
err_and_decref_buckets:
Py_DECREF(lowbucket);
Py_DECREF(highbucket);
err:
PER_UNUSE(self);
return NULL;
empty_and_decref_buckets:
Py_DECREF(lowbucket);
Py_DECREF(highbucket);
empty:
PER_UNUSE(self);
return newBTreeItems(type, 0, 0, 0, 0);
}
/*
** BTree_keys
*/
static PyObject *
BTree_keys(BTree *self, PyObject *args, PyObject *kw)
{
return BTree_rangeSearch(self, args, kw, 'k');
}
/*
** BTree_values
*/
static PyObject *
BTree_values(BTree *self, PyObject *args, PyObject *kw)
{
return BTree_rangeSearch(self, args, kw, 'v');
}
/*
** BTree_items
*/
static PyObject *
BTree_items(BTree *self, PyObject *args, PyObject *kw)
{
return BTree_rangeSearch(self, args, kw, 'i');
}
static PyObject *
BTree_byValue(BTree *self, PyObject *omin)
{
PyObject *r=0, *o=0, *item=0;
VALUE_TYPE min;
VALUE_TYPE v;
int copied=1;
SetIteration it = {0, 0, 1};
UNLESS (PER_USE(self)) return NULL;
COPY_VALUE_FROM_ARG(min, omin, copied);
UNLESS(copied) return NULL;
UNLESS (r=PyList_New(0)) goto err;
it.set=BTree_rangeSearch(self, NULL, NULL, 'i');
UNLESS(it.set) goto err;
if (nextBTreeItems(&it) < 0) goto err;
while (it.position >= 0)
{
if (TEST_VALUE(it.value, min) >= 0)
{
UNLESS (item = PyTuple_New(2)) goto err;
COPY_KEY_TO_OBJECT(o, it.key);
UNLESS (o) goto err;
PyTuple_SET_ITEM(item, 1, o);
COPY_VALUE(v, it.value);
NORMALIZE_VALUE(v, min);
COPY_VALUE_TO_OBJECT(o, v);
DECREF_VALUE(v);
UNLESS (o) goto err;
PyTuple_SET_ITEM(item, 0, o);
if (PyList_Append(r, item) < 0) goto err;
Py_DECREF(item);
item = 0;
}
if (nextBTreeItems(&it) < 0) goto err;
}
item=PyObject_GetAttr(r,sort_str);
UNLESS (item) goto err;
ASSIGN(item, PyObject_CallObject(item, NULL));
UNLESS (item) goto err;
ASSIGN(item, PyObject_GetAttr(r, reverse_str));
UNLESS (item) goto err;
ASSIGN(item, PyObject_CallObject(item, NULL));
UNLESS (item) goto err;
Py_DECREF(item);
finiSetIteration(&it);
PER_UNUSE(self);
return r;
err:
PER_UNUSE(self);
Py_XDECREF(r);
finiSetIteration(&it);
Py_XDECREF(item);
return NULL;
}
/*
** BTree_getm
*/
static PyObject *
BTree_getm(BTree *self, PyObject *args)
{
PyObject *key, *d=Py_None, *r;
UNLESS (PyArg_ParseTuple(args, "O|O", &key, &d)) return NULL;
if ((r=_BTree_get(self, key, 0))) return r;
UNLESS (PyErr_ExceptionMatches(PyExc_KeyError)) return NULL;
PyErr_Clear();
Py_INCREF(d);
return d;
}
static PyObject *
BTree_has_key(BTree *self, PyObject *key)
{
return _BTree_get(self, key, 1);
}
static PyObject *
BTree_setdefault(BTree *self, PyObject *args)
{
PyObject *key;
PyObject *failobj; /* default */
PyObject *value; /* return value */
if (! PyArg_UnpackTuple(args, "setdefault", 2, 2, &key, &failobj))
return NULL;
value = _BTree_get(self, key, 0);
if (value != NULL)
return value;
/* The key isn't in the tree. If that's not due to a KeyError exception,
* pass back the unexpected exception.
*/
if (! PyErr_ExceptionMatches(PyExc_KeyError))
return NULL;
PyErr_Clear();
/* Associate `key` with `failobj` in the tree, and return `failobj`. */
value = failobj;
if (_BTree_set(self, key, failobj, 0, 0) < 0)
value = NULL;
Py_XINCREF(value);
return value;
}
/* forward declaration */
static int
BTree_length_or_nonzero(BTree *self, int nonzero);
static PyObject *
BTree_pop(BTree *self, PyObject *args)
{
PyObject *key;
PyObject *failobj = NULL; /* default */
PyObject *value; /* return value */
if (! PyArg_UnpackTuple(args, "pop", 1, 2, &key, &failobj))
return NULL;
value = _BTree_get(self, key, 0);
if (value != NULL) {
/* Delete key and associated value. */
if (_BTree_set(self, key, NULL, 0, 0) < 0) {
Py_DECREF(value);
return NULL;;
}
return value;
}
/* The key isn't in the tree. If that's not due to a KeyError exception,
* pass back the unexpected exception.
*/
if (! PyErr_ExceptionMatches(PyExc_KeyError))
return NULL;
if (failobj != NULL) {
/* Clear the KeyError and return the explicit default. */
PyErr_Clear();
Py_INCREF(failobj);
return failobj;
}
/* No default given. The only difference in this case is the error
* message, which depends on whether the tree is empty.
*/
if (BTree_length_or_nonzero(self, 1) == 0) /* tree is empty */
PyErr_SetString(PyExc_KeyError, "pop(): BTree is empty");
return NULL;
}
/* Search BTree self for key. This is the sq_contains slot of the
* PySequenceMethods.
*
* Return:
* -1 error
* 0 not found
* 1 found
*/
static int
BTree_contains(BTree *self, PyObject *key)
{
PyObject *asobj = _BTree_get(self, key, 1);
int result = -1;
if (asobj != NULL) {
result = PyInt_AsLong(asobj) ? 1 : 0;
Py_DECREF(asobj);
}
return result;
}
static PyObject *
BTree_addUnique(BTree *self, PyObject *args)
{
int grew;
PyObject *key, *v;
UNLESS (PyArg_ParseTuple(args, "OO", &key, &v)) return NULL;
if ((grew=_BTree_set(self, key, v, 1, 0)) < 0) return NULL;
return PyInt_FromLong(grew);
}
/**************************************************************************/
/* Iterator support. */
/* A helper to build all the iterators for BTrees and TreeSets.
* If args is NULL, the iterator spans the entire structure. Else it's an
* argument tuple, with optional low and high arguments.
* kind is 'k', 'v' or 'i'.
* Returns a BTreeIter object, or NULL if error.
*/
static PyObject *
buildBTreeIter(BTree *self, PyObject *args, PyObject *kw, char kind)
{
BTreeIter *result = NULL;
BTreeItems *items = (BTreeItems *)BTree_rangeSearch(self, args, kw, kind);
if (items) {
result = BTreeIter_new(items);
Py_DECREF(items);
}
return (PyObject *)result;
}
/* The implementation of iter(BTree_or_TreeSet); the BTree tp_iter slot. */
static PyObject *
BTree_getiter(BTree *self)
{
return buildBTreeIter(self, NULL, NULL, 'k');
}
/* The implementation of BTree.iterkeys(). */
static PyObject *
BTree_iterkeys(BTree *self, PyObject *args, PyObject *kw)
{
return buildBTreeIter(self, args, kw, 'k');
}
/* The implementation of BTree.itervalues(). */
static PyObject *
BTree_itervalues(BTree *self, PyObject *args, PyObject *kw)
{
return buildBTreeIter(self, args, kw, 'v');
}
/* The implementation of BTree.iteritems(). */
static PyObject *
BTree_iteritems(BTree *self, PyObject *args, PyObject *kw)
{
return buildBTreeIter(self, args, kw, 'i');
}
/* End of iterator support. */
/* Caution: Even though the _firstbucket attribute is read-only, a program
could do arbitrary damage to the btree internals. For example, it could
call clear() on a bucket inside a BTree.
We need to decide if the convenience for inspecting BTrees is worth
the risk.
*/
static struct PyMemberDef BTree_members[] = {
{"_firstbucket", T_OBJECT, offsetof(BTree, firstbucket), RO},
{NULL}
};
static struct PyMethodDef BTree_methods[] = {
{"__getstate__", (PyCFunction) BTree_getstate, METH_NOARGS,
"__getstate__() -> state\n\n"
"Return the picklable state of the BTree."},
{"__setstate__", (PyCFunction) BTree_setstate, METH_O,
"__setstate__(state)\n\n"
"Set the state of the BTree."},
{"has_key", (PyCFunction) BTree_has_key, METH_O,
"has_key(key)\n\n"
"Return true if the BTree contains the given key."},
{"keys", (PyCFunction) BTree_keys, METH_KEYWORDS,
"keys([min, max]) -> list of keys\n\n"
"Returns the keys of the BTree. If min and max are supplied, only\n"
"keys greater than min and less than max are returned."},
{"values", (PyCFunction) BTree_values, METH_KEYWORDS,
"values([min, max]) -> list of values\n\n"
"Returns the values of the BTree. If min and max are supplied, only\n"
"values corresponding to keys greater than min and less than max are\n"
"returned."},
{"items", (PyCFunction) BTree_items, METH_KEYWORDS,
"items([min, max]) -> -- list of key, value pairs\n\n"
"Returns the items of the BTree. If min and max are supplied, only\n"
"items with keys greater than min and less than max are returned."},
{"byValue", (PyCFunction) BTree_byValue, METH_O,
"byValue(min) -> list of value, key pairs\n\n"
"Returns list of value, key pairs where the value is >= min. The\n"
"list is sorted by value. Note that items() returns keys in the\n"
"opposite order."},
{"get", (PyCFunction) BTree_getm, METH_VARARGS,
"get(key[, default=None]) -> Value for key or default\n\n"
"Return the value or the default if the key is not found."},
{"setdefault", (PyCFunction) BTree_setdefault, METH_VARARGS,
"D.setdefault(k, d) -> D.get(k, d), also set D[k]=d if k not in D.\n\n"
"Return the value like get() except that if key is missing, d is both\n"
"returned and inserted into the BTree as the value of k."},
{"pop", (PyCFunction) BTree_pop, METH_VARARGS,
"D.pop(k[, d]) -> v, remove key and return the corresponding value.\n\n"
"If key is not found, d is returned if given, otherwise KeyError\n"
"is raised."},
{"maxKey", (PyCFunction) BTree_maxKey, METH_VARARGS,
"maxKey([max]) -> key\n\n"
"Return the largest key in the BTree. If max is specified, return\n"
"the largest key <= max."},
{"minKey", (PyCFunction) BTree_minKey, METH_VARARGS,
"minKey([mi]) -> key\n\n"
"Return the smallest key in the BTree. If min is specified, return\n"
"the smallest key >= min."},
{"clear", (PyCFunction) BTree_clear, METH_NOARGS,
"clear()\n\nRemove all of the items from the BTree."},
{"insert", (PyCFunction)BTree_addUnique, METH_VARARGS,
"insert(key, value) -> 0 or 1\n\n"
"Add an item if the key is not already used. Return 1 if the item was\n"
"added, or 0 otherwise."},
{"update", (PyCFunction) Mapping_update, METH_O,
"update(collection)\n\n Add the items from the given collection."},
{"iterkeys", (PyCFunction) BTree_iterkeys, METH_KEYWORDS,
"B.iterkeys([min[,max]]) -> an iterator over the keys of B"},
{"itervalues", (PyCFunction) BTree_itervalues, METH_KEYWORDS,
"B.itervalues([min[,max]]) -> an iterator over the values of B"},
{"iteritems", (PyCFunction) BTree_iteritems, METH_KEYWORDS,
"B.iteritems([min[,max]]) -> an iterator over the (key, value) items of B"},
{"_check", (PyCFunction) BTree_check, METH_NOARGS,
"Perform sanity check on BTree, and raise exception if flawed."},
#ifdef PERSISTENT
{"_p_resolveConflict", (PyCFunction) BTree__p_resolveConflict,
METH_VARARGS,
"_p_resolveConflict() -- Reinitialize from a newly created copy"},
{"_p_deactivate", (PyCFunction) BTree__p_deactivate, METH_KEYWORDS,
"_p_deactivate()\n\nReinitialize from a newly created copy."},
#endif
{NULL, NULL}
};
static int
BTree_init(PyObject *self, PyObject *args, PyObject *kwds)
{
PyObject *v = NULL;
if (!PyArg_ParseTuple(args, "|O:" MOD_NAME_PREFIX "BTree", &v))
return -1;
if (v)
return update_from_seq(self, v);
else
return 0;
}
static void
BTree_dealloc(BTree *self)
{
if (self->state != cPersistent_GHOST_STATE)
_BTree_clear(self);
cPersistenceCAPI->pertype->tp_dealloc((PyObject *)self);
}
static int
BTree_traverse(BTree *self, visitproc visit, void *arg)
{
int err = 0;
int i, len;
#define VISIT(SLOT) \
if (SLOT) { \
err = visit((PyObject *)(SLOT), arg); \
if (err) \
goto Done; \
}
if (self->ob_type == &BTreeType)
assert(self->ob_type->tp_dictoffset == 0);
/* Call our base type's traverse function. Because BTrees are
* subclasses of Peristent, there must be one.
*/
err = cPersistenceCAPI->pertype->tp_traverse((PyObject *)self, visit, arg);
if (err)
goto Done;
/* If this is registered with the persistence system, cleaning up cycles
* is the database's problem. It would be horrid to unghostify BTree
* nodes here just to chase pointers every time gc runs.
*/
if (self->state == cPersistent_GHOST_STATE)
goto Done;
len = self->len;
#ifdef KEY_TYPE_IS_PYOBJECT
/* Keys are Python objects so need to be traversed. Note that the
* key 0 slot is unused and should not be traversed.
*/
for (i = 1; i < len; i++)
VISIT(self->data[i].key);
#endif
/* Children are always pointers, and child 0 is legit. */
for (i = 0; i < len; i++)
VISIT(self->data[i].child);
VISIT(self->firstbucket);
Done:
return err;
#undef VISIT
}
static int
BTree_tp_clear(BTree *self)
{
if (self->state != cPersistent_GHOST_STATE)
_BTree_clear(self);
return 0;
}
/*
* Return the number of elements in a BTree. nonzero is a Boolean, and
* when true requests just a non-empty/empty result. Testing for emptiness
* is efficient (constant-time). Getting the true length takes time
* proportional to the number of leaves (buckets).
*
* Return:
* When nonzero true:
* -1 error
* 0 empty
* 1 not empty
* When nonzero false (possibly expensive!):
* -1 error
* >= 0 number of elements.
*/
static int
BTree_length_or_nonzero(BTree *self, int nonzero)
{
int result;
Bucket *b;
Bucket *next;
PER_USE_OR_RETURN(self, -1);
b = self->firstbucket;
PER_UNUSE(self);
if (nonzero)
return b != NULL;
result = 0;
while (b) {
PER_USE_OR_RETURN(b, -1);
result += b->len;
next = b->next;
PER_UNUSE(b);
b = next;
}
return result;
}
static int
BTree_length( BTree *self)
{
return BTree_length_or_nonzero(self, 0);
}
static PyMappingMethods BTree_as_mapping = {
(inquiry)BTree_length, /*mp_length*/
(binaryfunc)BTree_get, /*mp_subscript*/
(objobjargproc)BTree_setitem, /*mp_ass_subscript*/
};
static PySequenceMethods BTree_as_sequence = {
(inquiry)0, /* sq_length */
(binaryfunc)0, /* sq_concat */
(intargfunc)0, /* sq_repeat */
(intargfunc)0, /* sq_item */
(intintargfunc)0, /* sq_slice */
(intobjargproc)0, /* sq_ass_item */
(intintobjargproc)0, /* sq_ass_slice */
(objobjproc)BTree_contains, /* sq_contains */
0, /* sq_inplace_concat */
0, /* sq_inplace_repeat */
};
static int
BTree_nonzero(BTree *self)
{
return BTree_length_or_nonzero(self, 1);
}
static PyNumberMethods BTree_as_number_for_nonzero = {
0,0,0,0,0,0,0,0,0,0,
(inquiry)BTree_nonzero};
static PyTypeObject BTreeType = {
PyObject_HEAD_INIT(NULL) /* PyPersist_Type */
0, /* ob_size */
MODULE_NAME MOD_NAME_PREFIX "BTree",/* tp_name */
sizeof(BTree), /* tp_basicsize */
0, /* tp_itemsize */
(destructor)BTree_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_compare */
0, /* tp_repr */
&BTree_as_number_for_nonzero, /* tp_as_number */
&BTree_as_sequence, /* tp_as_sequence */
&BTree_as_mapping, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
Py_TPFLAGS_BASETYPE, /* tp_flags */
0, /* tp_doc */
(traverseproc)BTree_traverse, /* tp_traverse */
(inquiry)BTree_tp_clear, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
(getiterfunc)BTree_getiter, /* tp_iter */
0, /* tp_iternext */
BTree_methods, /* tp_methods */
BTree_members, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
BTree_init, /* tp_init */
0, /* tp_alloc */
0, /*PyType_GenericNew,*/ /* tp_new */
};
ZODB/src/BTrees/objectvaluemacros.h 0000640 0001750 0000144 00000001015 10051217333 017006 0 ustar ctheune users
#define VALUEMACROS_H "$Id: objectvaluemacros.h 24682 2004-05-14 19:32:43Z tim_one $\n"
#define VALUE_TYPE PyObject *
#define VALUE_TYPE_IS_PYOBJECT
#define TEST_VALUE(VALUE, TARGET) PyObject_Compare((VALUE),(TARGET))
#define DECLARE_VALUE(NAME) VALUE_TYPE NAME
#define INCREF_VALUE(k) Py_INCREF(k)
#define DECREF_VALUE(k) Py_DECREF(k)
#define COPY_VALUE(k,e) k=(e)
#define COPY_VALUE_TO_OBJECT(O, K) O=(K); Py_INCREF(O)
#define COPY_VALUE_FROM_ARG(TARGET, ARG, S) TARGET=(ARG)
#define NORMALIZE_VALUE(V, MIN) Py_INCREF(V)
ZODB/src/BTrees/DEPENDENCIES.cfg 0000640 0001750 0000144 00000000046 10223352617 015445 0 ustar ctheune users persistent
transaction
zope.interface
ZODB/src/BTrees/TreeSetTemplate.c 0000640 0001750 0000144 00000015530 10057366465 016370 0 ustar ctheune users /*****************************************************************************
Copyright (c) 2001, 2002 Zope Corporation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
#define TREESETTEMPLATE_C "$Id: TreeSetTemplate.c 25186 2004-06-02 15:07:33Z jim $\n"
static PyObject *
TreeSet_insert(BTree *self, PyObject *args)
{
PyObject *key;
int i;
if (!PyArg_ParseTuple(args, "O:insert", &key))
return NULL;
i = _BTree_set(self, key, Py_None, 1, 1);
if (i < 0)
return NULL;
return PyInt_FromLong(i);
}
/* _Set_update and _TreeSet_update are identical except for the
function they call to add the element to the set.
*/
static int
_TreeSet_update(BTree *self, PyObject *seq)
{
int n = -1;
PyObject *iter, *v;
int ind;
iter = PyObject_GetIter(seq);
if (iter == NULL)
return -1;
while (1) {
v = PyIter_Next(iter);
if (v == NULL) {
if (PyErr_Occurred())
goto err;
else
break;
}
ind = _BTree_set(self, v, Py_None, 1, 1);
Py_DECREF(v);
if (ind < 0)
goto err;
else
n += ind;
}
/* n starts out at -1, which is the error return value. If
this point is reached, then there is no error. n must be
incremented to account for the initial value of -1 instead of
0.
*/
n++;
err:
Py_DECREF(iter);
return n;
}
static PyObject *
TreeSet_update(BTree *self, PyObject *args)
{
PyObject *seq = NULL;
int n = 0;
if (!PyArg_ParseTuple(args, "|O:update", &seq))
return NULL;
if (seq) {
n = _TreeSet_update(self, seq);
if (n < 0)
return NULL;
}
return PyInt_FromLong(n);
}
static PyObject *
TreeSet_remove(BTree *self, PyObject *args)
{
PyObject *key;
UNLESS (PyArg_ParseTuple(args, "O", &key)) return NULL;
if (_BTree_set(self, key, NULL, 0, 1) < 0) return NULL;
Py_INCREF(Py_None);
return Py_None;
}
static PyObject *
TreeSet_setstate(BTree *self, PyObject *args)
{
int r;
if (!PyArg_ParseTuple(args,"O",&args)) return NULL;
PER_PREVENT_DEACTIVATION(self);
r=_BTree_setstate(self, args, 1);
PER_UNUSE(self);
if (r < 0) return NULL;
Py_INCREF(Py_None);
return Py_None;
}
static struct PyMethodDef TreeSet_methods[] = {
{"__getstate__", (PyCFunction) BTree_getstate, METH_NOARGS,
"__getstate__() -> state\n\n"
"Return the picklable state of the TreeSet."},
{"__setstate__", (PyCFunction) TreeSet_setstate, METH_VARARGS,
"__setstate__(state)\n\n"
"Set the state of the TreeSet."},
{"has_key", (PyCFunction) BTree_has_key, METH_O,
"has_key(key)\n\n"
"Return true if the TreeSet contains the given key."},
{"keys", (PyCFunction) BTree_keys, METH_KEYWORDS,
"keys([min, max]) -> list of keys\n\n"
"Returns the keys of the TreeSet. If min and max are supplied, only\n"
"keys greater than min and less than max are returned."},
{"maxKey", (PyCFunction) BTree_maxKey, METH_VARARGS,
"maxKey([max]) -> key\n\n"
"Return the largest key in the BTree. If max is specified, return\n"
"the largest key <= max."},
{"minKey", (PyCFunction) BTree_minKey, METH_VARARGS,
"minKey([mi]) -> key\n\n"
"Return the smallest key in the BTree. If min is specified, return\n"
"the smallest key >= min."},
{"clear", (PyCFunction) BTree_clear, METH_NOARGS,
"clear()\n\nRemove all of the items from the BTree."},
{"insert", (PyCFunction)TreeSet_insert, METH_VARARGS,
"insert(id,[ignored]) -- Add an id to the set"},
{"update", (PyCFunction)TreeSet_update, METH_VARARGS,
"update(collection)\n\n Add the items from the given collection."},
{"remove", (PyCFunction)TreeSet_remove, METH_VARARGS,
"remove(id) -- Remove a key from the set"},
{"_check", (PyCFunction) BTree_check, METH_NOARGS,
"Perform sanity check on TreeSet, and raise exception if flawed."},
#ifdef PERSISTENT
{"_p_resolveConflict", (PyCFunction) BTree__p_resolveConflict, METH_VARARGS,
"_p_resolveConflict() -- Reinitialize from a newly created copy"},
{"_p_deactivate", (PyCFunction) BTree__p_deactivate, METH_KEYWORDS,
"_p_deactivate()\n\nReinitialize from a newly created copy."},
#endif
{NULL, NULL} /* sentinel */
};
static PyMappingMethods TreeSet_as_mapping = {
(inquiry)BTree_length, /*mp_length*/
};
static PySequenceMethods TreeSet_as_sequence = {
(inquiry)0, /* sq_length */
(binaryfunc)0, /* sq_concat */
(intargfunc)0, /* sq_repeat */
(intargfunc)0, /* sq_item */
(intintargfunc)0, /* sq_slice */
(intobjargproc)0, /* sq_ass_item */
(intintobjargproc)0, /* sq_ass_slice */
(objobjproc)BTree_contains, /* sq_contains */
0, /* sq_inplace_concat */
0, /* sq_inplace_repeat */
};
static int
TreeSet_init(PyObject *self, PyObject *args, PyObject *kwds)
{
PyObject *v = NULL;
if (!PyArg_ParseTuple(args, "|O:" MOD_NAME_PREFIX "TreeSet", &v))
return -1;
if (v)
return _TreeSet_update((BTree *)self, v);
else
return 0;
}
static PyTypeObject TreeSetType = {
PyObject_HEAD_INIT(NULL) /* PyPersist_Type */
0, /* ob_size */
MODULE_NAME MOD_NAME_PREFIX "TreeSet",/* tp_name */
sizeof(BTree), /* tp_basicsize */
0, /* tp_itemsize */
(destructor)BTree_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_compare */
0, /* tp_repr */
&BTree_as_number_for_nonzero, /* tp_as_number */
&TreeSet_as_sequence, /* tp_as_sequence */
&TreeSet_as_mapping, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
Py_TPFLAGS_BASETYPE, /* tp_flags */
0, /* tp_doc */
(traverseproc)BTree_traverse, /* tp_traverse */
(inquiry)BTree_tp_clear, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
(getiterfunc)BTree_getiter, /* tp_iter */
0, /* tp_iternext */
TreeSet_methods, /* tp_methods */
BTree_members, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
TreeSet_init, /* tp_init */
0, /* tp_alloc */
0, /*PyType_GenericNew,*/ /* tp_new */
};
ZODB/src/BTrees/SetTemplate.c 0000640 0001750 0000144 00000020717 10057366465 015553 0 ustar ctheune users /*****************************************************************************
Copyright (c) 2001, 2002 Zope Corporation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
#define SETTEMPLATE_C "$Id: SetTemplate.c 25186 2004-06-02 15:07:33Z jim $\n"
static PyObject *
Set_insert(Bucket *self, PyObject *args)
{
PyObject *key;
int i;
UNLESS (PyArg_ParseTuple(args, "O", &key)) return NULL;
if ( (i=_bucket_set(self, key, Py_None, 1, 1, 0)) < 0) return NULL;
return PyInt_FromLong(i);
}
/* _Set_update and _TreeSet_update are identical except for the
function they call to add the element to the set.
*/
static int
_Set_update(Bucket *self, PyObject *seq)
{
int n = -1;
PyObject *iter, *v;
int ind;
iter = PyObject_GetIter(seq);
if (iter == NULL)
return -1;
while (1) {
v = PyIter_Next(iter);
if (v == NULL) {
if (PyErr_Occurred())
goto err;
else
break;
}
ind = _bucket_set(self, v, Py_None, 1, 1, 0);
Py_DECREF(v);
if (ind < 0)
goto err;
else
n += ind;
}
/* n starts out at -1, which is the error return value. If
this point is reached, then there is no error. n must be
incremented to account for the initial value of -1 instead of
0.
*/
n++;
err:
Py_DECREF(iter);
return n;
}
static PyObject *
Set_update(Bucket *self, PyObject *args)
{
PyObject *seq = NULL;
int n = 0;
if (!PyArg_ParseTuple(args, "|O:update", &seq))
return NULL;
if (seq) {
n = _Set_update(self, seq);
if (n < 0)
return NULL;
}
return PyInt_FromLong(n);
}
static PyObject *
Set_remove(Bucket *self, PyObject *args)
{
PyObject *key;
UNLESS (PyArg_ParseTuple(args, "O", &key)) return NULL;
if (_bucket_set(self, key, NULL, 0, 1, 0) < 0) return NULL;
Py_INCREF(Py_None);
return Py_None;
}
static int
_set_setstate(Bucket *self, PyObject *args)
{
PyObject *k, *items;
Bucket *next=0;
int i, l, copied=1;
KEY_TYPE *keys;
UNLESS (PyArg_ParseTuple(args, "O|O", &items, &next))
return -1;
if ((l=PyTuple_Size(items)) < 0) return -1;
for (i=self->len; --i >= 0; )
{
DECREF_KEY(self->keys[i]);
}
self->len=0;
if (self->next)
{
Py_DECREF(self->next);
self->next=0;
}
if (l > self->size)
{
UNLESS (keys=BTree_Realloc(self->keys, sizeof(KEY_TYPE)*l)) return -1;
self->keys=keys;
self->size=l;
}
for (i=0; ikeys[i], k, copied);
UNLESS (copied) return -1;
INCREF_KEY(self->keys[i]);
}
self->len=l;
if (next)
{
self->next=next;
Py_INCREF(next);
}
return 0;
}
static PyObject *
set_setstate(Bucket *self, PyObject *args)
{
int r;
UNLESS (PyArg_ParseTuple(args, "O", &args)) return NULL;
PER_PREVENT_DEACTIVATION(self);
r=_set_setstate(self, args);
PER_UNUSE(self);
if (r < 0) return NULL;
Py_INCREF(Py_None);
return Py_None;
}
static struct PyMethodDef Set_methods[] = {
{"__getstate__", (PyCFunction) bucket_getstate, METH_VARARGS,
"__getstate__() -- Return the picklable state of the object"},
{"__setstate__", (PyCFunction) set_setstate, METH_VARARGS,
"__setstate__() -- Set the state of the object"},
{"keys", (PyCFunction) bucket_keys, METH_KEYWORDS,
"keys() -- Return the keys"},
{"has_key", (PyCFunction) bucket_has_key, METH_O,
"has_key(key) -- Test whether the bucket contains the given key"},
{"clear", (PyCFunction) bucket_clear, METH_VARARGS,
"clear() -- Remove all of the items from the bucket"},
{"maxKey", (PyCFunction) Bucket_maxKey, METH_VARARGS,
"maxKey([key]) -- Find the maximum key\n\n"
"If an argument is given, find the maximum <= the argument"},
{"minKey", (PyCFunction) Bucket_minKey, METH_VARARGS,
"minKey([key]) -- Find the minimum key\n\n"
"If an argument is given, find the minimum >= the argument"},
#ifdef PERSISTENT
{"_p_resolveConflict", (PyCFunction) bucket__p_resolveConflict, METH_VARARGS,
"_p_resolveConflict() -- Reinitialize from a newly created copy"},
{"_p_deactivate", (PyCFunction) bucket__p_deactivate, METH_KEYWORDS,
"_p_deactivate() -- Reinitialize from a newly created copy"},
#endif
{"insert", (PyCFunction)Set_insert, METH_VARARGS,
"insert(id,[ignored]) -- Add a key to the set"},
{"update", (PyCFunction)Set_update, METH_VARARGS,
"update(seq) -- Add the items from the given sequence to the set"},
{"remove", (PyCFunction)Set_remove, METH_VARARGS,
"remove(id) -- Remove an id from the set"},
{NULL, NULL} /* sentinel */
};
static int
Set_init(PyObject *self, PyObject *args, PyObject *kwds)
{
PyObject *v = NULL;
if (!PyArg_ParseTuple(args, "|O:" MOD_NAME_PREFIX "Set", &v))
return -1;
if (v)
return _Set_update((Bucket *)self, v);
else
return 0;
}
static PyObject *
set_repr(Bucket *self)
{
static PyObject *format;
PyObject *r, *t;
if (!format)
format = PyString_FromString(MOD_NAME_PREFIX "Set(%s)");
UNLESS (t = PyTuple_New(1)) return NULL;
UNLESS (r = bucket_keys(self, NULL, NULL)) goto err;
PyTuple_SET_ITEM(t, 0, r);
r = t;
ASSIGN(r, PyString_Format(format, r));
return r;
err:
Py_DECREF(t);
return NULL;
}
static int
set_length(Bucket *self)
{
int r;
PER_USE_OR_RETURN(self, -1);
r = self->len;
PER_UNUSE(self);
return r;
}
static PyObject *
set_item(Bucket *self, int index)
{
PyObject *r=0;
PER_USE_OR_RETURN(self, NULL);
if (index >= 0 && index < self->len)
{
COPY_KEY_TO_OBJECT(r, self->keys[index]);
}
else
IndexError(index);
PER_UNUSE(self);
return r;
}
static PySequenceMethods set_as_sequence = {
(inquiry)set_length, /* sq_length */
(binaryfunc)0, /* sq_concat */
(intargfunc)0, /* sq_repeat */
(intargfunc)set_item, /* sq_item */
(intintargfunc)0, /* sq_slice */
(intobjargproc)0, /* sq_ass_item */
(intintobjargproc)0, /* sq_ass_slice */
(objobjproc)bucket_contains, /* sq_contains */
0, /* sq_inplace_concat */
0, /* sq_inplace_repeat */
};
static PyTypeObject SetType = {
PyObject_HEAD_INIT(NULL) /* PyPersist_Type */
0, /* ob_size */
MODULE_NAME MOD_NAME_PREFIX "Set", /* tp_name */
sizeof(Bucket), /* tp_basicsize */
0, /* tp_itemsize */
(destructor)bucket_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_compare */
(reprfunc)set_repr, /* tp_repr */
0, /* tp_as_number */
&set_as_sequence, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
Py_TPFLAGS_BASETYPE, /* tp_flags */
0, /* tp_doc */
(traverseproc)bucket_traverse, /* tp_traverse */
(inquiry)bucket_tp_clear, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
(getiterfunc)Bucket_getiter, /* tp_iter */
0, /* tp_iternext */
Set_methods, /* tp_methods */
Bucket_members, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
Set_init, /* tp_init */
0, /* tp_alloc */
0, /*PyType_GenericNew,*/ /* tp_new */
};
static int
nextSet(SetIteration *i)
{
if (i->position >= 0)
{
UNLESS(PER_USE(BUCKET(i->set))) return -1;
if (i->position)
{
DECREF_KEY(i->key);
}
if (i->position < BUCKET(i->set)->len)
{
COPY_KEY(i->key, BUCKET(i->set)->keys[i->position]);
INCREF_KEY(i->key);
i->position ++;
}
else
{
i->position = -1;
PER_ACCESSED(BUCKET(i->set));
}
PER_ALLOW_DEACTIVATION(BUCKET(i->set));
}
return 0;
}
ZODB/src/BTrees/SETUP.cfg 0000640 0001750 0000144 00000005401 10156136553 014523 0 ustar ctheune users # Extension information for zpkg.
# These extensions depend on a header provided by another component;
# this header is named using depends-on in the sections for each
# extension that requires it, but must also be identified as a public
# header in the component that needs it.
source _fsBTree.c
# Common btree includes:
depends-on BTreeItemsTemplate.c
depends-on BTreeModuleTemplate.c
depends-on BTreeTemplate.c
depends-on BucketTemplate.c
depends-on MergeTemplate.c
depends-on SetOpTemplate.c
depends-on SetTemplate.c
depends-on TreeSetTemplate.c
depends-on sorters.c
source _IIBTree.c
# Specialization:
depends-on intkeymacros.h
depends-on intvaluemacros.h
# Common btree includes:
depends-on BTreeItemsTemplate.c
depends-on BTreeModuleTemplate.c
depends-on BTreeTemplate.c
depends-on BucketTemplate.c
depends-on MergeTemplate.c
depends-on SetOpTemplate.c
depends-on SetTemplate.c
depends-on TreeSetTemplate.c
depends-on sorters.c
source _IFBTree.c
# Specialization:
depends-on intkeymacros.h
depends-on floatvaluemacros.h
# Common btree includes:
depends-on BTreeItemsTemplate.c
depends-on BTreeModuleTemplate.c
depends-on BTreeTemplate.c
depends-on BucketTemplate.c
depends-on MergeTemplate.c
depends-on SetOpTemplate.c
depends-on SetTemplate.c
depends-on TreeSetTemplate.c
depends-on sorters.c
source _IOBTree.c
# Specialization:
depends-on intkeymacros.h
depends-on objectvaluemacros.h
# Common btree includes:
depends-on BTreeItemsTemplate.c
depends-on BTreeModuleTemplate.c
depends-on BTreeTemplate.c
depends-on BucketTemplate.c
depends-on MergeTemplate.c
depends-on SetOpTemplate.c
depends-on SetTemplate.c
depends-on TreeSetTemplate.c
depends-on sorters.c
source _OIBTree.c
# Specialization:
depends-on objectkeymacros.h
depends-on intvaluemacros.h
# Common btree includes:
depends-on BTreeItemsTemplate.c
depends-on BTreeModuleTemplate.c
depends-on BTreeTemplate.c
depends-on BucketTemplate.c
depends-on MergeTemplate.c
depends-on SetOpTemplate.c
depends-on SetTemplate.c
depends-on TreeSetTemplate.c
depends-on sorters.c
source _OOBTree.c
# Specialization:
depends-on objectkeymacros.h
depends-on objectvaluemacros.h
# Common btree includes:
depends-on BTreeItemsTemplate.c
depends-on BTreeModuleTemplate.c
depends-on BTreeTemplate.c
depends-on BucketTemplate.c
depends-on MergeTemplate.c
depends-on SetOpTemplate.c
depends-on SetTemplate.c
depends-on TreeSetTemplate.c
depends-on sorters.c
ZODB/src/BTrees/_IIBTree.c 0000640 0001750 0000144 00000002104 10057366465 014674 0 ustar ctheune users /*############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
############################################################################*/
#define MASTER_ID "$Id: _IIBTree.c 25186 2004-06-02 15:07:33Z jim $\n"
/* IIBTree - int key, int value BTree
Implements a collection using int type keys
and int type values
*/
/* Setup template macros */
#define PERSISTENT
#define MOD_NAME_PREFIX "II"
#define INITMODULE init_IIBTree
#define DEFAULT_MAX_BUCKET_SIZE 120
#define DEFAULT_MAX_BTREE_SIZE 500
#include "intkeymacros.h"
#include "intvaluemacros.h"
#include "BTreeModuleTemplate.c"
ZODB/src/BTrees/check.py 0000640 0001750 0000144 00000034720 10156136553 014577 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""
Utilities for working with BTrees (TreeSets, Buckets, and Sets) at a low
level.
The primary function is check(btree), which performs value-based consistency
checks of a kind btree._check() does not perform. See the function docstring
for details.
display(btree) displays the internal structure of a BTree (TreeSet, etc) to
stdout.
CAUTION: When a BTree node has only a single bucket child, it can be
impossible to get at the bucket from Python code (__getstate__() may squash
the bucket object out of existence, as a pickling storage optimization). In
such a case, the code here synthesizes a temporary bucket with the same keys
(and values, if the bucket is of a mapping type). This has no first-order
consequences, but can mislead if you pay close attention to reported object
addresses and/or object identity (the synthesized bucket has an address
that doesn't exist in the actual BTree).
"""
from types import TupleType
from BTrees.OOBTree import OOBTree, OOBucket, OOSet, OOTreeSet
from BTrees.OIBTree import OIBTree, OIBucket, OISet, OITreeSet
from BTrees.IOBTree import IOBTree, IOBucket, IOSet, IOTreeSet
from BTrees.IIBTree import IIBTree, IIBucket, IISet, IITreeSet
from BTrees.IFBTree import IFBTree, IFBucket, IFSet, IFTreeSet
from ZODB.utils import positive_id, oid_repr
TYPE_UNKNOWN, TYPE_BTREE, TYPE_BUCKET = range(3)
_type2kind = {IOBTree: (TYPE_BTREE, True),
IIBTree: (TYPE_BTREE, True),
IFBTree: (TYPE_BTREE, True),
OIBTree: (TYPE_BTREE, True),
OOBTree: (TYPE_BTREE, True),
IOBucket: (TYPE_BUCKET, True),
IIBucket: (TYPE_BUCKET, True),
IFBucket: (TYPE_BUCKET, True),
OIBucket: (TYPE_BUCKET, True),
OOBucket: (TYPE_BUCKET, True),
IOTreeSet: (TYPE_BTREE, False),
IITreeSet: (TYPE_BTREE, False),
IFTreeSet: (TYPE_BTREE, False),
OITreeSet: (TYPE_BTREE, False),
OOTreeSet: (TYPE_BTREE, False),
IOSet: (TYPE_BUCKET, False),
IISet: (TYPE_BUCKET, False),
IFSet: (TYPE_BUCKET, False),
OISet: (TYPE_BUCKET, False),
OOSet: (TYPE_BUCKET, False),
}
# Return pair
#
# TYPE_BTREE or TYPE_BUCKET, is_mapping
def classify(obj):
return _type2kind[type(obj)]
BTREE_EMPTY, BTREE_ONE, BTREE_NORMAL = range(3)
# If the BTree is empty, returns
#
# BTREE_EMPTY, [], []
#
# If the BTree has only one bucket, sometimes returns
#
# BTREE_ONE, bucket_state, None
#
# Else returns
#
# BTREE_NORMAL, list of keys, list of kids
#
# and the list of kids has one more entry than the list of keys.
#
# BTree.__getstate__() docs:
#
# For an empty BTree (self->len == 0), None.
#
# For a BTree with one child (self->len == 1), and that child is a bucket,
# and that bucket has a NULL oid, a one-tuple containing a one-tuple
# containing the bucket's state:
#
# (
# (
# child[0].__getstate__(),
# ),
# )
#
# Else a two-tuple. The first element is a tuple interleaving the BTree's
# keys and direct children, of size 2*self->len - 1 (key[0] is unused and
# is not saved). The second element is the firstbucket:
#
# (
# (child[0], key[1], child[1], key[2], child[2], ...,
# key[len-1], child[len-1]),
# self->firstbucket
# )
_btree2bucket = {IOBTree: IOBucket,
IOTreeSet: IOSet,
IIBTree: IIBucket,
IITreeSet: IISet,
IFBTree: IFBucket,
IFTreeSet: IFSet,
OIBTree: OIBucket,
OITreeSet: OISet,
OOBTree: OOBucket,
OOTreeSet: OOSet}
def crack_btree(t, is_mapping):
state = t.__getstate__()
if state is None:
return BTREE_EMPTY, [], []
assert isinstance(state, TupleType)
if len(state) == 1:
state = state[0]
assert isinstance(state, TupleType) and len(state) == 1
state = state[0]
return BTREE_ONE, state, None
assert len(state) == 2
data, firstbucket = state
n = len(data)
assert n & 1
kids = []
keys = []
i = 0
for x in data:
if i & 1:
keys.append(x)
else:
kids.append(x)
i += 1
return BTREE_NORMAL, keys, kids
# Returns
#
# keys, values # for a mapping; len(keys) == len(values) in this case
# or
# keys, [] # for a set
#
# bucket.__getstate__() docs:
#
# For a set bucket (self->values is NULL), a one-tuple or two-tuple. The
# first element is a tuple of keys, of length self->len. The second element
# is the next bucket, present if and only if next is non-NULL:
#
# (
# (keys[0], keys[1], ..., keys[len-1]),
# next iff non-NULL>
# )
#
# For a mapping bucket (self->values is not NULL), a one-tuple or two-tuple.
# The first element is a tuple interleaving keys and values, of length
# 2 * self->len. The second element is the next bucket, present iff next is
# non-NULL:
#
# (
# (keys[0], values[0], keys[1], values[1], ...,
# keys[len-1], values[len-1]),
# next iff non-NULL>
# )
def crack_bucket(b, is_mapping):
state = b.__getstate__()
assert isinstance(state, TupleType)
assert 1 <= len(state) <= 2
data = state[0]
if not is_mapping:
return data, []
keys = []
values = []
n = len(data)
assert n & 1 == 0
i = 0
for x in data:
if i & 1:
values.append(x)
else:
keys.append(x)
i += 1
return keys, values
def type_and_adr(obj):
if hasattr(obj, '_p_oid'):
oid = oid_repr(obj._p_oid)
else:
oid = 'None'
return "%s (0x%x oid=%s)" % (type(obj).__name__, positive_id(obj), oid)
# Walker implements a depth-first search of a BTree (or TreeSet or Set or
# Bucket). Subclasses must implement the visit_btree() and visit_bucket()
# methods, and arrange to call the walk() method. walk() calls the
# visit_XYZ() methods once for each node in the tree, in depth-first
# left-to-right order.
class Walker:
def __init__(self, obj):
self.obj = obj
# obj is the BTree (BTree or TreeSet).
# path is a list of indices, from the root. For example, if a BTree node
# is child[5] of child[3] of the root BTree, [3, 5].
# parent is the parent BTree object, or None if this is the root BTree.
# is_mapping is True for a BTree and False for a TreeSet.
# keys is a list of the BTree's internal keys.
# kids is a list of the BTree's children.
# If the BTree is an empty root node, keys == kids == [].
# Else len(kids) == len(keys) + 1.
# lo and hi are slice bounds on the values the elements of keys *should*
# lie in (lo inclusive, hi exclusive). lo is None if there is no lower
# bound known, and hi is None if no upper bound is known.
def visit_btree(self, obj, path, parent, is_mapping,
keys, kids, lo, hi):
raise NotImplementedError
# obj is the bucket (Bucket or Set).
# path is a list of indices, from the root. For example, if a bucket
# node is child[5] of child[3] of the root BTree, [3, 5].
# parent is the parent BTree object.
# is_mapping is True for a Bucket and False for a Set.
# keys is a list of the bucket's keys.
# values is a list of the bucket's values.
# If is_mapping is false, values == []. Else len(keys) == len(values).
# lo and hi are slice bounds on the values the elements of keys *should*
# lie in (lo inclusive, hi exclusive). lo is None if there is no lower
# bound known, and hi is None if no upper bound is known.
def visit_bucket(self, obj, path, parent, is_mapping,
keys, values, lo, hi):
raise NotImplementedError
def walk(self):
obj = self.obj
path = []
stack = [(obj, path, None, None, None)]
while stack:
obj, path, parent, lo, hi = stack.pop()
kind, is_mapping = classify(obj)
if kind is TYPE_BTREE:
bkind, keys, kids = crack_btree(obj, is_mapping)
if bkind is BTREE_NORMAL:
# push the kids, in reverse order (so they're popped off
# the stack in forward order)
n = len(kids)
for i in range(len(kids)-1, -1, -1):
newlo, newhi = lo, hi
if i < n-1:
newhi = keys[i]
if i > 0:
newlo = keys[i-1]
stack.append((kids[i],
path + [i],
obj,
newlo,
newhi))
elif bkind is BTREE_EMPTY:
pass
else:
assert bkind is BTREE_ONE
# Yuck. There isn't a bucket object to pass on, as
# the bucket state is embedded directly in the BTree
# state. Synthesize a bucket.
assert kids is None # and "keys" is really the bucket
# state
bucket = _btree2bucket[type(obj)]()
bucket.__setstate__(keys)
stack.append((bucket,
path + [0],
obj,
lo,
hi))
keys = []
kids = [bucket]
self.visit_btree(obj,
path,
parent,
is_mapping,
keys,
kids,
lo,
hi)
else:
assert kind is TYPE_BUCKET
keys, values = crack_bucket(obj, is_mapping)
self.visit_bucket(obj,
path,
parent,
is_mapping,
keys,
values,
lo,
hi)
class Checker(Walker):
def __init__(self, obj):
Walker.__init__(self, obj)
self.errors = []
def check(self):
self.walk()
if self.errors:
s = "Errors found in %s:" % type_and_adr(self.obj)
self.errors.insert(0, s)
s = "\n".join(self.errors)
raise AssertionError(s)
def visit_btree(self, obj, path, parent, is_mapping,
keys, kids, lo, hi):
self.check_sorted(obj, path, keys, lo, hi)
def visit_bucket(self, obj, path, parent, is_mapping,
keys, values, lo, hi):
self.check_sorted(obj, path, keys, lo, hi)
def check_sorted(self, obj, path, keys, lo, hi):
i, n = 0, len(keys)
for x in keys:
if lo is not None and not lo <= x:
s = "key %r < lower bound %r at index %d" % (x, lo, i)
self.complain(s, obj, path)
if hi is not None and not x < hi:
s = "key %r >= upper bound %r at index %d" % (x, hi, i)
self.complain(s, obj, path)
if i < n-1 and not x < keys[i+1]:
s = "key %r at index %d >= key %r at index %d" % (
x, i, keys[i+1], i+1)
self.complain(s, obj, path)
i += 1
def complain(self, msg, obj, path):
s = "%s, in %s, path from root %s" % (
msg,
type_and_adr(obj),
".".join(map(str, path)))
self.errors.append(s)
class Printer(Walker):
def __init__(self, obj):
Walker.__init__(self, obj)
def display(self):
self.walk()
def visit_btree(self, obj, path, parent, is_mapping,
keys, kids, lo, hi):
indent = " " * len(path)
print "%s%s %s with %d children" % (
indent,
".".join(map(str, path)),
type_and_adr(obj),
len(kids))
indent += " "
n = len(keys)
for i in range(n):
print "%skey %d: %r" % (indent, i, keys[i])
def visit_bucket(self, obj, path, parent, is_mapping,
keys, values, lo, hi):
indent = " " * len(path)
print "%s%s %s with %d keys" % (
indent,
".".join(map(str, path)),
type_and_adr(obj),
len(keys))
indent += " "
n = len(keys)
for i in range(n):
print "%skey %d: %r" % (indent, i, keys[i]),
if is_mapping:
print "value %r" % (values[i],)
def check(btree):
"""Check internal value-based invariants in a BTree or TreeSet.
The btree._check() method checks internal C-level pointer consistency.
The check() function here checks value-based invariants: whether the
keys in leaf bucket and internal nodes are in strictly increasing order,
and whether they all lie in their expected range. The latter is a subtle
invariant that can't be checked locally -- it requires propagating
range info down from the root of the tree, and modifying it at each
level for each child.
Raises AssertionError if anything is wrong, with a string detail
explaining the problems. The entire tree is checked before
AssertionError is raised, and the string detail may be large (depending
on how much went wrong).
"""
Checker(btree).check()
def display(btree):
"Display the internal structure of a BTree, Bucket, TreeSet or Set."
Printer(btree).display()
ZODB/src/BTrees/IFBTree.py 0000640 0001750 0000144 00000001307 10156136553 014735 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# hack to overcome dynamic-linking headache.
from _IFBTree import *
ZODB/src/BTrees/Interfaces.py 0000640 0001750 0000144 00000034350 10306151001 015562 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
from zope.interface import Interface
class ICollection(Interface):
def clear():
"""Remove all of the items from the collection."""
def __nonzero__():
"""Check if the collection is non-empty.
Return a true value if the collection is non-empty and a
false value otherwise.
"""
class IReadSequence(Interface):
def __getitem__(index):
"""Return the value at the given index.
An IndexError is raised if the index cannot be found.
"""
def __getslice__(index1, index2):
"""Return a subsequence from the original sequence.
The subsequence includes the items from index1 up to, but not
including, index2.
"""
class IKeyed(ICollection):
def has_key(key):
"""Check whether the object has an item with the given key.
Return a true value if the key is present, else a false value.
"""
def keys(min=None, max=None, excludemin=False, excludemax=False):
"""Return an IReadSequence containing the keys in the collection.
The type of the IReadSequence is not specified. It could be a list
or a tuple or some other type.
All arguments are optional, and may be specified as keyword
arguments, or by position.
If a min is specified, then output is constrained to keys greater
than or equal to the given min, and, if excludemin is specified and
true, is further constrained to keys strictly greater than min. A
min value of None is ignored. If min is None or not specified, and
excludemin is true, the smallest key is excluded.
If a max is specified, then output is constrained to keys less than
or equal to the given max, and, if excludemax is specified and
true, is further constrained to keys strictly less than max. A max
value of None is ignored. If max is None or not specified, and
excludemax is true, the largest key is excluded.
"""
def maxKey(key=None):
"""Return the maximum key.
If a key argument if provided and not None, return the largest key
that is less than or equal to the argument. Raise an exception if
no such key exists.
"""
def minKey(key=None):
"""Return the minimum key.
If a key argument if provided and not None, return the smallest key
that is greater than or equal to the argument. Raise an exception
if no such key exists.
"""
class ISetMutable(IKeyed):
def insert(key):
"""Add the key (value) to the set.
If the key was already in the set, return 0, otherwise return 1.
"""
def remove(key):
"""Remove the key from the set."""
def update(seq):
"""Add the items from the given sequence to the set."""
class ISized(Interface):
"""An object that supports __len__."""
def __len__():
"""Return the number of items in the container."""
class IKeySequence(IKeyed, ISized):
def __getitem__(index):
"""Return the key in the given index position.
This allows iteration with for loops and use in functions,
like map and list, that read sequences.
"""
class ISet(IKeySequence, ISetMutable):
pass
class ITreeSet(IKeyed, ISetMutable):
pass
class IMinimalDictionary(ISized, IKeyed):
def get(key, default):
"""Get the value associated with the given key.
Return the default if has_key(key) is false.
"""
def __setitem__(key, value):
"""Set the value associated with the given key."""
def __delitem__(key):
"""Delete the value associated with the given key.
Raise KeyError if the key if has_key(key) is false.
"""
def values(min=None, max=None, excludemin=False, excludemax=False):
"""Return an IReadSequence containing the values in the collection.
The type of the IReadSequence is not specified. It could be a list
or a tuple or some other type.
All arguments are optional, and may be specified as keyword
arguments, or by position.
If a min is specified, then output is constrained to values whose
keys are greater than or equal to the given min, and, if excludemin
is specified and true, is further constrained to values whose keys
are strictly greater than min. A min value of None is ignored. If
min is None or not specified, and excludemin is true, the value
corresponding to the smallest key is excluded.
If a max is specified, then output is constrained to values whose
keys are less than or equal to the given max, and, if excludemax is
specified and true, is further constrained to values whose keys are
strictly less than max. A max value of None is ignored. If max is
None or not specified, and excludemax is true, the value
corresponding to the largest key is excluded.
"""
def items(min=None, max=None, excludemin=False, excludemax=False):
"""Return an IReadSequence containing the items in the collection.
An item is a 2-tuple, a (key, value) pair.
The type of the IReadSequence is not specified. It could be a list
or a tuple or some other type.
All arguments are optional, and may be specified as keyword
arguments, or by position.
If a min is specified, then output is constrained to items whose
keys are greater than or equal to the given min, and, if excludemin
is specified and true, is further constrained to items whose keys
are strictly greater than min. A min value of None is ignored. If
min is None or not specified, and excludemin is true, the item with
the smallest key is excluded.
If a max is specified, then output is constrained to items whose
keys are less than or equal to the given max, and, if excludemax is
specified and true, is further constrained to items whose keys are
strictly less than max. A max value of None is ignored. If max is
None or not specified, and excludemax is true, the item with the
largest key is excluded.
"""
class IDictionaryIsh(IMinimalDictionary):
def update(collection):
"""Add the items from the given collection object to the collection.
The input collection must be a sequence of (key, value) 2-tuples,
or an object with an 'items' method that returns a sequence of
(key, value) pairs.
"""
def byValue(minValue):
"""Return a sequence of (value, key) pairs, sorted by value.
Values < minValue are omitted and other values are "normalized" by
the minimum value. This normalization may be a noop, but, for
integer values, the normalization is division.
"""
def setdefault(key, d):
"""D.setdefault(k, d) -> D.get(k, d), also set D[k]=d if k not in D.
Return the value like get() except that if key is missing, d is both
returned and inserted into the dictionary as the value of k.
Note that, unlike as for Python's dict.setdefault(), d is not
optional. Python defaults d to None, but that doesn't make sense
for mappings that can't have None as a value (for example, an
IIBTree can have only integers as values).
"""
def pop(key, d):
"""D.pop(k[, d]) -> v, remove key and return the corresponding value.
If key is not found, d is returned if given, otherwise KeyError is
raised.
"""
class IBTree(IDictionaryIsh):
def insert(key, value):
"""Insert a key and value into the collection.
If the key was already in the collection, then there is no
change and 0 is returned.
If the key was not already in the collection, then the item is
added and 1 is returned.
This method is here to allow one to generate random keys and
to insert and test whether the key was there in one operation.
A standard idiom for generating new keys will be::
key = generate_key()
while not t.insert(key, value):
key=generate_key()
"""
class IMerge(Interface):
"""Object with methods for merging sets, buckets, and trees.
These methods are supplied in modules that define collection
classes with particular key and value types. The operations apply
only to collections from the same module. For example, the
IIBTree.union can only be used with IIBTree.IIBTree,
IIBTree.IIBucket, IIBTree.IISet, and IIBTree.IITreeSet.
The implementing module has a value type. The IOBTree and OOBTree
modules have object value type. The IIBTree and OIBTree modules
have integer value types. Other modules may be defined in the
future that have other value types.
The individual types are classified into set (Set and TreeSet) and
mapping (Bucket and BTree) types.
"""
def difference(c1, c2):
"""Return the keys or items in c1 for which there is no key in c2.
If c1 is None, then None is returned. If c2 is None, then c1
is returned.
If neither c1 nor c2 is None, the output is a Set if c1 is a Set or
TreeSet, and is a Bucket if c1 is a Bucket or BTree.
"""
def union(c1, c2):
"""Compute the Union of c1 and c2.
If c1 is None, then c2 is returned, otherwise, if c2 is None,
then c1 is returned.
The output is a Set containing keys from the input
collections.
"""
def intersection(c1, c2):
"""Compute the intersection of c1 and c2.
If c1 is None, then c2 is returned, otherwise, if c2 is None,
then c1 is returned.
The output is a Set containing matching keys from the input
collections.
"""
class IIMerge(IMerge):
"""Merge collections with integer value type.
A primary intent is to support operations with no or integer
values, which are used as "scores" to rate indiviual keys. That
is, in this context, a BTree or Bucket is viewed as a set with
scored keys, using integer scores.
"""
def weightedUnion(c1, c2, weight1=1, weight2=1):
"""Compute the weighted union of c1 and c2.
If c1 and c2 are None, the output is (0, None).
If c1 is None and c2 is not None, the output is (weight2, c2).
If c1 is not None and c2 is None, the output is (weight1, c1).
Else, and hereafter, c1 is not None and c2 is not None.
If c1 and c2 are both sets, the output is 1 and the (unweighted)
union of the sets.
Else the output is 1 and a Bucket whose keys are the union of c1 and
c2's keys, and whose values are::
v1*weight1 + v2*weight2
where:
v1 is 0 if the key is not in c1
1 if the key is in c1 and c1 is a set
c1[key] if the key is in c1 and c1 is a mapping
v2 is 0 if the key is not in c2
1 if the key is in c2 and c2 is a set
c2[key] if the key is in c2 and c2 is a mapping
Note that c1 and c2 must be collections.
"""
def weightedIntersection(c1, c2, weight1=1, weight2=1):
"""Compute the weighted intersection of c1 and c2.
If c1 and c2 are None, the output is (0, None).
If c1 is None and c2 is not None, the output is (weight2, c2).
If c1 is not None and c2 is None, the output is (weight1, c1).
Else, and hereafter, c1 is not None and c2 is not None.
If c1 and c2 are both sets, the output is the sum of the weights
and the (unweighted) intersection of the sets.
Else the output is 1 and a Bucket whose keys are the intersection of
c1 and c2's keys, and whose values are::
v1*weight1 + v2*weight2
where:
v1 is 1 if c1 is a set
c1[key] if c1 is a mapping
v2 is 1 if c2 is a set
c2[key] if c2 is a mapping
Note that c1 and c2 must be collections.
"""
class IMergeIntegerKey(IMerge):
"""IMerge-able objects with integer keys.
Concretely, this means the types in IOBTree and IIBTree.
"""
def multiunion(seq):
"""Return union of (zero or more) integer sets, as an integer set.
seq is a sequence of objects each convertible to an integer set.
These objects are convertible to an integer set:
+ An integer, which is added to the union.
+ A Set or TreeSet from the same module (for example, an
IIBTree.TreeSet for IIBTree.multiunion()). The elements of the
set are added to the union.
+ A Bucket or BTree from the same module (for example, an
IOBTree.IOBTree for IOBTree.multiunion()). The keys of the
mapping are added to the union.
The union is returned as a Set from the same module (for example,
IIBTree.multiunion() returns an IIBTree.IISet).
The point to this method is that it can run much faster than
doing a sequence of two-input union() calls. Under the covers,
all the integers in all the inputs are sorted via a single
linear-time radix sort, then duplicates are removed in a second
linear-time pass.
"""
###############################################################
# IMPORTANT NOTE
#
# Getting the length of a BTree, TreeSet, or output of keys,
# values, or items of same is expensive. If you need to get the
# length, you need to maintain this separately.
#
# Eventually, I need to express this through the interfaces.
#
################################################################
ZODB/src/BTrees/Length.py 0000640 0001750 0000144 00000003433 10057366465 014747 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import persistent
class Length(persistent.Persistent):
"""BTree lengths are too expensive to compute
Objects that use BTrees need to keep track of lengths themselves.
This class provides an object for doing this.
As a bonus, the object support application-level conflict
resolution.
It is tempting to to assign length objects to __len__ attributes
to provide instance-specific __len__ methods. However, this no
longer works as expected, because new-style classes cache
class-defined slot methods (like __len__) in C type slots. Thus,
instance-define slot fillers are ignores.
"""
def __init__(self, v=0):
self.value = v
def __getstate__(self):
return self.value
def __setstate__(self, v):
self.value = v
def set(self, v):
self.value = v
def _p_resolveConflict(self, old, s1, s2):
return s1 + s2 - old
def _p_independent(self):
# My state doesn't depend on or materially effect the state of
# other objects.
return 1
def change(self, delta):
self.value += delta
def __call__(self, *args):
return self.value
ZODB/src/BTrees/_OIBTree.c 0000640 0001750 0000144 00000002117 10057366465 014706 0 ustar ctheune users /*############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
############################################################################*/
#define MASTER_ID "$Id: _OIBTree.c 25186 2004-06-02 15:07:33Z jim $\n"
/* OIBTree - object key, int value BTree
Implements a collection using object type keys
and int type values
*/
#define PERSISTENT
#define MOD_NAME_PREFIX "OI"
#define INITMODULE init_OIBTree
#define DEFAULT_MAX_BUCKET_SIZE 60
#define DEFAULT_MAX_BTREE_SIZE 250
#include "objectkeymacros.h"
#include "intvaluemacros.h"
#include "BTreeModuleTemplate.c"
ZODB/src/BTrees/_IOBTree.c 0000640 0001750 0000144 00000002117 10057366465 014706 0 ustar ctheune users /*############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
############################################################################*/
#define MASTER_ID "$Id: _IOBTree.c 25186 2004-06-02 15:07:33Z jim $\n"
/* IOBTree - int key, object value BTree
Implements a collection using int type keys
and object type values
*/
#define PERSISTENT
#define MOD_NAME_PREFIX "IO"
#define DEFAULT_MAX_BUCKET_SIZE 60
#define DEFAULT_MAX_BTREE_SIZE 500
#define INITMODULE init_IOBTree
#include "intkeymacros.h"
#include "objectvaluemacros.h"
#include "BTreeModuleTemplate.c"
ZODB/src/BTrees/_fsBTree.c 0000640 0001750 0000144 00000005336 10375443702 015006 0 ustar ctheune users /*############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
############################################################################*/
#define MASTER_ID "$Id: _fsBTree.c 41650 2006-02-17 22:01:38Z ctheune $\n"
/* fsBTree - FileStorage index BTree
This BTree implements a mapping from 2-character strings
to six-character strings. This allows us to efficiently store
a FileStorage index as a nested mapping of 6-character oid prefix
to mapping of 2-character oid suffix to 6-character (byte) file
positions.
*/
typedef unsigned char char2[2];
typedef unsigned char char6[6];
/* Setup template macros */
#define PERSISTENT
#define MOD_NAME_PREFIX "fs"
#define INITMODULE init_fsBTree
#define DEFAULT_MAX_BUCKET_SIZE 500
#define DEFAULT_MAX_BTREE_SIZE 500
/*#include "intkeymacros.h"*/
#define KEYMACROS_H "$Id: _fsBTree.c 41650 2006-02-17 22:01:38Z ctheune $\n"
#define KEY_TYPE char2
#undef KEY_TYPE_IS_PYOBJECT
#define KEY_CHECK(K) (PyString_Check(K) && PyString_GET_SIZE(K)==2)
#define TEST_KEY_SET_OR(V, K, T) if ( ( (V) = ((*(K) < *(T) || (*(K) == *(T) && (K)[1] < (T)[1])) ? -1 : ((*(K) == *(T) && (K)[1] == (T)[1]) ? 0 : 1)) ), 0 )
#define DECREF_KEY(KEY)
#define INCREF_KEY(k)
#define COPY_KEY(KEY, E) (*(KEY)=*(E), (KEY)[1]=(E)[1])
#define COPY_KEY_TO_OBJECT(O, K) O=PyString_FromStringAndSize((const char*)K,2)
#define COPY_KEY_FROM_ARG(TARGET, ARG, STATUS) \
if (KEY_CHECK(ARG)) memcpy(TARGET, PyString_AS_STRING(ARG), 2); else { \
PyErr_SetString(PyExc_TypeError, "expected two-character string key"); \
(STATUS)=0; }
/*#include "intvaluemacros.h"*/
#define VALUEMACROS_H "$Id: _fsBTree.c 41650 2006-02-17 22:01:38Z ctheune $\n"
#define VALUE_TYPE char6
#undef VALUE_TYPE_IS_PYOBJECT
#define TEST_VALUE(K, T) memcmp(K,T,6)
#define DECREF_VALUE(k)
#define INCREF_VALUE(k)
#define COPY_VALUE(V, E) (memcpy(V, E, 6))
#define COPY_VALUE_TO_OBJECT(O, K) O=PyString_FromStringAndSize((const char*)K,6)
#define COPY_VALUE_FROM_ARG(TARGET, ARG, STATUS) \
if ((PyString_Check(ARG) && PyString_GET_SIZE(ARG)==6)) \
memcpy(TARGET, PyString_AS_STRING(ARG), 6); else { \
PyErr_SetString(PyExc_TypeError, "expected six-character string key"); \
(STATUS)=0; }
#define NORMALIZE_VALUE(V, MIN)
#include "BTreeModuleTemplate.c"
ZODB/src/BTrees/objectkeymacros.h 0000640 0001750 0000144 00000000720 10051217333 016464 0 ustar ctheune users #define KEYMACROS_H "$Id: objectkeymacros.h 24682 2004-05-14 19:32:43Z tim_one $\n"
#define KEY_TYPE PyObject *
#define KEY_TYPE_IS_PYOBJECT
#define TEST_KEY_SET_OR(V, KEY, TARGET) if ( ( (V) = PyObject_Compare((KEY),(TARGET)) ), PyErr_Occurred() )
#define INCREF_KEY(k) Py_INCREF(k)
#define DECREF_KEY(KEY) Py_DECREF(KEY)
#define COPY_KEY(KEY, E) KEY=(E)
#define COPY_KEY_TO_OBJECT(O, K) O=(K); Py_INCREF(O)
#define COPY_KEY_FROM_ARG(TARGET, ARG, S) TARGET=(ARG)
ZODB/src/BTrees/IIBTree.py 0000640 0001750 0000144 00000001307 10057366465 014747 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# hack to overcome dynamic-linking headache.
from _IIBTree import *
ZODB/src/transaction/ 0000751 0001750 0000144 00000000000 10424632373 014302 5 ustar ctheune users ZODB/src/transaction/tests/ 0000751 0001750 0000144 00000000000 10424632373 015444 5 ustar ctheune users ZODB/src/transaction/tests/test_register_compat.py 0000640 0001750 0000144 00000007537 10232625353 022255 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Test backwards compatibility for resource managers using register().
The transaction package supports several different APIs for resource
managers. The original ZODB3 API was implemented by ZODB.Connection.
The Connection passed persistent objects to a Transaction's register()
method. It's possible that third-party code also used this API, hence
these tests that the code that adapts the old interface to the current
API works.
These tests use a TestConnection object that implements the old API.
They check that the right methods are called and in roughly the right
order.
Common cases
------------
First, check that a basic transaction commit works.
>>> cn = TestConnection()
>>> cn.register(Object())
>>> cn.register(Object())
>>> cn.register(Object())
>>> transaction.commit()
>>> len(cn.committed)
3
>>> len(cn.aborted)
0
>>> cn.calls
['begin', 'vote', 'finish']
Second, check that a basic transaction abort works. If the
application calls abort(), then the transaction never gets into the
two-phase commit. It just aborts each object.
>>> cn = TestConnection()
>>> cn.register(Object())
>>> cn.register(Object())
>>> cn.register(Object())
>>> transaction.abort()
>>> len(cn.committed)
0
>>> len(cn.aborted)
3
>>> cn.calls
[]
Error handling
--------------
The tricky part of the implementation is recovering from an error that
occurs during the two-phase commit. We override the commit() and
abort() methods of Object to cause errors during commit.
Note that the implementation uses lists internally, so that objects
are committed in the order they are registered. (In the presence of
multiple resource managers, objects from a single resource manager are
committed in order. I'm not sure if this is an accident of the
implementation or a feature that should be supported by any
implementation.)
The order of resource managers depends on sortKey().
>>> cn = TestConnection()
>>> cn.register(Object())
>>> cn.register(CommitError())
>>> cn.register(Object())
>>> transaction.commit()
Traceback (most recent call last):
...
RuntimeError: commit
>>> len(cn.committed)
1
>>> len(cn.aborted)
3
Clean up:
>>> transaction.abort()
"""
import transaction
class Object(object):
def commit(self):
pass
def abort(self):
pass
class CommitError(Object):
def commit(self):
raise RuntimeError("commit")
class AbortError(Object):
def abort(self):
raise RuntimeError("abort")
class BothError(CommitError, AbortError):
pass
class TestConnection:
def __init__(self):
self.committed = []
self.aborted = []
self.calls = []
def register(self, obj):
obj._p_jar = self
transaction.get().register(obj)
def sortKey(self):
return str(id(self))
def tpc_begin(self, txn):
self.calls.append("begin")
def tpc_vote(self, txn):
self.calls.append("vote")
def tpc_finish(self, txn):
self.calls.append("finish")
def tpc_abort(self, txn):
self.calls.append("abort")
def commit(self, obj, txn):
obj.commit()
self.committed.append(obj)
def abort(self, obj, txn):
obj.abort()
self.aborted.append(obj)
from zope.testing import doctest
def test_suite():
return doctest.DocTestSuite()
ZODB/src/transaction/tests/test_transaction.py 0000640 0001750 0000144 00000066477 10375443702 021427 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002, 2005 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Test transaction behavior for variety of cases.
I wrote these unittests to investigate some odd transaction
behavior when doing unittests of integrating non sub transaction
aware objects, and to insure proper txn behavior. these
tests test the transaction system independent of the rest of the
zodb.
you can see the method calls to a jar by passing the
keyword arg tracing to the modify method of a dataobject.
the value of the arg is a prefix used for tracing print calls
to that objects jar.
the number of times a jar method was called can be inspected
by looking at an attribute of the jar that is the method
name prefixed with a c (count/check).
i've included some tracing examples for tests that i thought
were illuminating as doc strings below.
TODO
add in tests for objects which are modified multiple times,
for example an object that gets modified in multiple sub txns.
$Id: test_transaction.py 41650 2006-02-17 22:01:38Z ctheune $
"""
import unittest
import warnings
import transaction
from ZODB.utils import positive_id
from ZODB.tests.warnhook import WarningsHook
# deprecated37 remove when subtransactions go away
# Don't complain about subtxns in these tests.
warnings.filterwarnings("ignore",
".*\nsubtransactions are deprecated",
DeprecationWarning, __name__)
class TransactionTests(unittest.TestCase):
def setUp(self):
mgr = self.transaction_manager = transaction.TransactionManager()
self.sub1 = DataObject(mgr)
self.sub2 = DataObject(mgr)
self.sub3 = DataObject(mgr)
self.nosub1 = DataObject(mgr, nost=1)
# basic tests with two sub trans jars
# really we only need one, so tests for
# sub1 should identical to tests for sub2
def testTransactionCommit(self):
self.sub1.modify()
self.sub2.modify()
self.transaction_manager.commit()
assert self.sub1._p_jar.ccommit_sub == 0
assert self.sub1._p_jar.ctpc_finish == 1
def testTransactionAbort(self):
self.sub1.modify()
self.sub2.modify()
self.transaction_manager.abort()
assert self.sub2._p_jar.cabort == 1
def testTransactionNote(self):
t = self.transaction_manager.get()
t.note('This is a note.')
self.assertEqual(t.description, 'This is a note.')
t.note('Another.')
self.assertEqual(t.description, 'This is a note.\n\nAnother.')
t.abort()
# repeat adding in a nonsub trans jars
def testNSJTransactionCommit(self):
self.nosub1.modify()
self.transaction_manager.commit()
assert self.nosub1._p_jar.ctpc_finish == 1
def testNSJTransactionAbort(self):
self.nosub1.modify()
self.transaction_manager.abort()
assert self.nosub1._p_jar.ctpc_finish == 0
assert self.nosub1._p_jar.cabort == 1
def BUGtestNSJSubTransactionCommitAbort(self):
"""
this reveals a bug in transaction.py
the nosub jar should not have tpc_finish
called on it till the containing txn
ends.
sub calling method commit
nosub calling method tpc_begin
sub calling method tpc_finish
nosub calling method tpc_finish
nosub calling method abort
sub calling method abort_sub
"""
self.sub1.modify(tracing='sub')
self.nosub1.modify(tracing='nosub')
self.transaction_manager.commit(1)
assert self.sub1._p_jar.ctpc_finish == 1
# bug, non sub trans jars are getting finished
# in a subtrans
assert self.nosub1._p_jar.ctpc_finish == 0
self.transaction_manager.abort()
assert self.nosub1._p_jar.cabort == 1
assert self.sub1._p_jar.cabort_sub == 1
### Failure Mode Tests
#
# ok now we do some more interesting
# tests that check the implementations
# error handling by throwing errors from
# various jar methods
###
# first the recoverable errors
def testExceptionInAbort(self):
self.sub1._p_jar = SubTransactionJar(errors='abort')
self.nosub1.modify()
self.sub1.modify(nojar=1)
self.sub2.modify()
try:
self.transaction_manager.abort()
except TestTxnException: pass
assert self.nosub1._p_jar.cabort == 1
assert self.sub2._p_jar.cabort == 1
def testExceptionInCommit(self):
self.sub1._p_jar = SubTransactionJar(errors='commit')
self.nosub1.modify()
self.sub1.modify(nojar=1)
try:
self.transaction_manager.commit()
except TestTxnException: pass
assert self.nosub1._p_jar.ctpc_finish == 0
assert self.nosub1._p_jar.ccommit == 1
assert self.nosub1._p_jar.ctpc_abort == 1
def testExceptionInTpcVote(self):
self.sub1._p_jar = SubTransactionJar(errors='tpc_vote')
self.nosub1.modify()
self.sub1.modify(nojar=1)
try:
self.transaction_manager.commit()
except TestTxnException: pass
assert self.nosub1._p_jar.ctpc_finish == 0
assert self.nosub1._p_jar.ccommit == 1
assert self.nosub1._p_jar.ctpc_abort == 1
assert self.sub1._p_jar.ctpc_abort == 1
def testExceptionInTpcBegin(self):
"""
ok this test reveals a bug in the TM.py
as the nosub tpc_abort there is ignored.
nosub calling method tpc_begin
nosub calling method commit
sub calling method tpc_begin
sub calling method abort
sub calling method tpc_abort
nosub calling method tpc_abort
"""
self.sub1._p_jar = SubTransactionJar(errors='tpc_begin')
self.nosub1.modify()
self.sub1.modify(nojar=1)
try:
self.transaction_manager.commit()
except TestTxnException:
pass
assert self.nosub1._p_jar.ctpc_abort == 1
assert self.sub1._p_jar.ctpc_abort == 1
def testExceptionInTpcAbort(self):
self.sub1._p_jar = SubTransactionJar(
errors=('tpc_abort', 'tpc_vote'))
self.nosub1.modify()
self.sub1.modify(nojar=1)
try:
self.transaction_manager.commit()
except TestTxnException:
pass
assert self.nosub1._p_jar.ctpc_abort == 1
# last test, check the hosing mechanism
## def testHoserStoppage(self):
## # It's hard to test the "hosed" state of the database, where
## # hosed means that a failure occurred in the second phase of
## # the two phase commit. It's hard because the database can
## # recover from such an error if it occurs during the very first
## # tpc_finish() call of the second phase.
## for obj in self.sub1, self.sub2:
## j = HoserJar(errors='tpc_finish')
## j.reset()
## obj._p_jar = j
## obj.modify(nojar=1)
## try:
## transaction.commit()
## except TestTxnException:
## pass
## self.assert_(Transaction.hosed)
## self.sub2.modify()
## try:
## transaction.commit()
## except Transaction.POSException.TransactionError:
## pass
## else:
## self.fail("Hosed Application didn't stop commits")
class DataObject:
def __init__(self, transaction_manager, nost=0):
self.transaction_manager = transaction_manager
self.nost = nost
self._p_jar = None
def modify(self, nojar=0, tracing=0):
if not nojar:
if self.nost:
self._p_jar = NoSubTransactionJar(tracing=tracing)
else:
self._p_jar = SubTransactionJar(tracing=tracing)
self.transaction_manager.get().join(self._p_jar)
class TestTxnException(Exception):
pass
class BasicJar:
def __init__(self, errors=(), tracing=0):
if not isinstance(errors, tuple):
errors = errors,
self.errors = errors
self.tracing = tracing
self.cabort = 0
self.ccommit = 0
self.ctpc_begin = 0
self.ctpc_abort = 0
self.ctpc_vote = 0
self.ctpc_finish = 0
self.cabort_sub = 0
self.ccommit_sub = 0
def __repr__(self):
return "<%s %X %s>" % (self.__class__.__name__,
positive_id(self),
self.errors)
def sortKey(self):
# All these jars use the same sort key, and Python's list.sort()
# is stable. These two
return self.__class__.__name__
def check(self, method):
if self.tracing:
print '%s calling method %s'%(str(self.tracing),method)
if method in self.errors:
raise TestTxnException("error %s" % method)
## basic jar txn interface
def abort(self, *args):
self.check('abort')
self.cabort += 1
def commit(self, *args):
self.check('commit')
self.ccommit += 1
def tpc_begin(self, txn, sub=0):
self.check('tpc_begin')
self.ctpc_begin += 1
def tpc_vote(self, *args):
self.check('tpc_vote')
self.ctpc_vote += 1
def tpc_abort(self, *args):
self.check('tpc_abort')
self.ctpc_abort += 1
def tpc_finish(self, *args):
self.check('tpc_finish')
self.ctpc_finish += 1
class SubTransactionJar(BasicJar):
def abort_sub(self, txn):
self.check('abort_sub')
self.cabort_sub = 1
def commit_sub(self, txn):
self.check('commit_sub')
self.ccommit_sub = 1
class NoSubTransactionJar(BasicJar):
pass
class HoserJar(BasicJar):
# The HoserJars coordinate their actions via the class variable
# committed. The check() method will only raise its exception
# if committed > 0.
committed = 0
def reset(self):
# Calling reset() on any instance will reset the class variable.
HoserJar.committed = 0
def check(self, method):
if HoserJar.committed > 0:
BasicJar.check(self, method)
def tpc_finish(self, *args):
self.check('tpc_finish')
self.ctpc_finish += 1
HoserJar.committed += 1
def test_join():
"""White-box test of the join method
The join method is provided for "backward-compatability" with ZODB 4
data managers.
The argument to join must be a zodb4 data manager,
transaction.interfaces.IDataManager.
>>> from ZODB.tests.sampledm import DataManager
>>> from transaction._transaction import DataManagerAdapter
>>> t = transaction.Transaction()
>>> dm = DataManager()
>>> t.join(dm)
The end result is that a data manager adapter is one of the
transaction's objects:
>>> isinstance(t._resources[0], DataManagerAdapter)
True
>>> t._resources[0]._datamanager is dm
True
"""
def hook():
pass
# deprecated38; remove this then
def test_beforeCommitHook():
"""Test beforeCommitHook.
Let's define a hook to call, and a way to see that it was called.
>>> log = []
>>> def reset_log():
... del log[:]
>>> def hook(arg='no_arg', kw1='no_kw1', kw2='no_kw2'):
... log.append("arg %r kw1 %r kw2 %r" % (arg, kw1, kw2))
beforeCommitHook is deprecated, so we need cruft to suppress the
warnings.
>>> whook = WarningsHook()
>>> whook.install()
Fool the warnings module into delivering the warnings despite that
they've been seen before; this is needed in case this test is run
more than once.
>>> import warnings
>>> warnings.filterwarnings("always", category=DeprecationWarning)
Now register the hook with a transaction.
>>> import transaction
>>> t = transaction.begin()
>>> t.beforeCommitHook(hook, '1')
Make sure it triggered a deprecation warning:
>>> len(whook.warnings)
1
>>> message, category, filename, lineno = whook.warnings[0]
>>> print message
This will be removed in ZODB 3.8:
Use addBeforeCommitHook instead of beforeCommitHook.
>>> category.__name__
'DeprecationWarning'
>>> whook.clear()
We can see that the hook is indeed registered.
>>> [(hook.func_name, args, kws)
... for hook, args, kws in t.getBeforeCommitHooks()]
[('hook', ('1',), {})]
When transaction commit starts, the hook is called, with its
arguments.
>>> log
[]
>>> t.commit()
>>> log
["arg '1' kw1 'no_kw1' kw2 'no_kw2'"]
>>> reset_log()
A hook's registration is consumed whenever the hook is called. Since
the hook above was called, it's no longer registered:
>>> len(list(t.getBeforeCommitHooks()))
0
>>> transaction.commit()
>>> log
[]
The hook is only called for a full commit, not for a savepoint or
subtransaction.
>>> t = transaction.begin()
>>> t.beforeCommitHook(hook, 'A', kw1='B')
>>> dummy = t.savepoint()
>>> log
[]
>>> t.commit(subtransaction=True)
>>> log
[]
>>> t.commit()
>>> log
["arg 'A' kw1 'B' kw2 'no_kw2'"]
>>> reset_log()
If a transaction is aborted, no hook is called.
>>> t = transaction.begin()
>>> t.beforeCommitHook(hook, "OOPS!")
>>> transaction.abort()
>>> log
[]
>>> transaction.commit()
>>> log
[]
The hook is called before the commit does anything, so even if the
commit fails the hook will have been called. To provoke failures in
commit, we'll add failing resource manager to the transaction.
>>> class CommitFailure(Exception):
... pass
>>> class FailingDataManager:
... def tpc_begin(self, txn, sub=False):
... raise CommitFailure
... def abort(self, txn):
... pass
>>> t = transaction.begin()
>>> t.join(FailingDataManager())
>>> t.beforeCommitHook(hook, '2')
>>> t.commit()
Traceback (most recent call last):
...
CommitFailure
>>> log
["arg '2' kw1 'no_kw1' kw2 'no_kw2'"]
>>> reset_log()
Let's register several hooks.
>>> t = transaction.begin()
>>> t.beforeCommitHook(hook, '4', kw1='4.1')
>>> t.beforeCommitHook(hook, '5', kw2='5.2')
They are returned in the same order by getBeforeCommitHooks.
>>> [(hook.func_name, args, kws) #doctest: +NORMALIZE_WHITESPACE
... for hook, args, kws in t.getBeforeCommitHooks()]
[('hook', ('4',), {'kw1': '4.1'}),
('hook', ('5',), {'kw2': '5.2'})]
And commit also calls them in this order.
>>> t.commit()
>>> len(log)
2
>>> log #doctest: +NORMALIZE_WHITESPACE
["arg '4' kw1 '4.1' kw2 'no_kw2'",
"arg '5' kw1 'no_kw1' kw2 '5.2'"]
>>> reset_log()
While executing, a hook can itself add more hooks, and they will all
be called before the real commit starts.
>>> def recurse(txn, arg):
... log.append('rec' + str(arg))
... if arg:
... txn.beforeCommitHook(hook, '-')
... txn.beforeCommitHook(recurse, txn, arg-1)
>>> t = transaction.begin()
>>> t.beforeCommitHook(recurse, t, 3)
>>> transaction.commit()
>>> log #doctest: +NORMALIZE_WHITESPACE
['rec3',
"arg '-' kw1 'no_kw1' kw2 'no_kw2'",
'rec2',
"arg '-' kw1 'no_kw1' kw2 'no_kw2'",
'rec1',
"arg '-' kw1 'no_kw1' kw2 'no_kw2'",
'rec0']
>>> reset_log()
We have to uninstall the warnings hook so that other warnings don't get
lost.
>>> whook.uninstall()
Obscure: There is no API call for removing the filter we added, but
filters appears to be a public variable.
>>> del warnings.filters[0]
"""
def test_addBeforeCommitHook():
"""Test addBeforeCommitHook.
Let's define a hook to call, and a way to see that it was called.
>>> log = []
>>> def reset_log():
... del log[:]
>>> def hook(arg='no_arg', kw1='no_kw1', kw2='no_kw2'):
... log.append("arg %r kw1 %r kw2 %r" % (arg, kw1, kw2))
Now register the hook with a transaction.
>>> import transaction
>>> t = transaction.begin()
>>> t.addBeforeCommitHook(hook, '1')
We can see that the hook is indeed registered.
>>> [(hook.func_name, args, kws)
... for hook, args, kws in t.getBeforeCommitHooks()]
[('hook', ('1',), {})]
When transaction commit starts, the hook is called, with its
arguments.
>>> log
[]
>>> t.commit()
>>> log
["arg '1' kw1 'no_kw1' kw2 'no_kw2'"]
>>> reset_log()
A hook's registration is consumed whenever the hook is called. Since
the hook above was called, it's no longer registered:
>>> len(list(t.getBeforeCommitHooks()))
0
>>> transaction.commit()
>>> log
[]
The hook is only called for a full commit, not for a savepoint or
subtransaction.
>>> t = transaction.begin()
>>> t.addBeforeCommitHook(hook, 'A', dict(kw1='B'))
>>> dummy = t.savepoint()
>>> log
[]
>>> t.commit(subtransaction=True)
>>> log
[]
>>> t.commit()
>>> log
["arg 'A' kw1 'B' kw2 'no_kw2'"]
>>> reset_log()
If a transaction is aborted, no hook is called.
>>> t = transaction.begin()
>>> t.addBeforeCommitHook(hook, ["OOPS!"])
>>> transaction.abort()
>>> log
[]
>>> transaction.commit()
>>> log
[]
The hook is called before the commit does anything, so even if the
commit fails the hook will have been called. To provoke failures in
commit, we'll add failing resource manager to the transaction.
>>> class CommitFailure(Exception):
... pass
>>> class FailingDataManager:
... def tpc_begin(self, txn, sub=False):
... raise CommitFailure
... def abort(self, txn):
... pass
>>> t = transaction.begin()
>>> t.join(FailingDataManager())
>>> t.addBeforeCommitHook(hook, '2')
>>> t.commit()
Traceback (most recent call last):
...
CommitFailure
>>> log
["arg '2' kw1 'no_kw1' kw2 'no_kw2'"]
>>> reset_log()
Let's register several hooks.
>>> t = transaction.begin()
>>> t.addBeforeCommitHook(hook, '4', dict(kw1='4.1'))
>>> t.addBeforeCommitHook(hook, '5', dict(kw2='5.2'))
They are returned in the same order by getBeforeCommitHooks.
>>> [(hook.func_name, args, kws) #doctest: +NORMALIZE_WHITESPACE
... for hook, args, kws in t.getBeforeCommitHooks()]
[('hook', ('4',), {'kw1': '4.1'}),
('hook', ('5',), {'kw2': '5.2'})]
And commit also calls them in this order.
>>> t.commit()
>>> len(log)
2
>>> log #doctest: +NORMALIZE_WHITESPACE
["arg '4' kw1 '4.1' kw2 'no_kw2'",
"arg '5' kw1 'no_kw1' kw2 '5.2'"]
>>> reset_log()
While executing, a hook can itself add more hooks, and they will all
be called before the real commit starts.
>>> def recurse(txn, arg):
... log.append('rec' + str(arg))
... if arg:
... txn.addBeforeCommitHook(hook, '-')
... txn.addBeforeCommitHook(recurse, (txn, arg-1))
>>> t = transaction.begin()
>>> t.addBeforeCommitHook(recurse, (t, 3))
>>> transaction.commit()
>>> log #doctest: +NORMALIZE_WHITESPACE
['rec3',
"arg '-' kw1 'no_kw1' kw2 'no_kw2'",
'rec2',
"arg '-' kw1 'no_kw1' kw2 'no_kw2'",
'rec1',
"arg '-' kw1 'no_kw1' kw2 'no_kw2'",
'rec0']
>>> reset_log()
When modifing persitent objects within before commit hooks
modifies the objects, of course :)
Start a new transaction
>>> t = transaction.begin()
Create a DB instance and add a IOBTree within
>>> from ZODB.tests.util import DB
>>> from ZODB.tests.util import P
>>> db = DB()
>>> con = db.open()
>>> root = con.root()
>>> root['p'] = P('julien')
>>> p = root['p']
>>> p.name
'julien'
This hook will get the object from the `DB` instance and change
the flag attribute.
>>> def hookmodify(status, arg=None, kw1='no_kw1', kw2='no_kw2'):
... p.name = 'jul'
Now register this hook and commit.
>>> t.addBeforeCommitHook(hookmodify, (p, 1))
>>> transaction.commit()
Nothing should have changed since it should have been aborted.
>>> p.name
'jul'
>>> db.close()
"""
def test_addAfterCommitHook():
"""Test addAfterCommitHook.
Let's define a hook to call, and a way to see that it was called.
>>> log = []
>>> def reset_log():
... del log[:]
>>> def hook(status, arg='no_arg', kw1='no_kw1', kw2='no_kw2'):
... log.append("%r arg %r kw1 %r kw2 %r" % (status, arg, kw1, kw2))
Now register the hook with a transaction.
>>> import transaction
>>> t = transaction.begin()
>>> t.addAfterCommitHook(hook, '1')
We can see that the hook is indeed registered.
>>> [(hook.func_name, args, kws)
... for hook, args, kws in t.getAfterCommitHooks()]
[('hook', ('1',), {})]
When transaction commit is done, the hook is called, with its
arguments.
>>> log
[]
>>> t.commit()
>>> log
["True arg '1' kw1 'no_kw1' kw2 'no_kw2'"]
>>> reset_log()
A hook's registration is consumed whenever the hook is called. Since
the hook above was called, it's no longer registered:
>>> len(list(t.getAfterCommitHooks()))
0
>>> transaction.commit()
>>> log
[]
The hook is only called after a full commit, not for a savepoint or
subtransaction.
>>> t = transaction.begin()
>>> t.addAfterCommitHook(hook, 'A', dict(kw1='B'))
>>> dummy = t.savepoint()
>>> log
[]
>>> t.commit(subtransaction=True)
>>> log
[]
>>> t.commit()
>>> log
["True arg 'A' kw1 'B' kw2 'no_kw2'"]
>>> reset_log()
If a transaction is aborted, no hook is called.
>>> t = transaction.begin()
>>> t.addAfterCommitHook(hook, ["OOPS!"])
>>> transaction.abort()
>>> log
[]
>>> transaction.commit()
>>> log
[]
The hook is called after the commit is done, so even if the
commit fails the hook will have been called. To provoke failures in
commit, we'll add failing resource manager to the transaction.
>>> class CommitFailure(Exception):
... pass
>>> class FailingDataManager:
... def tpc_begin(self, txn, sub=False):
... raise CommitFailure
... def abort(self, txn):
... pass
>>> t = transaction.begin()
>>> t.join(FailingDataManager())
>>> t.addAfterCommitHook(hook, '2')
>>> t.commit()
Traceback (most recent call last):
...
CommitFailure
>>> log
["False arg '2' kw1 'no_kw1' kw2 'no_kw2'"]
>>> reset_log()
Let's register several hooks.
>>> t = transaction.begin()
>>> t.addAfterCommitHook(hook, '4', dict(kw1='4.1'))
>>> t.addAfterCommitHook(hook, '5', dict(kw2='5.2'))
They are returned in the same order by getAfterCommitHooks.
>>> [(hook.func_name, args, kws) #doctest: +NORMALIZE_WHITESPACE
... for hook, args, kws in t.getAfterCommitHooks()]
[('hook', ('4',), {'kw1': '4.1'}),
('hook', ('5',), {'kw2': '5.2'})]
And commit also calls them in this order.
>>> t.commit()
>>> len(log)
2
>>> log #doctest: +NORMALIZE_WHITESPACE
["True arg '4' kw1 '4.1' kw2 'no_kw2'",
"True arg '5' kw1 'no_kw1' kw2 '5.2'"]
>>> reset_log()
While executing, a hook can itself add more hooks, and they will all
be called before the real commit starts.
>>> def recurse(status, txn, arg):
... log.append('rec' + str(arg))
... if arg:
... txn.addAfterCommitHook(hook, '-')
... txn.addAfterCommitHook(recurse, (txn, arg-1))
>>> t = transaction.begin()
>>> t.addAfterCommitHook(recurse, (t, 3))
>>> transaction.commit()
>>> log #doctest: +NORMALIZE_WHITESPACE
['rec3',
"True arg '-' kw1 'no_kw1' kw2 'no_kw2'",
'rec2',
"True arg '-' kw1 'no_kw1' kw2 'no_kw2'",
'rec1',
"True arg '-' kw1 'no_kw1' kw2 'no_kw2'",
'rec0']
>>> reset_log()
If an after commit hook is raising an exception then it will log a
message at error level so that if other hooks are registered they
can be executed. We don't support execution dependencies at this level.
>>> mgr = transaction.TransactionManager()
>>> do = DataObject(mgr)
>>> def hookRaise(status, arg='no_arg', kw1='no_kw1', kw2='no_kw2'):
... raise TypeError("Fake raise")
>>> t = transaction.begin()
>>> t.addAfterCommitHook(hook, ('-', 1))
>>> t.addAfterCommitHook(hookRaise, ('-', 2))
>>> t.addAfterCommitHook(hook, ('-', 3))
>>> transaction.commit()
>>> log
["True arg '-' kw1 1 kw2 'no_kw2'", "True arg '-' kw1 3 kw2 'no_kw2'"]
>>> reset_log()
Test that the associated transaction manager has been cleanup when
after commit hooks are registered
>>> mgr = transaction.TransactionManager()
>>> do = DataObject(mgr)
>>> t = transaction.begin()
>>> len(t._manager._txns)
1
>>> t.addAfterCommitHook(hook, ('-', 1))
>>> transaction.commit()
>>> log
["True arg '-' kw1 1 kw2 'no_kw2'"]
>>> len(t._manager._txns)
0
>>> reset_log()
The transaction is already committed when the after commit hooks
will be executed. Executing the hooks must not have further
effects on persistent objects.
Start a new transaction
>>> t = transaction.begin()
Create a DB instance and add a IOBTree within
>>> from ZODB.tests.util import DB
>>> from ZODB.tests.util import P
>>> db = DB()
>>> con = db.open()
>>> root = con.root()
>>> root['p'] = P('julien')
>>> p = root['p']
>>> p.name
'julien'
This hook will get the object from the `DB` instance and change
the flag attribute.
>>> def badhook(status, arg=None, kw1='no_kw1', kw2='no_kw2'):
... p.name = 'jul'
Now register this hook and commit.
>>> t.addAfterCommitHook(badhook, (p, 1))
>>> transaction.commit()
Nothing should have changed since it should have been aborted.
>>> p.name
'julien'
>>> db.close()
"""
def test_suite():
from zope.testing.doctest import DocTestSuite
return unittest.TestSuite((
DocTestSuite(),
unittest.makeSuite(TransactionTests),
))
if __name__ == '__main__':
unittest.TextTestRunner().run(test_suite())
ZODB/src/transaction/tests/savepointsample.py 0000640 0001750 0000144 00000015634 10264566555 021253 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Savepoint data manager implementation example.
Sample data manager implementation that illustrates how to implement
savepoints.
See savepoint.txt in the transaction package.
$Id: savepointsample.py 33274 2005-07-11 22:06:37Z tim_one $
"""
import UserDict
from zope import interface
import transaction.interfaces
class SampleDataManager(UserDict.DictMixin):
"""Sample implementation of data manager that doesn't support savepoints
This data manager stores named simple values, like strings and numbers.
"""
interface.implements(transaction.interfaces.IDataManager)
def __init__(self, transaction_manager=None):
if transaction_manager is None:
# Use the thread-local transaction manager if none is provided:
transaction_manager = transaction.manager
self.transaction_manager = transaction_manager
# Our committed and uncommitted data:
self.committed = {}
self.uncommitted = self.committed.copy()
# Our transaction state:
#
# If our uncommitted data is modified, we'll join a transaction
# and keep track of the transaction we joined. Any commit
# related messages we get should be for this same transaction
self.transaction = None
# What phase, if any, of two-phase commit we are in:
self.tpc_phase = None
#######################################################################
# Provide a mapping interface to uncommitted data. We provide
# a basic subset of the interface. DictMixin does the rest.
def __getitem__(self, name):
return self.uncommitted[name]
def __setitem__(self, name, value):
self._join() # join the current transaction, if we haven't already
self.uncommitted[name] = value
def __delitem__(self, name):
self._join() # join the current transaction, if we haven't already
del self.uncommitted[name]
def keys(self):
return self.uncommitted.keys()
#
#######################################################################
#######################################################################
# Transaction methods
def _join(self):
# If this is the first change in the transaction, join the transaction
if self.transaction is None:
self.transaction = self.transaction_manager.get()
self.transaction.join(self)
def _resetTransaction(self):
self.transaction = None
self.tpc_phase = None
def abort(self, transaction):
"""Throw away changes made before the commit process has started
"""
assert ((transaction is self.transaction) or (self.transaction is None)
), "Must not change transactions"
assert self.tpc_phase is None, "Must be called outside of tpc"
self.uncommitted = self.committed.copy()
self._resetTransaction()
def tpc_begin(self, transaction):
"""Enter two-phase commit
"""
assert transaction is self.transaction, "Must not change transactions"
assert self.tpc_phase is None, "Must be called outside of tpc"
self.tpc_phase = 1
def commit(self, transaction):
"""Record data modified during the transaction
"""
assert transaction is self.transaction, "Must not change transactions"
assert self.tpc_phase == 1, "Must be called in first phase of tpc"
# In our simple example, we don't need to do anything.
# A more complex data manager would typically write to some sort
# of log.
def tpc_vote(self, transaction):
assert transaction is self.transaction, "Must not change transactions"
assert self.tpc_phase == 1, "Must be called in first phase of tpc"
# This particular data manager is always ready to vote.
# Real data managers will usually need to take some steps to
# make sure that the finish will succeed
self.tpc_phase = 2
def tpc_finish(self, transaction):
assert transaction is self.transaction, "Must not change transactions"
assert self.tpc_phase == 2, "Must be called in second phase of tpc"
self.committed = self.uncommitted.copy()
self._resetTransaction()
def tpc_abort(self, transaction):
assert transaction is self.transaction, "Must not change transactions"
assert self.tpc_phase is not None, "Must be called inside of tpc"
self.uncommitted = self.committed.copy()
self._resetTransaction()
#
#######################################################################
#######################################################################
# Other data manager methods
def sortKey(self):
# Commit operations on multiple data managers are performed in
# sort key order. This important to avoid deadlock when data
# managers are shared among multiple threads or processes and
# use locks to manage that sharing. We aren't going to bother
# with that here.
return str(id(self))
#
#######################################################################
class SampleSavepointDataManager(SampleDataManager):
"""Sample implementation of a savepoint-supporting data manager
This extends the basic data manager with savepoint support.
"""
interface.implements(transaction.interfaces.ISavepointDataManager)
def savepoint(self):
# When we create the savepoint, we save the existing database state.
return SampleSavepoint(self, self.uncommitted.copy())
def _rollback_savepoint(self, savepoint):
# When we rollback the savepoint, we restore the saved data.
# Caution: without the copy(), further changes to the database
# could reflect in savepoint.data, and then `savepoint` would no
# longer contain the originally saved data, and so `savepoint`
# couldn't restore the original state if a rollback to this
# savepoint was done again. IOW, copy() is necessary.
self.uncommitted = savepoint.data.copy()
class SampleSavepoint:
interface.implements(transaction.interfaces.IDataManagerSavepoint)
def __init__(self, data_manager, data):
self.data_manager = data_manager
self.data = data
def rollback(self):
self.data_manager._rollback_savepoint(self)
ZODB/src/transaction/tests/__init__.py 0000640 0001750 0000144 00000000002 10051215041 017526 0 ustar ctheune users #
ZODB/src/transaction/tests/test_SampleDataManager.py 0000640 0001750 0000144 00000025107 10225135606 022364 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Sample objects for use in tests
$Id: test_SampleDataManager.py 29896 2005-04-07 04:48:06Z tim_one $
"""
class DataManager(object):
"""Sample data manager
This class provides a trivial data-manager implementation and doc
strings to illustrate the the protocol and to provide a tool for
writing tests.
Our sample data manager has state that is updated through an inc
method and through transaction operations.
When we create a sample data manager:
>>> dm = DataManager()
It has two bits of state, state:
>>> dm.state
0
and delta:
>>> dm.delta
0
Both of which are initialized to 0. state is meant to model
committed state, while delta represents tentative changes within a
transaction. We change the state by calling inc:
>>> dm.inc()
which updates delta:
>>> dm.delta
1
but state isn't changed until we commit the transaction:
>>> dm.state
0
To commit the changes, we use 2-phase commit. We execute the first
stage by calling prepare. We need to pass a transation. Our
sample data managers don't really use the transactions for much,
so we'll be lazy and use strings for transactions:
>>> t1 = '1'
>>> dm.prepare(t1)
The sample data manager updates the state when we call prepare:
>>> dm.state
1
>>> dm.delta
1
This is mainly so we can detect some affect of calling the methods.
Now if we call commit:
>>> dm.commit(t1)
Our changes are"permanent". The state reflects the changes and the
delta has been reset to 0.
>>> dm.state
1
>>> dm.delta
0
"""
def __init__(self):
self.state = 0
self.sp = 0
self.transaction = None
self.delta = 0
self.prepared = False
def inc(self, n=1):
self.delta += n
def prepare(self, transaction):
"""Prepare to commit data
>>> dm = DataManager()
>>> dm.inc()
>>> t1 = '1'
>>> dm.prepare(t1)
>>> dm.commit(t1)
>>> dm.state
1
>>> dm.inc()
>>> t2 = '2'
>>> dm.prepare(t2)
>>> dm.abort(t2)
>>> dm.state
1
It is en error to call prepare more than once without an intervening
commit or abort:
>>> dm.prepare(t1)
>>> dm.prepare(t1)
Traceback (most recent call last):
...
TypeError: Already prepared
>>> dm.prepare(t2)
Traceback (most recent call last):
...
TypeError: Already prepared
>>> dm.abort(t1)
If there was a preceeding savepoint, the transaction must match:
>>> rollback = dm.savepoint(t1)
>>> dm.prepare(t2)
Traceback (most recent call last):
,,,
TypeError: ('Transaction missmatch', '2', '1')
>>> dm.prepare(t1)
"""
if self.prepared:
raise TypeError('Already prepared')
self._checkTransaction(transaction)
self.prepared = True
self.transaction = transaction
self.state += self.delta
def _checkTransaction(self, transaction):
if (transaction is not self.transaction
and self.transaction is not None):
raise TypeError("Transaction missmatch",
transaction, self.transaction)
def abort(self, transaction):
"""Abort a transaction
The abort method can be called before two-phase commit to
throw away work done in the transaction:
>>> dm = DataManager()
>>> dm.inc()
>>> dm.state, dm.delta
(0, 1)
>>> t1 = '1'
>>> dm.abort(t1)
>>> dm.state, dm.delta
(0, 0)
The abort method also throws away work done in savepoints:
>>> dm.inc()
>>> r = dm.savepoint(t1)
>>> dm.inc()
>>> r = dm.savepoint(t1)
>>> dm.state, dm.delta
(0, 2)
>>> dm.abort(t1)
>>> dm.state, dm.delta
(0, 0)
If savepoints are used, abort must be passed the same
transaction:
>>> dm.inc()
>>> r = dm.savepoint(t1)
>>> t2 = '2'
>>> dm.abort(t2)
Traceback (most recent call last):
...
TypeError: ('Transaction missmatch', '2', '1')
>>> dm.abort(t1)
The abort method is also used to abort a two-phase commit:
>>> dm.inc()
>>> dm.state, dm.delta
(0, 1)
>>> dm.prepare(t1)
>>> dm.state, dm.delta
(1, 1)
>>> dm.abort(t1)
>>> dm.state, dm.delta
(0, 0)
Of course, the transactions passed to prepare and abort must
match:
>>> dm.prepare(t1)
>>> dm.abort(t2)
Traceback (most recent call last):
...
TypeError: ('Transaction missmatch', '2', '1')
>>> dm.abort(t1)
"""
self._checkTransaction(transaction)
if self.transaction is not None:
self.transaction = None
if self.prepared:
self.state -= self.delta
self.prepared = False
self.delta = 0
def commit(self, transaction):
"""Complete two-phase commit
>>> dm = DataManager()
>>> dm.state
0
>>> dm.inc()
We start two-phase commit by calling prepare:
>>> t1 = '1'
>>> dm.prepare(t1)
We complete it by calling commit:
>>> dm.commit(t1)
>>> dm.state
1
It is an error ro call commit without calling prepare first:
>>> dm.inc()
>>> t2 = '2'
>>> dm.commit(t2)
Traceback (most recent call last):
...
TypeError: Not prepared to commit
>>> dm.prepare(t2)
>>> dm.commit(t2)
If course, the transactions given to prepare and commit must
be the same:
>>> dm.inc()
>>> t3 = '3'
>>> dm.prepare(t3)
>>> dm.commit(t2)
Traceback (most recent call last):
...
TypeError: ('Transaction missmatch', '2', '3')
"""
if not self.prepared:
raise TypeError('Not prepared to commit')
self._checkTransaction(transaction)
self.delta = 0
self.transaction = None
self.prepared = False
def savepoint(self, transaction):
"""Provide the ability to rollback transaction state
Savepoints provide a way to:
- Save partial transaction work. For some data managers, this
could allow resources to be used more efficiently.
- Provide the ability to revert state to a point in a
transaction without aborting the entire transaction. In
other words, savepoints support partial aborts.
Savepoints don't use two-phase commit. If there are errors in
setting or rolling back to savepoints, the application should
abort the containing transaction. This is *not* the
responsibility of the data manager.
Savepoints are always associated with a transaction. Any work
done in a savepoint's transaction is tentative until the
transaction is committed using two-phase commit.
>>> dm = DataManager()
>>> dm.inc()
>>> t1 = '1'
>>> r = dm.savepoint(t1)
>>> dm.state, dm.delta
(0, 1)
>>> dm.inc()
>>> dm.state, dm.delta
(0, 2)
>>> r.rollback()
>>> dm.state, dm.delta
(0, 1)
>>> dm.prepare(t1)
>>> dm.commit(t1)
>>> dm.state, dm.delta
(1, 0)
Savepoints must have the same transaction:
>>> r1 = dm.savepoint(t1)
>>> dm.state, dm.delta
(1, 0)
>>> dm.inc()
>>> dm.state, dm.delta
(1, 1)
>>> t2 = '2'
>>> r2 = dm.savepoint(t2)
Traceback (most recent call last):
...
TypeError: ('Transaction missmatch', '2', '1')
>>> r2 = dm.savepoint(t1)
>>> dm.inc()
>>> dm.state, dm.delta
(1, 2)
If we rollback to an earlier savepoint, we discard all work
done later:
>>> r1.rollback()
>>> dm.state, dm.delta
(1, 0)
and we can no longer rollback to the later savepoint:
>>> r2.rollback()
Traceback (most recent call last):
...
TypeError: ('Attempt to roll back to invalid save point', 3, 2)
We can roll back to a savepoint as often as we like:
>>> r1.rollback()
>>> r1.rollback()
>>> r1.rollback()
>>> dm.state, dm.delta
(1, 0)
>>> dm.inc()
>>> dm.inc()
>>> dm.inc()
>>> dm.state, dm.delta
(1, 3)
>>> r1.rollback()
>>> dm.state, dm.delta
(1, 0)
But we can't rollback to a savepoint after it has been
committed:
>>> dm.prepare(t1)
>>> dm.commit(t1)
>>> r1.rollback()
Traceback (most recent call last):
...
TypeError: Attempt to rollback stale rollback
"""
if self.prepared:
raise TypeError("Can't get savepoint during two-phase commit")
self._checkTransaction(transaction)
self.transaction = transaction
self.sp += 1
return Rollback(self)
class Rollback(object):
def __init__(self, dm):
self.dm = dm
self.sp = dm.sp
self.delta = dm.delta
self.transaction = dm.transaction
def rollback(self):
if self.transaction is not self.dm.transaction:
raise TypeError("Attempt to rollback stale rollback")
if self.dm.sp < self.sp:
raise TypeError("Attempt to roll back to invalid save point",
self.sp, self.dm.sp)
self.dm.sp = self.sp
self.dm.delta = self.delta
def test_suite():
from zope.testing.doctest import DocTestSuite
return DocTestSuite()
if __name__ == '__main__':
unittest.main()
ZODB/src/transaction/tests/abstestIDataManager.py 0000640 0001750 0000144 00000003766 10227311640 021665 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Test cases for objects implementing IDataManager.
This is a combo test between Connection and DB, since the two are
rather incestuous and the DB Interface is not defined that I was
able to find.
To do a full test suite one would probably want to write a dummy
storage that will raise errors as needed for testing.
I started this test suite to reproduce a very simple error (tpc_abort
had an error and wouldn't even run if called). So it is *very*
incomplete, and even the tests that exist do not make sure that
the data actually gets written/not written to the storge.
Obviously this test suite should be expanded.
$Id: abstestIDataManager.py 29966 2005-04-13 21:48:16Z tim_one $
"""
from unittest import TestCase
class IDataManagerTests(TestCase, object):
def setUp(self):
self.datamgr = None # subclass should override
self.obj = None # subclass should define Persistent object
self.txn_factory = None
def get_transaction(self):
return self.txn_factory()
################################
# IDataManager interface tests #
################################
def testCommitObj(self):
tran = self.get_transaction()
self.datamgr.prepare(tran)
self.datamgr.commit(tran)
def testAbortTran(self):
tran = self.get_transaction()
self.datamgr.prepare(tran)
self.datamgr.abort(tran)
ZODB/src/transaction/tests/test_SampleResourceManager.py 0000640 0001750 0000144 00000027346 10070356662 023316 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Sample objects for use in tests
$Id: test_SampleResourceManager.py 26003 2004-06-29 21:22:58Z tim_one $
"""
class ResourceManager(object):
"""Sample resource manager.
This class provides a trivial resource-manager implementation and doc
strings to illustrate the protocol and to provide a tool for writing
tests.
Our sample resource manager has state that is updated through an inc
method and through transaction operations.
When we create a sample resource manager:
>>> rm = ResourceManager()
It has two pieces state, state and delta, both initialized to 0:
>>> rm.state
0
>>> rm.delta
0
state is meant to model committed state, while delta represents
tentative changes within a transaction. We change the state by
calling inc:
>>> rm.inc()
which updates delta:
>>> rm.delta
1
but state isn't changed until we commit the transaction:
>>> rm.state
0
To commit the changes, we use 2-phase commit. We execute the first
stage by calling prepare. We need to pass a transation. Our
sample resource managers don't really use the transactions for much,
so we'll be lazy and use strings for transactions. The sample
resource manager updates the state when we call tpc_vote:
>>> t1 = '1'
>>> rm.tpc_begin(t1)
>>> rm.state, rm.delta
(0, 1)
>>> rm.tpc_vote(t1)
>>> rm.state, rm.delta
(1, 1)
Now if we call tpc_finish:
>>> rm.tpc_finish(t1)
Our changes are "permanent". The state reflects the changes and the
delta has been reset to 0.
>>> rm.state, rm.delta
(1, 0)
"""
def __init__(self):
self.state = 0
self.sp = 0
self.transaction = None
self.delta = 0
self.txn_state = None
def _check_state(self, *ok_states):
if self.txn_state not in ok_states:
raise ValueError("txn in state %r but expected one of %r" %
(self.txn_state, ok_states))
def _checkTransaction(self, transaction):
if (transaction is not self.transaction
and self.transaction is not None):
raise TypeError("Transaction missmatch",
transaction, self.transaction)
def inc(self, n=1):
self.delta += n
def tpc_begin(self, transaction):
"""Prepare to commit data.
>>> rm = ResourceManager()
>>> rm.inc()
>>> t1 = '1'
>>> rm.tpc_begin(t1)
>>> rm.tpc_vote(t1)
>>> rm.tpc_finish(t1)
>>> rm.state
1
>>> rm.inc()
>>> t2 = '2'
>>> rm.tpc_begin(t2)
>>> rm.tpc_vote(t2)
>>> rm.tpc_abort(t2)
>>> rm.state
1
It is an error to call tpc_begin more than once without completing
two-phase commit:
>>> rm.tpc_begin(t1)
>>> rm.tpc_begin(t1)
Traceback (most recent call last):
...
ValueError: txn in state 'tpc_begin' but expected one of (None,)
>>> rm.tpc_abort(t1)
If there was a preceeding savepoint, the transaction must match:
>>> rollback = rm.savepoint(t1)
>>> rm.tpc_begin(t2)
Traceback (most recent call last):
,,,
TypeError: ('Transaction missmatch', '2', '1')
>>> rm.tpc_begin(t1)
"""
self._checkTransaction(transaction)
self._check_state(None)
self.transaction = transaction
self.txn_state = 'tpc_begin'
def tpc_vote(self, transaction):
"""Verify that a data manager can commit the transaction.
This is the last chance for a data manager to vote 'no'. A
data manager votes 'no' by raising an exception.
transaction is the ITransaction instance associated with the
transaction being committed.
"""
self._checkTransaction(transaction)
self._check_state('tpc_begin')
self.state += self.delta
self.txn_state = 'tpc_vote'
def tpc_finish(self, transaction):
"""Complete two-phase commit
>>> rm = ResourceManager()
>>> rm.state
0
>>> rm.inc()
We start two-phase commit by calling prepare:
>>> t1 = '1'
>>> rm.tpc_begin(t1)
>>> rm.tpc_vote(t1)
We complete it by calling tpc_finish:
>>> rm.tpc_finish(t1)
>>> rm.state
1
It is an error ro call tpc_finish without calling tpc_vote:
>>> rm.inc()
>>> t2 = '2'
>>> rm.tpc_begin(t2)
>>> rm.tpc_finish(t2)
Traceback (most recent call last):
...
ValueError: txn in state 'tpc_begin' but expected one of ('tpc_vote',)
>>> rm.tpc_abort(t2) # clean slate
>>> rm.tpc_begin(t2)
>>> rm.tpc_vote(t2)
>>> rm.tpc_finish(t2)
Of course, the transactions given to tpc_begin and tpc_finish must
be the same:
>>> rm.inc()
>>> t3 = '3'
>>> rm.tpc_begin(t3)
>>> rm.tpc_vote(t3)
>>> rm.tpc_finish(t2)
Traceback (most recent call last):
...
TypeError: ('Transaction missmatch', '2', '3')
"""
self._checkTransaction(transaction)
self._check_state('tpc_vote')
self.delta = 0
self.transaction = None
self.prepared = False
self.txn_state = None
def tpc_abort(self, transaction):
"""Abort a transaction
The abort method can be called before two-phase commit to
throw away work done in the transaction:
>>> rm = ResourceManager()
>>> rm.inc()
>>> rm.state, rm.delta
(0, 1)
>>> t1 = '1'
>>> rm.tpc_abort(t1)
>>> rm.state, rm.delta
(0, 0)
The abort method also throws away work done in savepoints:
>>> rm.inc()
>>> r = rm.savepoint(t1)
>>> rm.inc()
>>> r = rm.savepoint(t1)
>>> rm.state, rm.delta
(0, 2)
>>> rm.tpc_abort(t1)
>>> rm.state, rm.delta
(0, 0)
If savepoints are used, abort must be passed the same
transaction:
>>> rm.inc()
>>> r = rm.savepoint(t1)
>>> t2 = '2'
>>> rm.tpc_abort(t2)
Traceback (most recent call last):
...
TypeError: ('Transaction missmatch', '2', '1')
>>> rm.tpc_abort(t1)
The abort method is also used to abort a two-phase commit:
>>> rm.inc()
>>> rm.state, rm.delta
(0, 1)
>>> rm.tpc_begin(t1)
>>> rm.state, rm.delta
(0, 1)
>>> rm.tpc_vote(t1)
>>> rm.state, rm.delta
(1, 1)
>>> rm.tpc_abort(t1)
>>> rm.state, rm.delta
(0, 0)
Of course, the transactions passed to prepare and abort must
match:
>>> rm.tpc_begin(t1)
>>> rm.tpc_abort(t2)
Traceback (most recent call last):
...
TypeError: ('Transaction missmatch', '2', '1')
>>> rm.tpc_abort(t1)
This should never fail.
"""
self._checkTransaction(transaction)
if self.transaction is not None:
self.transaction = None
if self.txn_state == 'tpc_vote':
self.state -= self.delta
self.txn_state = None
self.delta = 0
def savepoint(self, transaction):
"""Provide the ability to rollback transaction state
Savepoints provide a way to:
- Save partial transaction work. For some resource managers, this
could allow resources to be used more efficiently.
- Provide the ability to revert state to a point in a
transaction without aborting the entire transaction. In
other words, savepoints support partial aborts.
Savepoints don't use two-phase commit. If there are errors in
setting or rolling back to savepoints, the application should
abort the containing transaction. This is *not* the
responsibility of the resource manager.
Savepoints are always associated with a transaction. Any work
done in a savepoint's transaction is tentative until the
transaction is committed using two-phase commit.
>>> rm = ResourceManager()
>>> rm.inc()
>>> t1 = '1'
>>> r = rm.savepoint(t1)
>>> rm.state, rm.delta
(0, 1)
>>> rm.inc()
>>> rm.state, rm.delta
(0, 2)
>>> r.rollback()
>>> rm.state, rm.delta
(0, 1)
>>> rm.tpc_begin(t1)
>>> rm.tpc_vote(t1)
>>> rm.tpc_finish(t1)
>>> rm.state, rm.delta
(1, 0)
Savepoints must have the same transaction:
>>> r1 = rm.savepoint(t1)
>>> rm.state, rm.delta
(1, 0)
>>> rm.inc()
>>> rm.state, rm.delta
(1, 1)
>>> t2 = '2'
>>> r2 = rm.savepoint(t2)
Traceback (most recent call last):
...
TypeError: ('Transaction missmatch', '2', '1')
>>> r2 = rm.savepoint(t1)
>>> rm.inc()
>>> rm.state, rm.delta
(1, 2)
If we rollback to an earlier savepoint, we discard all work
done later:
>>> r1.rollback()
>>> rm.state, rm.delta
(1, 0)
and we can no longer rollback to the later savepoint:
>>> r2.rollback()
Traceback (most recent call last):
...
TypeError: ('Attempt to roll back to invalid save point', 3, 2)
We can roll back to a savepoint as often as we like:
>>> r1.rollback()
>>> r1.rollback()
>>> r1.rollback()
>>> rm.state, rm.delta
(1, 0)
>>> rm.inc()
>>> rm.inc()
>>> rm.inc()
>>> rm.state, rm.delta
(1, 3)
>>> r1.rollback()
>>> rm.state, rm.delta
(1, 0)
But we can't rollback to a savepoint after it has been
committed:
>>> rm.tpc_begin(t1)
>>> rm.tpc_vote(t1)
>>> rm.tpc_finish(t1)
>>> r1.rollback()
Traceback (most recent call last):
...
TypeError: Attempt to rollback stale rollback
"""
if self.txn_state is not None:
raise TypeError("Can't get savepoint during two-phase commit")
self._checkTransaction(transaction)
self.transaction = transaction
self.sp += 1
return SavePoint(self)
def discard(self, transaction):
pass
class SavePoint(object):
def __init__(self, rm):
self.rm = rm
self.sp = rm.sp
self.delta = rm.delta
self.transaction = rm.transaction
def rollback(self):
if self.transaction is not self.rm.transaction:
raise TypeError("Attempt to rollback stale rollback")
if self.rm.sp < self.sp:
raise TypeError("Attempt to roll back to invalid save point",
self.sp, self.rm.sp)
self.rm.sp = self.sp
self.rm.delta = self.delta
def discard(self):
pass
def test_suite():
from doctest import DocTestSuite
return DocTestSuite()
if __name__ == '__main__':
unittest.main()
ZODB/src/transaction/tests/test_savepoint.py 0000640 0001750 0000144 00000003542 10233233103 021053 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Tests of savepoint feature
$Id: test_savepoint.py 30169 2005-04-25 18:26:11Z tim_one $
"""
import unittest
from zope.testing import doctest
def testRollbackRollsbackDataManagersThatJoinedLater():
"""
A savepoint needs to not just rollback it's savepoints, but needs to
rollback savepoints for data managers that joined savepoints after the
savepoint:
>>> import transaction.tests.savepointsample
>>> dm = transaction.tests.savepointsample.SampleSavepointDataManager()
>>> dm['name'] = 'bob'
>>> sp1 = transaction.savepoint()
>>> dm['job'] = 'geek'
>>> sp2 = transaction.savepoint()
>>> dm['salary'] = 'fun'
>>> dm2 = transaction.tests.savepointsample.SampleSavepointDataManager()
>>> dm2['name'] = 'sally'
>>> 'name' in dm
True
>>> 'job' in dm
True
>>> 'salary' in dm
True
>>> 'name' in dm2
True
>>> sp1.rollback()
>>> 'name' in dm
True
>>> 'job' in dm
False
>>> 'salary' in dm
False
>>> 'name' in dm2
False
"""
def test_suite():
return unittest.TestSuite((
doctest.DocFileSuite('../savepoint.txt'),
doctest.DocTestSuite(),
))
if __name__ == '__main__':
unittest.main(defaultTest='test_suite')
ZODB/src/transaction/_manager.py 0000640 0001750 0000144 00000013474 10265347560 016442 0 ustar ctheune users ############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
############################################################################
"""A TransactionManager controls transaction boundaries.
It coordinates application code and resource managers, so that they
are associated with the right transaction.
"""
import thread
from transaction._transaction import Transaction
# Used for deprecated arguments. ZODB.utils.DEPRECATED_ARGUMENT is
# too hard to use here, due to the convoluted import dance across
# __init__.py files.
_marker = object()
# We have to remember sets of synch objects, especially Connections.
# But we don't want mere registration with a transaction manager to
# keep a synch object alive forever; in particular, it's common
# practice not to explicitly close Connection objects, and keeping
# a Connection alive keeps a potentially huge number of other objects
# alive (e.g., the cache, and everything reachable from it too).
# Therefore we use "weak sets" internally.
#
# Obscure: because of the __init__.py maze, we can't import WeakSet
# at top level here.
# Call the ISynchronizer newTransaction() method on every element of
# WeakSet synchs.
# A transaction manager needs to do this whenever begin() is called.
# Since it would be good if tm.get() returned the new transaction while
# newTransaction() is running, calling this has to be delayed until after
# the transaction manager has done whatever it needs to do to make its
# get() return the new txn.
def _new_transaction(txn, synchs):
if synchs:
synchs.map(lambda s: s.newTransaction(txn))
# Important: we must always pass a WeakSet (even if empty) to the Transaction
# constructor: synchronizers are registered with the TM, but the
# ISynchronizer xyzCompletion() methods are called by Transactions without
# consulting the TM, so we need to pass a mutable collection of synchronizers
# so that Transactions "see" synchronizers that get registered after the
# Transaction object is constructed.
class TransactionManager(object):
def __init__(self):
from ZODB.utils import WeakSet
self._txn = None
self._synchs = WeakSet()
def begin(self):
if self._txn is not None:
self._txn.abort()
txn = self._txn = Transaction(self._synchs, self)
_new_transaction(txn, self._synchs)
return txn
def get(self):
if self._txn is None:
self._txn = Transaction(self._synchs, self)
return self._txn
def free(self, txn):
assert txn is self._txn
self._txn = None
def registerSynch(self, synch):
self._synchs.add(synch)
def unregisterSynch(self, synch):
self._synchs.remove(synch)
def commit(self, sub=_marker):
if sub is _marker:
sub = None
else:
from ZODB.utils import deprecated37
deprecated37("subtransactions are deprecated; use "
"transaction.savepoint() instead of "
"transaction.commit(1)")
return self.get().commit(sub, deprecation_wng=False)
def abort(self, sub=_marker):
if sub is _marker:
sub = None
else:
from ZODB.utils import deprecated37
deprecated37("subtransactions are deprecated; use "
"sp.rollback() instead of "
"transaction.abort(1), where `sp` is the "
"corresponding savepoint captured earlier")
return self.get().abort(sub, deprecation_wng=False)
def savepoint(self, optimistic=False):
return self.get().savepoint(optimistic)
class ThreadTransactionManager(TransactionManager):
"""Thread-aware transaction manager.
Each thread is associated with a unique transaction.
"""
def __init__(self):
# _threads maps thread ids to transactions
self._txns = {}
# _synchs maps a thread id to a WeakSet of registered synchronizers.
# The WeakSet is passed to the Transaction constructor, because the
# latter needs to call the synchronizers when it commits.
self._synchs = {}
def begin(self):
tid = thread.get_ident()
txn = self._txns.get(tid)
if txn is not None:
txn.abort()
synchs = self._synchs.get(tid)
if synchs is None:
from ZODB.utils import WeakSet
synchs = self._synchs[tid] = WeakSet()
txn = self._txns[tid] = Transaction(synchs, self)
_new_transaction(txn, synchs)
return txn
def get(self):
tid = thread.get_ident()
txn = self._txns.get(tid)
if txn is None:
synchs = self._synchs.get(tid)
if synchs is None:
from ZODB.utils import WeakSet
synchs = self._synchs[tid] = WeakSet()
txn = self._txns[tid] = Transaction(synchs, self)
return txn
def free(self, txn):
tid = thread.get_ident()
assert txn is self._txns.get(tid)
del self._txns[tid]
def registerSynch(self, synch):
tid = thread.get_ident()
ws = self._synchs.get(tid)
if ws is None:
from ZODB.utils import WeakSet
ws = self._synchs[tid] = WeakSet()
ws.add(synch)
def unregisterSynch(self, synch):
tid = thread.get_ident()
ws = self._synchs[tid]
ws.remove(synch)
ZODB/src/transaction/DEPENDENCIES.cfg 0000640 0001750 0000144 00000000024 10223352372 016600 0 ustar ctheune users ZODB
zope.interface
ZODB/src/transaction/__init__.py 0000640 0001750 0000144 00000001775 10375443702 016426 0 ustar ctheune users ############################################################################
#
# Copyright (c) 2001, 2002, 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
############################################################################
"""Exported transaction functions.
$Id: __init__.py 41650 2006-02-17 22:01:38Z ctheune $
"""
from transaction._transaction import Transaction
from transaction._manager import TransactionManager, ThreadTransactionManager
manager = ThreadTransactionManager()
get = manager.get
begin = manager.begin
commit = manager.commit
abort = manager.abort
savepoint = manager.savepoint
ZODB/src/transaction/savepoint.txt 0000640 0001750 0000144 00000017746 10375443702 017073 0 ustar ctheune users Savepoints
==========
Savepoints provide a way to save to disk intermediate work done during
a transaction allowing:
- partial transaction (subtransaction) rollback (abort)
- state of saved objects to be freed, freeing on-line memory for other
uses
Savepoints make it possible to write atomic subroutines that don't
make top-level transaction commitments.
Applications
------------
To demonstrate how savepoints work with transactions, we've provided a sample
data manager implementation that provides savepoint support. The primary
purpose of this data manager is to provide code that can be read to understand
how savepoints work. The secondary purpose is to provide support for
demonstrating the correct operation of savepoint support within the
transaction system. This data manager is very simple. It provides flat
storage of named immutable values, like strings and numbers.
>>> import transaction.tests.savepointsample
>>> dm = transaction.tests.savepointsample.SampleSavepointDataManager()
>>> dm['name'] = 'bob'
As with other data managers, we can commit changes:
>>> transaction.commit()
>>> dm['name']
'bob'
and abort changes:
>>> dm['name'] = 'sally'
>>> dm['name']
'sally'
>>> transaction.abort()
>>> dm['name']
'bob'
Now, let's look at an application that manages funds for people. It allows
deposits and debits to be entered for multiple people. It accepts a sequence
of entries and generates a sequence of status messages. For each entry, it
applies the change and then validates the user's account. If the user's
account is invalid, we roll back the change for that entry. The success or
failure of an entry is indicated in the output status. First we'll initialize
some accounts:
>>> dm['bob-balance'] = 0.0
>>> dm['bob-credit'] = 0.0
>>> dm['sally-balance'] = 0.0
>>> dm['sally-credit'] = 100.0
>>> transaction.commit()
Now, we'll define a validation function to validate an account:
>>> def validate_account(name):
... if dm[name+'-balance'] + dm[name+'-credit'] < 0:
... raise ValueError('Overdrawn', name)
And a function to apply entries. If the function fails in some unexpected
way, it rolls back all of its changes and prints the error:
>>> def apply_entries(entries):
... savepoint = transaction.savepoint()
... try:
... for name, amount in entries:
... entry_savepoint = transaction.savepoint()
... try:
... dm[name+'-balance'] += amount
... validate_account(name)
... except ValueError, error:
... entry_savepoint.rollback()
... print 'Error', str(error)
... else:
... print 'Updated', name
... except Exception, error:
... savepoint.rollback()
... print 'Unexpected exception', error
Now let's try applying some entries:
>>> apply_entries([
... ('bob', 10.0),
... ('sally', 10.0),
... ('bob', 20.0),
... ('sally', 10.0),
... ('bob', -100.0),
... ('sally', -100.0),
... ])
Updated bob
Updated sally
Updated bob
Updated sally
Error ('Overdrawn', 'bob')
Updated sally
>>> dm['bob-balance']
30.0
>>> dm['sally-balance']
-80.0
If we provide entries that cause an unexpected error:
>>> apply_entries([
... ('bob', 10.0),
... ('sally', 10.0),
... ('bob', '20.0'),
... ('sally', 10.0),
... ])
Updated bob
Updated sally
Unexpected exception unsupported operand type(s) for +=: 'float' and 'str'
Because the apply_entries used a savepoint for the entire function, it was
able to rollback the partial changes without rolling back changes made in the
previous call to ``apply_entries``:
>>> dm['bob-balance']
30.0
>>> dm['sally-balance']
-80.0
If we now abort the outer transactions, the earlier changes will go
away:
>>> transaction.abort()
>>> dm['bob-balance']
0.0
>>> dm['sally-balance']
0.0
Savepoint invalidation
----------------------
A savepoint can be used any number of times:
>>> dm['bob-balance'] = 100.0
>>> dm['bob-balance']
100.0
>>> savepoint = transaction.savepoint()
>>> dm['bob-balance'] = 200.0
>>> dm['bob-balance']
200.0
>>> savepoint.rollback()
>>> dm['bob-balance']
100.0
>>> savepoint.rollback() # redundant, but should be harmless
>>> dm['bob-balance']
100.0
>>> dm['bob-balance'] = 300.0
>>> dm['bob-balance']
300.0
>>> savepoint.rollback()
>>> dm['bob-balance']
100.0
However, using a savepoint invalidates any savepoints that come after it:
>>> dm['bob-balance'] = 200.0
>>> dm['bob-balance']
200.0
>>> savepoint1 = transaction.savepoint()
>>> dm['bob-balance'] = 300.0
>>> dm['bob-balance']
300.0
>>> savepoint2 = transaction.savepoint()
>>> savepoint.rollback()
>>> dm['bob-balance']
100.0
>>> savepoint2.rollback()
Traceback (most recent call last):
...
InvalidSavepointRollbackError
>>> savepoint1.rollback()
Traceback (most recent call last):
...
InvalidSavepointRollbackError
>>> transaction.abort()
Databases without savepoint support
-----------------------------------
Normally it's an error to use savepoints with databases that don't support
savepoints:
>>> dm_no_sp = transaction.tests.savepointsample.SampleDataManager()
>>> dm_no_sp['name'] = 'bob'
>>> transaction.commit()
>>> dm_no_sp['name'] = 'sally'
>>> savepoint = transaction.savepoint()
Traceback (most recent call last):
...
TypeError: ('Savepoints unsupported', {'name': 'bob'})
>>> transaction.abort()
However, a flag can be passed to the transaction savepoint method to indicate
that databases without savepoint support should be tolerated until a savepoint
is rolled back. This allows transactions to proceed if there are no reasons
to roll back:
>>> dm_no_sp['name'] = 'sally'
>>> savepoint = transaction.savepoint(1)
>>> dm_no_sp['name'] = 'sue'
>>> transaction.commit()
>>> dm_no_sp['name']
'sue'
>>> dm_no_sp['name'] = 'sam'
>>> savepoint = transaction.savepoint(1)
>>> savepoint.rollback()
Traceback (most recent call last):
...
TypeError: ('Savepoints unsupported', {'name': 'sam'})
Failures
--------
If a failure occurs when creating or rolling back a savepoint, the transaction
state will be uncertain and the transaction will become uncommitable. From
that point on, most transaction operations, including commit, will fail until
the transaction is aborted.
In the previous example, we got an error when we tried to rollback the
savepoint. If we try to commit the transaction, the commit will fail:
>>> transaction.commit() # doctest: +ELLIPSIS
Traceback (most recent call last):
...
TransactionFailedError: An operation previously failed, with traceback:
...
TypeError: ('Savepoints unsupported', {'name': 'sam'})
We have to abort it to make any progress:
>>> transaction.abort()
Similarly, in our earlier example, where we tried to take a savepoint with a
data manager that didn't support savepoints:
>>> dm_no_sp['name'] = 'sally'
>>> dm['name'] = 'sally'
>>> savepoint = transaction.savepoint()
Traceback (most recent call last):
...
TypeError: ('Savepoints unsupported', {'name': 'sue'})
>>> transaction.commit() # doctest: +ELLIPSIS
Traceback (most recent call last):
...
TransactionFailedError: An operation previously failed, with traceback:
...
TypeError: ('Savepoints unsupported', {'name': 'sue'})
>>> transaction.abort()
After clearing the transaction with an abort, we can get on with new
transactions:
>>> dm_no_sp['name'] = 'sally'
>>> dm['name'] = 'sally'
>>> transaction.commit()
>>> dm_no_sp['name']
'sally'
>>> dm['name']
'sally'
ZODB/src/transaction/interfaces.py 0000640 0001750 0000144 00000042475 10375443702 017014 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Transaction Interfaces
$Id: interfaces.py 41650 2006-02-17 22:01:38Z ctheune $
"""
import zope.interface
class ITransactionManager(zope.interface.Interface):
"""An object that manages a sequence of transactions.
Applications use transaction managers to establish transaction boundaries.
"""
def begin():
"""Begin a new transaction.
If an existing transaction is in progress, it will be aborted.
The newTransaction() method of registered synchronizers is called,
passing the new transaction object.
"""
def get():
"""Get the current transaction.
"""
def commit():
"""Commit the current transaction.
"""
def abort():
"""Abort the current transaction.
"""
def savepoint(optimistic=False):
"""Create a savepoint from the current transaction.
If the optimistic argument is true, then data managers that
don't support savepoints can be used, but an error will be
raised if the savepoint is rolled back.
An ISavepoint object is returned.
"""
def registerSynch(synch):
"""Register an ISynchronizer.
Synchronizers are notified about some major events in a transaction's
life. See ISynchronizer for details.
"""
def unregisterSynch(synch):
"""Unregister an ISynchronizer.
Synchronizers are notified about some major events in a transaction's
life. See ISynchronizer for details.
"""
class ITransaction(zope.interface.Interface):
"""Object representing a running transaction.
Objects with this interface may represent different transactions
during their lifetime (.begin() can be called to start a new
transaction using the same instance, although that example is
deprecated and will go away in ZODB 3.6).
"""
user = zope.interface.Attribute(
"""A user name associated with the transaction.
The format of the user name is defined by the application. The value
is of Python type str. Storages record the user value, as meta-data,
when a transaction commits.
A storage may impose a limit on the size of the value; behavior is
undefined if such a limit is exceeded (for example, a storage may
raise an exception, or truncate the value).
""")
description = zope.interface.Attribute(
"""A textual description of the transaction.
The value is of Python type str. Method note() is the intended
way to set the value. Storages record the description, as meta-data,
when a transaction commits.
A storage may impose a limit on the size of the description; behavior
is undefined if such a limit is exceeded (for example, a storage may
raise an exception, or truncate the value).
""")
def commit():
"""Finalize the transaction.
This executes the two-phase commit algorithm for all
IDataManager objects associated with the transaction.
"""
def abort():
"""Abort the transaction.
This is called from the application. This can only be called
before the two-phase commit protocol has been started.
"""
def savepoint(optimistic=False):
"""Create a savepoint.
If the optimistic argument is true, then data managers that don't
support savepoints can be used, but an error will be raised if the
savepoint is rolled back.
An ISavepoint object is returned.
"""
def join(datamanager):
"""Add a data manager to the transaction.
`datamanager` must provide the transactions.interfaces.IDataManager
interface.
"""
def note(text):
"""Add text to the transaction description.
This modifies the `.description` attribute; see its docs for more
detail. First surrounding whitespace is stripped from `text`. If
`.description` is currently an empty string, then the stripped text
becomes its value, else two newlines and the stripped text are
appended to `.description`.
"""
def setUser(user_name, path="/"):
"""Set the user name.
path should be provided if needed to further qualify the
identified user. This is a convenience method used by Zope.
It sets the .user attribute to str(path) + " " + str(user_name).
This sets the `.user` attribute; see its docs for more detail.
"""
def setExtendedInfo(name, value):
"""Add extension data to the transaction.
name is the name of the extension property to set, of Python type
str; value must be picklable. Multiple calls may be made to set
multiple extension properties, provided the names are distinct.
Storages record the extension data, as meta-data, when a transaction
commits.
A storage may impose a limit on the size of extension data; behavior
is undefined if such a limit is exceeded (for example, a storage may
raise an exception, or remove `` pairs).
"""
# deprecated38
def beforeCommitHook(__hook, *args, **kws):
"""Register a hook to call before the transaction is committed.
THIS IS DEPRECATED IN ZODB 3.6. Use addBeforeCommitHook() instead.
The specified hook function will be called after the transaction's
commit method has been called, but before the commit process has been
started. The hook will be passed the specified positional and keyword
arguments.
Multiple hooks can be registered and will be called in the order they
were registered (first registered, first called). This method can
also be called from a hook: an executing hook can register more
hooks. Applications should take care to avoid creating infinite loops
by recursively registering hooks.
Hooks are called only for a top-level commit. A subtransaction
commit does not call any hooks. If the transaction is aborted, hooks
are not called, and are discarded. Calling a hook "consumes" its
registration too: hook registrations do not persist across
transactions. If it's desired to call the same hook on every
transaction commit, then beforeCommitHook() must be called with that
hook during every transaction; in such a case consider registering a
synchronizer object via a TransactionManager's registerSynch() method
instead.
"""
def addBeforeCommitHook(hook, args=(), kws=None):
"""Register a hook to call before the transaction is committed.
The specified hook function will be called after the transaction's
commit method has been called, but before the commit process has been
started. The hook will be passed the specified positional (`args`)
and keyword (`kws`) arguments. `args` is a sequence of positional
arguments to be passed, defaulting to an empty tuple (no positional
arguments are passed). `kws` is a dictionary of keyword argument
names and values to be passed, or the default None (no keyword
arguments are passed).
Multiple hooks can be registered and will be called in the order they
were registered (first registered, first called). This method can
also be called from a hook: an executing hook can register more
hooks. Applications should take care to avoid creating infinite loops
by recursively registering hooks.
Hooks are called only for a top-level commit. A subtransaction
commit or savepoint creation does not call any hooks. If the
transaction is aborted, hooks are not called, and are discarded.
Calling a hook "consumes" its registration too: hook registrations
do not persist across transactions. If it's desired to call the same
hook on every transaction commit, then addBeforeCommitHook() must be
called with that hook during every transaction; in such a case
consider registering a synchronizer object via a TransactionManager's
registerSynch() method instead.
"""
def getBeforeCommitHooks():
"""Return iterable producing the registered addBeforeCommit hooks.
A triple (hook, args, kws) is produced for each registered hook.
The hooks are produced in the order in which they would be invoked
by a top-level transaction commit.
"""
def addAfterCommitHook(hook, args=(), kws=None):
"""Register a hook to call after a transaction commit attempt.
The specified hook function will be called after the transaction
commit succeeds or aborts. The first argument passed to the hook
is a Boolean value, true if the commit succeeded, or false if the
commit aborted. `args` specifies additional positional, and `kws`
keyword, arguments to pass to the hook. `args` is a sequence of
positional arguments to be passed, defaulting to an empty tuple
(only the true/false success argument is passed). `kws` is a
dictionary of keyword argument names and values to be passed, or
the default None (no keyword arguments are passed).
Multiple hooks can be registered and will be called in the order they
were registered (first registered, first called). This method can
also be called from a hook: an executing hook can register more
hooks. Applications should take care to avoid creating infinite loops
by recursively registering hooks.
Hooks are called only for a top-level commit. A subtransaction
commit or savepoint creation does not call any hooks. Calling a
hook "consumes" its registration: hook registrations do not
persist across transactions. If it's desired to call the same
hook on every transaction commit, then addAfterCommitHook() must be
called with that hook during every transaction; in such a case
consider registering a synchronizer object via a TransactionManager's
registerSynch() method instead.
"""
def getAfterCommitHooks():
"""Return iterable producing the registered addAfterCommit hooks.
A triple (hook, args, kws) is produced for each registered hook.
The hooks are produced in the order in which they would be invoked
by a top-level transaction commit.
"""
class ITransactionDeprecated(zope.interface.Interface):
"""Deprecated parts of the transaction API."""
def begin(info=None):
"""Begin a new transaction.
If the transaction is in progress, it is aborted and a new
transaction is started using the same transaction object.
"""
# TODO: deprecate this for 3.6.
def register(object):
"""Register the given object for transaction control."""
class IDataManager(zope.interface.Interface):
"""Objects that manage transactional storage.
These objects may manage data for other objects, or they may manage
non-object storages, such as relational databases. For example,
a ZODB.Connection.
Note that when some data is modified, that data's data manager should
join a transaction so that data can be committed when the user commits
the transaction.
"""
transaction_manager = zope.interface.Attribute(
"""The transaction manager (TM) used by this data manager.
This is a public attribute, intended for read-only use. The value
is an instance of ITransactionManager, typically set by the data
manager's constructor.
""")
def abort(transaction):
"""Abort a transaction and forget all changes.
Abort must be called outside of a two-phase commit.
Abort is called by the transaction manager to abort transactions
that are not yet in a two-phase commit.
"""
# Two-phase commit protocol. These methods are called by the ITransaction
# object associated with the transaction being committed. The sequence
# of calls normally follows this regular expression:
# tpc_begin commit tpc_vote (tpc_finish | tpc_abort)
def tpc_begin(transaction):
"""Begin commit of a transaction, starting the two-phase commit.
transaction is the ITransaction instance associated with the
transaction being committed.
"""
def commit(transaction):
"""Commit modifications to registered objects.
Save changes to be made persistent if the transaction commits (if
tpc_finish is called later). If tpc_abort is called later, changes
must not persist.
This includes conflict detection and handling. If no conflicts or
errors occur, the data manager should be prepared to make the
changes persist when tpc_finish is called.
"""
def tpc_vote(transaction):
"""Verify that a data manager can commit the transaction.
This is the last chance for a data manager to vote 'no'. A
data manager votes 'no' by raising an exception.
transaction is the ITransaction instance associated with the
transaction being committed.
"""
def tpc_finish(transaction):
"""Indicate confirmation that the transaction is done.
Make all changes to objects modified by this transaction persist.
transaction is the ITransaction instance associated with the
transaction being committed.
This should never fail. If this raises an exception, the
database is not expected to maintain consistency; it's a
serious error.
"""
def tpc_abort(transaction):
"""Abort a transaction.
This is called by a transaction manager to end a two-phase commit on
the data manager. Abandon all changes to objects modified by this
transaction.
transaction is the ITransaction instance associated with the
transaction being committed.
This should never fail.
"""
def sortKey():
"""Return a key to use for ordering registered DataManagers.
ZODB uses a global sort order to prevent deadlock when it commits
transactions involving multiple resource managers. The resource
manager must define a sortKey() method that provides a global ordering
for resource managers.
"""
# Alternate version:
#"""Return a consistent sort key for this connection.
#
#This allows ordering multiple connections that use the same storage in
#a consistent manner. This is unique for the lifetime of a connection,
#which is good enough to avoid ZEO deadlocks.
#"""
class ISavepointDataManager(IDataManager):
def savepoint():
"""Return a data-manager savepoint (IDataManagerSavepoint).
"""
class IDataManagerSavepoint(zope.interface.Interface):
"""Savepoint for data-manager changes for use in transaction savepoints.
Datamanager savepoints are used by, and only by, transaction savepoints.
Note that data manager savepoints don't have any notion of, or
responsibility for, validity. It isn't the responsibility of
data-manager savepoints to prevent multiple rollbacks or rollbacks after
transaction termination. Preventing invalid savepoint rollback is the
responsibility of transaction rollbacks. Application code should never
use data-manager savepoints.
"""
def rollback():
"""Rollback any work done since the savepoint.
"""
class ISavepoint(zope.interface.Interface):
"""A transaction savepoint.
"""
def rollback():
"""Rollback any work done since the savepoint.
InvalidSavepointRollbackError is raised if the savepoint isn't valid.
"""
valid = zope.interface.Attribute(
"Boolean indicating whether the savepoint is valid")
class InvalidSavepointRollbackError(Exception):
"""Attempt to rollback an invalid savepoint.
A savepoint may be invalid because:
- The surrounding transaction has committed or aborted.
- An earlier savepoint in the same transaction has been rolled back.
"""
class ISynchronizer(zope.interface.Interface):
"""Objects that participate in the transaction-boundary notification API.
"""
def beforeCompletion(transaction):
"""Hook that is called by the transaction at the start of a commit.
"""
def afterCompletion(transaction):
"""Hook that is called by the transaction after completing a commit.
"""
def newTransaction(transaction):
"""Hook that is called at the start of a transaction.
This hook is called when, and only when, a transaction manager's
begin() method is called explictly.
"""
ZODB/src/transaction/README.txt 0000640 0001750 0000144 00000000763 10375443702 016007 0 ustar ctheune users ============
Transactions
============
This package contains a generic transaction implementation for Python. It is
mainly used by the ZODB, though.
Note that the data manager API, ``transaction.interfaces.IDataManager``,
is syntactically simple, but semantically complex. The semantics
were not easy to express in the interface. This could probably use
more work. The semantics are presented in detail through examples of
a sample data manager in ``transaction.tests.test_SampleDataManager``.
ZODB/src/transaction/_transaction.py 0000640 0001750 0000144 00000071017 10375443702 017347 0 ustar ctheune users ############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
############################################################################
"""Transaction objects manage resources for an individual activity.
Compatibility issues
--------------------
The implementation of Transaction objects involves two layers of
backwards compatibility, because this version of transaction supports
both ZODB 3 and ZODB 4. Zope is evolving towards the ZODB4
interfaces.
Transaction has two methods for a resource manager to call to
participate in a transaction -- register() and join(). join() takes a
resource manager and adds it to the list of resources. register() is
for backwards compatibility. It takes a persistent object and
registers its _p_jar attribute. TODO: explain adapter
Subtransactions
---------------
Note: Subtransactions are deprecated! Use savepoint/rollback instead.
A subtransaction applies the transaction notion recursively. It
allows a set of modifications within a transaction to be committed or
aborted as a group. A subtransaction is a strictly local activity;
its changes are not visible to any other database connection until the
top-level transaction commits. In addition to its use to organize a
large transaction, subtransactions can be used to optimize memory use.
ZODB must keep modified objects in memory until a transaction commits
and it can write the changes to the storage. A subtransaction uses a
temporary disk storage for its commits, allowing modified objects to
be flushed from memory when the subtransaction commits.
The commit() and abort() methods take an optional subtransaction
argument that defaults to false. If it is a true, the operation is
performed on a subtransaction.
Subtransactions add a lot of complexity to the transaction
implementation. Some resource managers support subtransactions, but
they are not required to. (ZODB Connection is the only standard
resource manager that supports subtransactions.) Resource managers
that do support subtransactions implement abort_sub() and commit_sub()
methods and support a second argument to tpc_begin().
The second argument to tpc_begin() indicates that a subtransaction
commit is beginning (if it is true). In a subtransaction, there is no
tpc_vote() call, because sub-transactions don't need 2-phase commit.
If a sub-transaction abort or commit fails, we can abort the outer
transaction. The tpc_finish() or tpc_abort() call applies just to
that subtransaction.
Once a resource manager is involved in a subtransaction, all
subsequent transactions will be treated as subtransactions until
abort_sub() or commit_sub() is called. abort_sub() will undo all the
changes of the subtransactions. commit_sub() will begin a top-level
transaction and store all the changes from subtransactions. After
commit_sub(), the transaction must still call tpc_vote() and
tpc_finish().
If the resource manager does not support subtransactions, nothing
happens when the subtransaction commits. Instead, the resource
manager is put on a list of managers to commit when the actual
top-level transaction commits. If this happens, it will not be
possible to abort subtransactions.
Two-phase commit
----------------
A transaction commit involves an interaction between the transaction
object and one or more resource managers. The transaction manager
calls the following four methods on each resource manager; it calls
tpc_begin() on each resource manager before calling commit() on any of
them.
1. tpc_begin(txn)
2. commit(txn)
3. tpc_vote(txn)
4. tpc_finish(txn)
Subtransaction commit
---------------------
Note: Subtransactions are deprecated!
When a subtransaction commits, the protocol is different.
1. tpc_begin() is passed a second argument, which indicates that a
subtransaction is being committed.
2. tpc_vote() is not called.
Once a subtransaction has been committed, the top-level transaction
commit will start with a commit_sub() call instead of a tpc_begin()
call.
Before-commit hook
------------------
Sometimes, applications want to execute some code when a transaction is
committed. For example, one might want to delay object indexing until a
transaction commits, rather than indexing every time an object is changed.
Or someone might want to check invariants only after a set of operations. A
pre-commit hook is available for such use cases: use addBeforeCommitHook(),
passing it a callable and arguments. The callable will be called with its
arguments at the start of the commit (but not for substransaction commits).
After-commit hook
------------------
Sometimes, applications want to execute code after a transaction is
committed or aborted. For example, one might want to launch non
transactional code after a successful commit. Or still someone might
want to launch asynchronous code after. A post-commit hook is
available for such use cases: use addAfterCommitHook(), passing it a
callable and arguments. The callable will be called with a Boolean
value representing the status of the commit operation as first
argument (true if successfull or false iff aborted) preceding its
arguments at the start of the commit (but not for substransaction
commits).
Error handling
--------------
When errors occur during two-phase commit, the transaction manager
aborts all the resource managers. The specific methods it calls
depend on whether the error occurs before or after the call to
tpc_vote() on that transaction manager.
If the resource manager has not voted, then the resource manager will
have one or more uncommitted objects. There are two cases that lead
to this state; either the transaction manager has not called commit()
for any objects on this resource manager or the call that failed was a
commit() for one of the objects of this resource manager. For each
uncommitted object, including the object that failed in its commit(),
call abort().
Once uncommitted objects are aborted, tpc_abort() or abort_sub() is
called on each resource manager.
Synchronization
---------------
You can register sychronization objects (synchronizers) with the
tranasction manager. The synchronizer must implement
beforeCompletion() and afterCompletion() methods. The transaction
manager calls beforeCompletion() when it starts a top-level two-phase
commit. It calls afterCompletion() when a top-level transaction is
committed or aborted. The methods are passed the current Transaction
as their only argument.
"""
import logging
import sys
import thread
import warnings
import weakref
import traceback
from cStringIO import StringIO
from zope import interface
from transaction import interfaces
# Sigh. In the maze of __init__.py's, ZODB.__init__.py takes 'get'
# out of transaction.__init__.py, in order to stuff the 'get_transaction'
# alias in __builtin__. So here in _transaction.py, we can't import
# exceptions from ZODB.POSException at top level (we're imported by
# our __init__.py, which is imported by ZODB's __init__, so the ZODB
# package isn't well-formed when we're first imported).
# from ZODB.POSException import TransactionError, TransactionFailedError
_marker = object()
# The point of this is to avoid hiding exceptions (which the builtin
# hasattr() does).
def myhasattr(obj, attr):
return getattr(obj, attr, _marker) is not _marker
class Status:
# ACTIVE is the initial state.
ACTIVE = "Active"
COMMITTING = "Committing"
COMMITTED = "Committed"
# commit() or commit(True) raised an exception. All further attempts
# to commit or join this transaction will raise TransactionFailedError.
COMMITFAILED = "Commit failed"
class Transaction(object):
interface.implements(interfaces.ITransaction,
interfaces.ITransactionDeprecated)
# Assign an index to each savepoint so we can invalidate later savepoints
# on rollback. The first index assigned is 1, and it goes up by 1 each
# time.
_savepoint_index = 0
# If savepoints are used, keep a weak key dict of them. This maps a
# savepoint to its index (see above).
_savepoint2index = None
# Remember the savepoint for the last subtransaction.
_subtransaction_savepoint = None
# Meta data. ._extension is also metadata, but is initialized to an
# emtpy dict in __init__.
user = ""
description = ""
def __init__(self, synchronizers=None, manager=None):
self.status = Status.ACTIVE
# List of resource managers, e.g. MultiObjectResourceAdapters.
self._resources = []
# Weak set of synchronizer objects to call.
if synchronizers is None:
from ZODB.utils import WeakSet
synchronizers = WeakSet()
self._synchronizers = synchronizers
self._manager = manager
# _adapters: Connection/_p_jar -> MultiObjectResourceAdapter[Sub]
self._adapters = {}
self._voted = {} # id(Connection) -> boolean, True if voted
# _voted and other dictionaries use the id() of the resource
# manager as a key, because we can't guess whether the actual
# resource managers will be safe to use as dict keys.
# The user, description, and _extension attributes are accessed
# directly by storages, leading underscore notwithstanding.
self._extension = {}
self.log = logging.getLogger("txn.%d" % thread.get_ident())
self.log.debug("new transaction")
# If a commit fails, the traceback is saved in _failure_traceback.
# If another attempt is made to commit, TransactionFailedError is
# raised, incorporating this traceback.
self._failure_traceback = None
# List of (hook, args, kws) tuples added by addBeforeCommitHook().
self._before_commit = []
# List of (hook, args, kws) tuples added by addAfterCommitHook().
self._after_commit = []
# Raise TransactionFailedError, due to commit()/join()/register()
# getting called when the current transaction has already suffered
# a commit/savepoint failure.
def _prior_operation_failed(self):
from ZODB.POSException import TransactionFailedError
assert self._failure_traceback is not None
raise TransactionFailedError("An operation previously failed, "
"with traceback:\n\n%s" %
self._failure_traceback.getvalue())
def join(self, resource):
if self.status is Status.COMMITFAILED:
self._prior_operation_failed() # doesn't return
if self.status is not Status.ACTIVE:
# TODO: Should it be possible to join a committing transaction?
# I think some users want it.
raise ValueError("expected txn status %r, but it's %r" % (
Status.ACTIVE, self.status))
# TODO: the prepare check is a bit of a hack, perhaps it would
# be better to use interfaces. If this is a ZODB4-style
# resource manager, it needs to be adapted, too.
if myhasattr(resource, "prepare"):
# TODO: deprecate 3.6
resource = DataManagerAdapter(resource)
self._resources.append(resource)
if self._savepoint2index:
# A data manager has joined a transaction *after* a savepoint
# was created. A couple of things are different in this case:
#
# 1. We need to add its savepoint to all previous savepoints.
# so that if they are rolled back, we roll this one back too.
#
# 2. We don't actually need to ask the data manager for a
# savepoint: because it's just joining, we can just abort it to
# roll back to the current state, so we simply use an
# AbortSavepoint.
datamanager_savepoint = AbortSavepoint(resource, self)
for transaction_savepoint in self._savepoint2index.keys():
transaction_savepoint._savepoints.append(
datamanager_savepoint)
def savepoint(self, optimistic=False):
if self.status is Status.COMMITFAILED:
self._prior_operation_failed() # doesn't return, it raises
try:
savepoint = Savepoint(self, optimistic, *self._resources)
except:
self._cleanup(self._resources)
self._saveAndRaiseCommitishError() # reraises!
if self._savepoint2index is None:
self._savepoint2index = weakref.WeakKeyDictionary()
self._savepoint_index += 1
self._savepoint2index[savepoint] = self._savepoint_index
return savepoint
# Remove and invalidate all savepoints we know about with an index
# larger than `savepoint`'s. This is what's needed when a rollback
# _to_ `savepoint` is done.
def _remove_and_invalidate_after(self, savepoint):
savepoint2index = self._savepoint2index
index = savepoint2index[savepoint]
# use items() to make copy to avoid mutating while iterating
for savepoint, i in savepoint2index.items():
if i > index:
savepoint.transaction = None # invalidate
del savepoint2index[savepoint]
# Invalidate and forget about all savepoints.
def _invalidate_all_savepoints(self):
for savepoint in self._savepoint2index.keys():
savepoint.transaction = None # invalidate
self._savepoint2index.clear()
def register(self, obj):
# The old way of registering transaction participants.
#
# register() is passed either a persisent object or a
# resource manager like the ones defined in ZODB.DB.
# If it is passed a persistent object, that object should
# be stored when the transaction commits. For other
# objects, the object implements the standard two-phase
# commit protocol.
manager = getattr(obj, "_p_jar", obj)
if manager is None:
raise ValueError("Register with no manager")
adapter = self._adapters.get(manager)
if adapter is None:
adapter = MultiObjectResourceAdapter(manager)
adapter.objects.append(obj)
self._adapters[manager] = adapter
self.join(adapter)
else:
# TODO: comment out this expensive assert later
# Use id() to guard against proxies.
assert id(obj) not in map(id, adapter.objects)
adapter.objects.append(obj)
def commit(self, subtransaction=_marker, deprecation_wng=True):
if subtransaction is _marker:
subtransaction = 0
elif deprecation_wng:
from ZODB.utils import deprecated37
deprecated37("subtransactions are deprecated; instead of "
"transaction.commit(1), use "
"transaction.savepoint(optimistic=True) in "
"contexts where a subtransaction abort will never "
"occur, or sp=transaction.savepoint() if later "
"rollback is possible and then sp.rollback() "
"instead of transaction.abort(1)")
if self._savepoint2index:
self._invalidate_all_savepoints()
if subtransaction:
# TODO deprecate subtransactions
self._subtransaction_savepoint = self.savepoint(optimistic=True)
return
if self.status is Status.COMMITFAILED:
self._prior_operation_failed() # doesn't return
self._callBeforeCommitHooks()
self._synchronizers.map(lambda s: s.beforeCompletion(self))
self.status = Status.COMMITTING
try:
self._commitResources()
self.status = Status.COMMITTED
except:
t, v, tb = self._saveAndGetCommitishError()
self._callAfterCommitHooks(status=False)
raise t, v, tb
else:
if self._manager:
self._manager.free(self)
self._synchronizers.map(lambda s: s.afterCompletion(self))
self._callAfterCommitHooks(status=True)
self.log.debug("commit")
def _saveAndGetCommitishError(self):
self.status = Status.COMMITFAILED
# Save the traceback for TransactionFailedError.
ft = self._failure_traceback = StringIO()
t, v, tb = sys.exc_info()
# Record how we got into commit().
traceback.print_stack(sys._getframe(1), None, ft)
# Append the stack entries from here down to the exception.
traceback.print_tb(tb, None, ft)
# Append the exception type and value.
ft.writelines(traceback.format_exception_only(t, v))
return t, v, tb
def _saveAndRaiseCommitishError(self):
t, v, tb = self._saveAndGetCommitishError()
raise t, v, tb
def getBeforeCommitHooks(self):
return iter(self._before_commit)
def addBeforeCommitHook(self, hook, args=(), kws=None):
if kws is None:
kws = {}
self._before_commit.append((hook, tuple(args), kws))
def beforeCommitHook(self, hook, *args, **kws):
from ZODB.utils import deprecated38
deprecated38("Use addBeforeCommitHook instead of beforeCommitHook.")
self.addBeforeCommitHook(hook, args, kws)
def _callBeforeCommitHooks(self):
# Call all hooks registered, allowing further registrations
# during processing. Note that calls to addBeforeCommitHook() may
# add additional hooks while hooks are running, and iterating over a
# growing list is well-defined in Python.
for hook, args, kws in self._before_commit:
hook(*args, **kws)
self._before_commit = []
def getAfterCommitHooks(self):
return iter(self._after_commit)
def addAfterCommitHook(self, hook, args=(), kws=None):
if kws is None:
kws = {}
self._after_commit.append((hook, tuple(args), kws))
def _callAfterCommitHooks(self, status=True):
# Avoid to abort anything at the end if no hooks are registred.
if not self._after_commit:
return
# Call all hooks registered, allowing further registrations
# during processing. Note that calls to addAterCommitHook() may
# add additional hooks while hooks are running, and iterating over a
# growing list is well-defined in Python.
for hook, args, kws in self._after_commit:
# The first argument passed to the hook is a Boolean value,
# true if the commit succeeded, or false if the commit aborted.
try:
hook(status, *args, **kws)
except:
# We need to catch the exceptions if we want all hooks
# to be called
self.log.error("Error in after commit hook exec in %s ",
hook, exc_info=sys.exc_info())
# The transaction is already committed. It must not have
# further effects after the commit.
for rm in self._resources:
try:
rm.abort(self)
except:
# XXX should we take further actions here ?
self.log.error("Error in abort() on manager %s",
rm, exc_info=sys.exc_info())
self._after_commit = []
self._before_commit = []
def _commitResources(self):
# Execute the two-phase commit protocol.
L = list(self._resources)
L.sort(rm_cmp)
try:
for rm in L:
rm.tpc_begin(self)
for rm in L:
rm.commit(self)
self.log.debug("commit %r" % rm)
for rm in L:
rm.tpc_vote(self)
self._voted[id(rm)] = True
try:
for rm in L:
rm.tpc_finish(self)
except:
# TODO: do we need to make this warning stronger?
# TODO: It would be nice if the system could be configured
# to stop committing transactions at this point.
self.log.critical("A storage error occurred during the second "
"phase of the two-phase commit. Resources "
"may be in an inconsistent state.")
raise
except:
# If an error occurs committing a transaction, we try
# to revert the changes in each of the resource managers.
t, v, tb = sys.exc_info()
try:
self._cleanup(L)
finally:
self._synchronizers.map(lambda s: s.afterCompletion(self))
raise t, v, tb
def _cleanup(self, L):
# Called when an exception occurs during tpc_vote or tpc_finish.
for rm in L:
if id(rm) not in self._voted:
try:
rm.abort(self)
except Exception:
self.log.error("Error in abort() on manager %s",
rm, exc_info=sys.exc_info())
for rm in L:
try:
rm.tpc_abort(self)
except Exception:
self.log.error("Error in tpc_abort() on manager %s",
rm, exc_info=sys.exc_info())
def abort(self, subtransaction=_marker, deprecation_wng=True):
if subtransaction is _marker:
subtransaction = 0
elif deprecation_wng:
from ZODB.utils import deprecated37
deprecated37("subtransactions are deprecated; use "
"sp.rollback() instead of "
"transaction.abort(1), where `sp` is the "
"corresponding savepoint captured earlier")
if subtransaction:
# TODO deprecate subtransactions.
if not self._subtransaction_savepoint:
raise interfaces.InvalidSavepointRollbackError
if self._subtransaction_savepoint.valid:
self._subtransaction_savepoint.rollback()
# We're supposed to be able to call abort(1) multiple
# times without additional effect, so mark the subtxn
# savepoint invalid now.
self._subtransaction_savepoint.transaction = None
assert not self._subtransaction_savepoint.valid
return
if self._savepoint2index:
self._invalidate_all_savepoints()
self._synchronizers.map(lambda s: s.beforeCompletion(self))
tb = None
for rm in self._resources:
try:
rm.abort(self)
except:
if tb is None:
t, v, tb = sys.exc_info()
self.log.error("Failed to abort resource manager: %s",
rm, exc_info=sys.exc_info())
if self._manager:
self._manager.free(self)
self._synchronizers.map(lambda s: s.afterCompletion(self))
self.log.debug("abort")
if tb is not None:
raise t, v, tb
def note(self, text):
text = text.strip()
if self.description:
self.description += "\n\n" + text
else:
self.description = text
def setUser(self, user_name, path="/"):
self.user = "%s %s" % (path, user_name)
def setExtendedInfo(self, name, value):
self._extension[name] = value
# TODO: We need a better name for the adapters.
class MultiObjectResourceAdapter(object):
"""Adapt the old-style register() call to the new-style join().
With join(), a resource mananger like a Connection registers with
the transaction manager. With register(), an individual object
is passed to register().
"""
def __init__(self, jar):
self.manager = jar
self.objects = []
self.ncommitted = 0
def __repr__(self):
return "<%s for %s at %s>" % (self.__class__.__name__,
self.manager, id(self))
def sortKey(self):
return self.manager.sortKey()
def tpc_begin(self, txn):
self.manager.tpc_begin(txn)
def tpc_finish(self, txn):
self.manager.tpc_finish(txn)
def tpc_abort(self, txn):
self.manager.tpc_abort(txn)
def commit(self, txn):
for o in self.objects:
self.manager.commit(o, txn)
self.ncommitted += 1
def tpc_vote(self, txn):
self.manager.tpc_vote(txn)
def abort(self, txn):
tb = None
for o in self.objects:
try:
self.manager.abort(o, txn)
except:
# Capture the first exception and re-raise it after
# aborting all the other objects.
if tb is None:
t, v, tb = sys.exc_info()
txn.log.error("Failed to abort object: %s",
object_hint(o), exc_info=sys.exc_info())
if tb is not None:
raise t, v, tb
def rm_cmp(rm1, rm2):
return cmp(rm1.sortKey(), rm2.sortKey())
def object_hint(o):
"""Return a string describing the object.
This function does not raise an exception.
"""
from ZODB.utils import oid_repr
# We should always be able to get __class__.
klass = o.__class__.__name__
# oid would be great, but may this isn't a persistent object.
oid = getattr(o, "_p_oid", _marker)
if oid is not _marker:
oid = oid_repr(oid)
return "%s oid=%s" % (klass, oid)
# TODO: deprecate for 3.6.
class DataManagerAdapter(object):
"""Adapt zodb 4-style data managers to zodb3 style
Adapt transaction.interfaces.IDataManager to
ZODB.interfaces.IPureDatamanager
"""
# Note that it is pretty important that this does not have a _p_jar
# attribute. This object will be registered with a zodb3 TM, which
# will then try to get a _p_jar from it, using it as the default.
# (Objects without a _p_jar are their own data managers.)
def __init__(self, datamanager):
self._datamanager = datamanager
# TODO: I'm not sure why commit() doesn't do anything
def commit(self, transaction):
# We don't do anything here because ZODB4-style data managers
# didn't have a separate commit step
pass
def abort(self, transaction):
self._datamanager.abort(transaction)
def tpc_begin(self, transaction):
# We don't do anything here because ZODB4-style data managers
# didn't have a separate tpc_begin step
pass
def tpc_abort(self, transaction):
self._datamanager.abort(transaction)
def tpc_finish(self, transaction):
self._datamanager.commit(transaction)
def tpc_vote(self, transaction):
self._datamanager.prepare(transaction)
def sortKey(self):
return self._datamanager.sortKey()
class Savepoint:
"""Transaction savepoint.
Transaction savepoints coordinate savepoints for data managers
participating in a transaction.
"""
interface.implements(interfaces.ISavepoint)
valid = property(lambda self: self.transaction is not None)
def __init__(self, transaction, optimistic, *resources):
self.transaction = transaction
self._savepoints = savepoints = []
for datamanager in resources:
try:
savepoint = datamanager.savepoint
except AttributeError:
if not optimistic:
raise TypeError("Savepoints unsupported", datamanager)
savepoint = NoRollbackSavepoint(datamanager)
else:
savepoint = savepoint()
savepoints.append(savepoint)
def rollback(self):
transaction = self.transaction
if transaction is None:
raise interfaces.InvalidSavepointRollbackError
transaction._remove_and_invalidate_after(self)
try:
for savepoint in self._savepoints:
savepoint.rollback()
except:
# Mark the transaction as failed.
transaction._saveAndRaiseCommitishError() # reraises!
class AbortSavepoint:
def __init__(self, datamanager, transaction):
self.datamanager = datamanager
self.transaction = transaction
def rollback(self):
self.datamanager.abort(self.transaction)
class NoRollbackSavepoint:
def __init__(self, datamanager):
self.datamanager = datamanager
def rollback(self):
raise TypeError("Savepoints unsupported", self.datamanager)
ZODB/src/scripts/ 0000751 0001750 0000144 00000000000 10424632373 013444 5 ustar ctheune users ZODB/src/scripts/manual_tests/ 0000751 0001750 0000144 00000000000 10424632373 016143 5 ustar ctheune users ZODB/src/scripts/manual_tests/test-checker.fs 0000640 0001750 0000144 00000001442 10050174114 021044 0 ustar ctheune users FS21?ž4\B3 “ initial database creation ?ž4\B3 9(cPersistence
PersistentMapping
qNt.}qU
_containerq}s. “?ž4™"£™ š ?ž4™"£™ 4 Ÿ Y((U PersistenceqUPersistentMappingqtqNt.}qU
_containerq}qUaUThis is a test.qss. š?ž5±\w 1 ?ž5±\w ¶ A „((U PersistenceqUPersistentMappingqtqNt.}qU
_containerq}q(UaUThis is a test.qK(U q(hUPersistentMappingq ttQus. ?ž5±\w A B((U PersistenceqUPersistentMappingqtqNt.}qU
_containerq}qs. 1?ž5.5°w ?ž5.5°w z _((U PersistenceqUPersistentMappingqtqNt.}qU
_containerq}qUaUThis is another test.qss. ZODB/src/scripts/manual_tests/testfstest.py 0000640 0001750 0000144 00000012646 10214421637 020732 0 ustar ctheune users """Verify that fstest.py can find errors.
Note: To run this test script fstest.py must be on your PYTHONPATH.
"""
from cStringIO import StringIO
import os
import re
import struct
import tempfile
import unittest
import fstest
from fstest import FormatError, U64
class TestCorruptedFS(unittest.TestCase):
f = open('test-checker.fs', 'rb')
datafs = f.read()
f.close()
del f
def setUp(self):
self._temp = tempfile.mktemp()
self._file = open(self._temp, 'wb')
def tearDown(self):
if not self._file.closed:
self._file.close()
if os.path.exists(self._temp):
try:
os.remove(self._temp)
except os.error:
pass
def noError(self):
if not self._file.closed:
self._file.close()
fstest.check(self._temp)
def detectsError(self, rx):
if not self._file.closed:
self._file.close()
try:
fstest.check(self._temp)
except FormatError, msg:
mo = re.search(rx, str(msg))
self.failIf(mo is None, "unexpected error: %s" % msg)
else:
self.fail("fstest did not detect corruption")
def getHeader(self):
buf = self._datafs.read(16)
if not buf:
return 0, ''
tl = U64(buf[8:])
return tl, buf
def copyTransactions(self, n):
"""Copy at most n transactions from the good data"""
f = self._datafs = StringIO(self.datafs)
self._file.write(f.read(4))
for i in range(n):
tl, data = self.getHeader()
if not tl:
return
self._file.write(data)
rec = f.read(tl - 8)
self._file.write(rec)
def testGood(self):
self._file.write(self.datafs)
self.noError()
def testTwoTransactions(self):
self.copyTransactions(2)
self.noError()
def testEmptyFile(self):
self.detectsError("empty file")
def testInvalidHeader(self):
self._file.write('SF12')
self.detectsError("invalid file header")
def testTruncatedTransaction(self):
self._file.write(self.datafs[:4+22])
self.detectsError("truncated")
def testCheckpointFlag(self):
self.copyTransactions(2)
tl, data = self.getHeader()
assert tl > 0, "ran out of good transaction data"
self._file.write(data)
self._file.write('c')
self._file.write(self._datafs.read(tl - 9))
self.detectsError("checkpoint flag")
def testInvalidStatus(self):
self.copyTransactions(2)
tl, data = self.getHeader()
assert tl > 0, "ran out of good transaction data"
self._file.write(data)
self._file.write('Z')
self._file.write(self._datafs.read(tl - 9))
self.detectsError("invalid status")
def testTruncatedRecord(self):
self.copyTransactions(3)
tl, data = self.getHeader()
assert tl > 0, "ran out of good transaction data"
self._file.write(data)
buf = self._datafs.read(tl / 2)
self._file.write(buf)
self.detectsError("truncated possibly")
def testBadLength(self):
self.copyTransactions(2)
tl, data = self.getHeader()
assert tl > 0, "ran out of good transaction data"
self._file.write(data)
buf = self._datafs.read(tl - 8)
self._file.write(buf[0])
assert tl <= 1<<16, "can't use this transaction for this test"
self._file.write("\777\777")
self._file.write(buf[3:])
self.detectsError("invalid transaction header")
def testDecreasingTimestamps(self):
self.copyTransactions(0)
tl, data = self.getHeader()
buf = self._datafs.read(tl - 8)
t1 = data + buf
tl, data = self.getHeader()
buf = self._datafs.read(tl - 8)
t2 = data + buf
self._file.write(t2[:8] + t1[8:])
self._file.write(t1[:8] + t2[8:])
self.detectsError("time-stamp")
def testTruncatedData(self):
# This test must re-write the transaction header length in
# order to trigger the error in check_drec(). If it doesn't,
# the truncated data record would also caught a truncated
# transaction record.
self.copyTransactions(1)
tl, data = self.getHeader()
pos = self._file.tell()
self._file.write(data)
buf = self._datafs.read(tl - 8)
hdr = buf[:15]
ul, dl, el = struct.unpack(">HHH", hdr[-6:])
self._file.write(buf[:15 + ul + dl + el])
data = buf[15 + ul + dl + el:]
self._file.write(data[:24])
self._file.seek(pos + 8, 0)
newlen = struct.pack(">II", 0, tl - (len(data) - 24))
self._file.write(newlen)
self.detectsError("truncated at")
def testBadDataLength(self):
self.copyTransactions(1)
tl, data = self.getHeader()
self._file.write(data)
buf = self._datafs.read(tl - 8)
hdr = buf[:7]
# write the transaction meta data
ul, dl, el = struct.unpack(">HHH", hdr[-6:])
self._file.write(buf[:7 + ul + dl + el])
# write the first part of the data header
data = buf[7 + ul + dl + el:]
self._file.write(data[:24])
self._file.write("\000" * 4 + "\077" + "\000" * 3)
self._file.write(data[32:])
self.detectsError("record exceeds transaction")
if __name__ == "__main__":
unittest.main()
ZODB/src/scripts/manual_tests/testzeopack.py 0000640 0001750 0000144 00000006456 10214421637 021060 0 ustar ctheune users # Some simple tests for zeopack.py
# For this to work, zeopack.py must by on your PATH.
from ZODB.FileStorage import FileStorage
from ZODB.tests.StorageTestBase import StorageTestBase
from ZEO.tests import forker
import ZODB
import os
import socket
import tempfile
import threading
import time
import unittest
# TODO: The forker interface isn't clearly defined. It's different on
# different branches of ZEO. This will break someday.
# TODO: Only handle the Unix variant of the forker. Just to give Tim
# something to do.
class PackerTests(StorageTestBase):
def setUp(self):
self.started = 0
def start(self):
self.started =1
self.path = tempfile.mktemp(suffix=".fs")
self._storage = FileStorage(self.path)
self.db = ZODB.DB(self._storage)
self.do_updates()
self.pid, self.exit = forker.start_zeo_server(self._storage, self.addr)
def do_updates(self):
for i in range(100):
self._dostore()
def tearDown(self):
if not self.started:
return
self.db.close()
self._storage.close()
self.exit.close()
try:
os.kill(self.pid, 9)
except os.error:
pass
try:
os.waitpid(self.pid, 0)
except os.error, err:
##print "waitpid failed", err
pass
for ext in '', '.old', '.lock', '.index', '.tmp':
path = self.path + ext
try:
os.remove(path)
except os.error:
pass
def set_inet_addr(self):
self.host = socket.gethostname()
self.port = forker.get_port()
self.addr = self.host, self.port
def testPack(self):
self.set_inet_addr()
self.start()
status = os.system("zeopack.py -h %s -p %s" % (self.host, self.port))
assert status == 0
assert os.path.exists(self.path + ".old")
def testPackDays(self):
self.set_inet_addr()
self.start()
status = os.system("zeopack.py -h %s -p %s -d 1" % (self.host,
self.port))
# Since we specified one day, nothing should get packed
assert status == 0
assert not os.path.exists(self.path + ".old")
def testAF_UNIXPack(self):
self.addr = tempfile.mktemp(suffix=".zeo-socket")
self.start()
status = os.system("zeopack.py -U %s" % self.addr)
assert status == 0
assert os.path.exists(self.path + ".old")
def testNoServer(self):
status = os.system("zeopack.py -p 19")
assert status != 0
def testWaitForServer(self):
self.set_inet_addr()
def delayed_start():
time.sleep(11)
self.start()
t = threading.Thread(target=delayed_start)
t.start()
status = os.system("zeopack.py -h %s -p %s -W" % (self.host,
self.port))
t.join()
assert status == 0
assert os.path.exists(self.path + ".old")
class UpTest(unittest.TestCase):
def testUp(self):
status = os.system("zeoup.py -p 19")
# There is no ZEO server on port 19, so we should see non-zero
# exit status.
assert status != 0
if __name__ == "__main__":
unittest.main()
ZODB/src/scripts/manual_tests/testrepozo.py 0000640 0001750 0000144 00000010520 10220102506 020711 0 ustar ctheune users #!/usr/bin/env python
##############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Test repozo.py.
This is a by-hand test. It succeeds iff it doesn't blow up. Run it with
its home directory as the current directory. It will destroy all files
matching Data.* and Copy.* in this directory, and anything in a
subdirectory of name 'backup'.
"""
import os
import random
import time
import glob
import sys
import shutil
import ZODB
from ZODB import FileStorage
import transaction
PYTHON = sys.executable + ' '
def cleanup():
for fname in glob.glob('Data.*') + glob.glob('Copy.*'):
os.remove(fname)
if os.path.isdir('backup'):
for fname in os.listdir('backup'):
os.remove(os.path.join('backup', fname))
os.rmdir('backup')
class OurDB:
def __init__(self):
from BTrees.OOBTree import OOBTree
self.getdb()
conn = self.db.open()
conn.root()['tree'] = OOBTree()
transaction.commit()
self.close()
def getdb(self):
storage = FileStorage.FileStorage('Data.fs')
self.db = ZODB.DB(storage)
def gettree(self):
self.getdb()
conn = self.db.open()
return conn.root()['tree']
def pack(self):
self.getdb()
self.db.pack()
def close(self):
if self.db is not None:
self.db.close()
self.db = None
# Do recovery to time 'when', and check that it's identical to correctpath.
def check(correctpath='Data.fs', when=None):
if when is None:
extra = ''
else:
extra = ' -D ' + when
cmd = PYTHON + '../repozo.py -vRr backup -o Copy.fs' + extra
os.system(cmd)
f = file(correctpath, 'rb')
g = file('Copy.fs', 'rb')
fguts = f.read()
gguts = g.read()
f.close()
g.close()
if fguts != gguts:
raise ValueError("guts don't match\n"
" correctpath=%r when=%r\n"
" cmd=%r" % (correctpath, when, cmd))
def mutatedb(db):
# Make random mutations to the btree in the database.
tree = db.gettree()
for dummy in range(100):
if random.random() < 0.6:
tree[random.randrange(100000)] = random.randrange(100000)
else:
keys = tree.keys()
if keys:
del tree[keys[0]]
transaction.commit()
db.close()
def main():
cleanup()
os.mkdir('backup')
d = OurDB()
# Every 9th time thru the loop, we save a full copy of Data.fs,
# and at the end we ensure we can reproduce those too.
saved_snapshots = [] # list of (name, time) pairs for copies.
for i in range(100):
# Make some mutations.
mutatedb(d)
# Pack about each tenth time.
if random.random() < 0.1:
print "packing"
d.pack()
d.close()
# Make an incremental backup, half the time with gzip (-z).
if random.random() < 0.5:
os.system(PYTHON + '../repozo.py -vBQr backup -f Data.fs')
else:
os.system(PYTHON + '../repozo.py -zvBQr backup -f Data.fs')
if i % 9 == 0:
copytime = '%04d-%02d-%02d-%02d-%02d-%02d' % (time.gmtime()[:6])
copyname = os.path.join('backup', "Data%d" % i) + '.fs'
shutil.copyfile('Data.fs', copyname)
saved_snapshots.append((copyname, copytime))
# Make sure the clock moves at least a second.
time.sleep(1.01)
# Verify current Data.fs can be reproduced exactly.
check()
# Verify snapshots can be reproduced exactly.
for copyname, copytime in saved_snapshots:
print "Checking that", copyname, "at", copytime, "is reproducible."
check(copyname, copytime)
# Tear it all down.
cleanup()
print 'Test passed!'
if __name__ == '__main__':
main()
ZODB/src/scripts/zodbload.py 0000640 0001750 0000144 00000074417 10216353117 015624 0 ustar ctheune users #!/usr/bin/env python2.3
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Test script for testing ZODB under a heavy zope-like load.
Note that, to be as realistic as possible with ZEO, you should run this
script multiple times, to simulate multiple clients.
Here's how this works.
The script starts some number of threads. Each thread, sequentially
executes jobs. There is a job producer that produces jobs.
Input data are provided by a mail producer that hands out message from
a mailbox.
Execution continues until there is an error, which will normally occur
when the mailbox is exhausted.
Command-line options are used to provide job definitions. Job
definitions have perameters of the form name=value. Jobs have 2
standard parameters:
frequency=integer
The frequency of the job. The default is 1.
sleep=float
The number os seconds to sleep before performing the job. The
default is 0.
Usage: loadmail2 [options]
Options:
-edit [frequency=integer] [sleep=float]
Define an edit job. An edit job edits a random already-saved
email message, deleting and inserting a random number of words.
After editing the message, the message is (re)cataloged.
-insert [number=int] [frequency=integer] [sleep=float]
Insert some number of email messages.
-index [number=int] [frequency=integer] [sleep=float]
Insert and index (catalog) some number of email messages.
-search [terms='word1 word2 ...'] [frequency=integer] [sleep=float]
Search the catalog. A query is givem with one or more terms as
would be entered into a typical seach box. If no query is
given, then queries will be randomly selected based on a set of
built-in word list.
-setup
Set up the database. This will delete any existing Data.fs
file. (Of course, this may have no effect, if there is a
custom_zodb that defined a different storage.) It also adds a
mail folder and a catalog.
-options file
Read options from the given file. Th efile should be a python
source file that defines a sequence of options named 'options'.
-threads n
Specify the number of threads to execute. If not specified (< 2),
then jobs are run in a single (main) thread.
-mbox filename
Specify the mailbox for getting input data.
There is a (lame) syntax for providing options within the
filename. The filename may be followed by up to 3 integers,
min, max, and start:
-mbox 'foo.mbox 0 100 10000'
The messages from min to max will be read from the mailbox.
They will be assigned message numbers starting with start.
So, in the example above, we read the first hundred messages
and assign thgem message numbers starting with 10001.
The maxmum can be given as a negative number, in which case, it
specifies the number of messages to read.
The start defaults to the minimum. The following two options:
-mbox 'foo.mbox 300 400 300'
and
-mbox 'foo.mbox 300 -100'
are equivalent
$Id: zodbload.py 29533 2005-03-17 19:11:11Z tim_one $
"""
import mailbox
import math
import os
import random
import re
import sys
import threading
import time
import transaction
class JobProducer:
def __init__(self):
self.jobs = []
def add(self, callable, frequency, sleep, repeatp=0):
self.jobs.extend([(callable, sleep, repeatp)] * int(frequency))
random.shuffle(self.jobs)
def next(self):
factory, sleep, repeatp = random.choice(self.jobs)
time.sleep(sleep)
callable, args = factory.create()
return factory, callable, args, repeatp
def __nonzero__(self):
return not not self.jobs
class MBox:
def __init__(self, filename):
if ' ' in filename:
filename = filename.split()
if len(filename) < 4:
filename += [0, 0, -1][-(4-len(filename)):]
filename, min, max, start = filename
min = int(min)
max = int(max)
start = int(start)
if start < 0:
start = min
if max < 0:
# negative max is treated as a count
self._max = start - max
elif max > 0:
self._max = start + max - min
else:
self._max = 0
else:
self._max = 0
min = start = 0
if filename.endswith('.bz2'):
f = os.popen("bunzip2 <"+filename, 'r')
filename = filename[-4:]
else:
f = open(filename)
self._mbox = mb = mailbox.UnixMailbox(f)
self.number = start
while min:
mb.next()
min -= 1
self._lock = threading.Lock()
self.__name__ = os.path.splitext(os.path.split(filename)[1])[0]
self._max = max
def next(self):
self._lock.acquire()
try:
if self._max > 0 and self.number >= self._max:
raise IndexError(self.number + 1)
message = self._mbox.next()
message.body = message.fp.read()
message.headers = list(message.headers)
self.number += 1
message.number = self.number
message.mbox = self.__name__
return message
finally:
self._lock.release()
bins = 9973
#bins = 11
def mailfolder(app, mboxname, number):
mail = getattr(app, mboxname, None)
if mail is None:
app.manage_addFolder(mboxname)
mail = getattr(app, mboxname)
from BTrees.Length import Length
mail.length = Length()
for i in range(bins):
mail.manage_addFolder('b'+str(i))
bin = hash(str(number))%bins
return getattr(mail, 'b'+str(bin))
def VmSize():
try:
f = open('/proc/%s/status' % os.getpid())
except:
return 0
else:
l = filter(lambda l: l[:7] == 'VmSize:', f.readlines())
if l:
l = l[0][7:].strip().split()[0]
return int(l)
return 0
def setup(lib_python):
try:
os.remove(os.path.join(lib_python, '..', '..', 'var', 'Data.fs'))
except:
pass
import Zope2
import Products
import AccessControl.SecurityManagement
app=Zope2.app()
Products.ZCatalog.ZCatalog.manage_addZCatalog(app, 'cat', '')
from Products.ZCTextIndex.ZCTextIndex import PLexicon
from Products.ZCTextIndex.Lexicon import Splitter, CaseNormalizer
app.cat._setObject('lex',
PLexicon('lex', '', Splitter(), CaseNormalizer())
)
class extra:
doc_attr = 'PrincipiaSearchSource'
lexicon_id = 'lex'
index_type = 'Okapi BM25 Rank'
app.cat.addIndex('PrincipiaSearchSource', 'ZCTextIndex', extra)
transaction.commit()
system = AccessControl.SpecialUsers.system
AccessControl.SecurityManagement.newSecurityManager(None, system)
app._p_jar.close()
def do(db, f, args):
"""Do something in a transaction, retrying of necessary
Measure the speed of both the compurartion and the commit
"""
from ZODB.POSException import ConflictError
wcomp = ccomp = wcommit = ccommit = 0.0
rconflicts = wconflicts = 0
start = time.time()
while 1:
connection = db.open()
try:
transaction.begin()
t=time.time()
c=time.clock()
try:
try:
r = f(connection, *args)
except ConflictError:
rconflicts += 1
transaction.abort()
continue
finally:
wcomp += time.time() - t
ccomp += time.clock() - c
t=time.time()
c=time.clock()
try:
try:
transaction.commit()
break
except ConflictError:
wconflicts += 1
transaction.abort()
continue
finally:
wcommit += time.time() - t
ccommit += time.clock() - c
finally:
connection.close()
return start, wcomp, ccomp, rconflicts, wconflicts, wcommit, ccommit, r
def run1(tid, db, factory, job, args):
(start, wcomp, ccomp, rconflicts, wconflicts, wcommit, ccommit, r
) = do(db, job, args)
start = "%.4d-%.2d-%.2d %.2d:%.2d:%.2d" % time.localtime(start)[:6]
print "%s %s %8.3g %8.3g %s %s\t%8.3g %8.3g %s %r" % (
start, tid, wcomp, ccomp, rconflicts, wconflicts, wcommit, ccommit,
factory.__name__, r)
def run(jobs, tid=''):
import Zope2
while 1:
factory, job, args, repeatp = jobs.next()
run1(tid, Zope2.DB, factory, job, args)
if repeatp:
while 1:
i = random.randint(0,100)
if i > repeatp:
break
run1(tid, Zope2.DB, factory, job, args)
def index(connection, messages, catalog, max):
app = connection.root()['Application']
for message in messages:
mail = mailfolder(app, message.mbox, message.number)
if max:
# Cheat and use folder implementation secrets
# to avoid having to read the old data
_objects = mail._objects
if len(_objects) >= max:
for d in _objects[:len(_objects)-max+1]:
del mail.__dict__[d['id']]
mail._objects = _objects[len(_objects)-max+1:]
docid = 'm'+str(message.number)
mail.manage_addDTMLDocument(docid, file=message.body)
# increment counted
getattr(app, message.mbox).length.change(1)
doc = mail[docid]
for h in message.headers:
h = h.strip()
l = h.find(':')
if l <= 0:
continue
name = h[:l].lower()
if name=='subject':
name='title'
v = h[l+1:].strip()
type='string'
if name=='title':
doc.manage_changeProperties(title=h)
else:
try:
doc.manage_addProperty(name, v, type)
except:
pass
if catalog:
app.cat.catalog_object(doc)
return message.number
class IndexJob:
needs_mbox = 1
catalog = 1
prefix = 'index'
def __init__(self, mbox, number=1, max=0):
self.__name__ = "%s%s_%s" % (self.prefix, number, mbox.__name__)
self.mbox, self.number, self.max = mbox, int(number), int(max)
def create(self):
messages = [self.mbox.next() for i in range(self.number)]
return index, (messages, self.catalog, self.max)
class InsertJob(IndexJob):
catalog = 0
prefix = 'insert'
wordre = re.compile(r'(\w{3,20})')
stop = 'and', 'not'
def edit(connection, mbox, catalog=1):
app = connection.root()['Application']
mail = getattr(app, mbox.__name__, None)
if mail is None:
time.sleep(1)
return "No mailbox %s" % mbox.__name__
nmessages = mail.length()
if nmessages < 2:
time.sleep(1)
return "No messages to edit in %s" % mbox.__name__
# find a message to edit:
while 1:
number = random.randint(1, nmessages-1)
did = 'm' + str(number)
mail = mailfolder(app, mbox.__name__, number)
doc = getattr(mail, did, None)
if doc is not None:
break
text = doc.raw.split()
norig = len(text)
if norig > 10:
ndel = int(math.exp(random.randint(0, int(math.log(norig)))))
nins = int(math.exp(random.randint(0, int(math.log(norig)))))
else:
ndel = 0
nins = 10
for j in range(ndel):
j = random.randint(0,len(text)-1)
word = text[j]
m = wordre.search(word)
if m:
word = m.group(1).lower()
if (not wordsd.has_key(word)) and word not in stop:
words.append(word)
wordsd[word] = 1
del text[j]
for j in range(nins):
word = random.choice(words)
text.append(word)
doc.raw = ' '.join(text)
if catalog:
app.cat.catalog_object(doc)
return norig, ndel, nins
class EditJob:
needs_mbox = 1
prefix = 'edit'
catalog = 1
def __init__(self, mbox):
self.__name__ = "%s_%s" % (self.prefix, mbox.__name__)
self.mbox = mbox
def create(self):
return edit, (self.mbox, self.catalog)
class ModifyJob(EditJob):
prefix = 'modify'
catalog = 0
def search(connection, terms, number):
app = connection.root()['Application']
cat = app.cat
n = 0
for i in number:
term = random.choice(terms)
results = cat(PrincipiaSearchSource=term)
n += len(results)
for result in results:
obj = result.getObject()
# Apparently, there is a bug in Zope that leads obj to be None
# on occasion.
if obj is not None:
obj.getId()
return n
class SearchJob:
def __init__(self, terms='', number=10):
if terms:
terms = terms.split()
self.__name__ = "search_" + '_'.join(terms)
self.terms = terms
else:
self.__name__ = 'search'
self.terms = words
number = min(int(number), len(self.terms))
self.number = range(number)
def create(self):
return search, (self.terms, self.number)
words=['banishment', 'indirectly', 'imprecise', 'peeks',
'opportunely', 'bribe', 'sufficiently', 'Occidentalized', 'elapsing',
'fermenting', 'listen', 'orphanage', 'younger', 'draperies', 'Ida',
'cuttlefish', 'mastermind', 'Michaels', 'populations', 'lent',
'cater', 'attentional', 'hastiness', 'dragnet', 'mangling',
'scabbards', 'princely', 'star', 'repeat', 'deviation', 'agers',
'fix', 'digital', 'ambitious', 'transit', 'jeeps', 'lighted',
'Prussianizations', 'Kickapoo', 'virtual', 'Andrew', 'generally',
'boatsman', 'amounts', 'promulgation', 'Malay', 'savaging',
'courtesan', 'nursed', 'hungered', 'shiningly', 'ship', 'presides',
'Parke', 'moderns', 'Jonas', 'unenlightening', 'dearth', 'deer',
'domesticates', 'recognize', 'gong', 'penetrating', 'dependents',
'unusually', 'complications', 'Dennis', 'imbalances', 'nightgown',
'attached', 'testaments', 'congresswoman', 'circuits', 'bumpers',
'braver', 'Boreas', 'hauled', 'Howe', 'seethed', 'cult', 'numismatic',
'vitality', 'differences', 'collapsed', 'Sandburg', 'inches', 'head',
'rhythmic', 'opponent', 'blanketer', 'attorneys', 'hen', 'spies',
'indispensably', 'clinical', 'redirection', 'submit', 'catalysts',
'councilwoman', 'kills', 'topologies', 'noxious', 'exactions',
'dashers', 'balanced', 'slider', 'cancerous', 'bathtubs', 'legged',
'respectably', 'crochets', 'absenteeism', 'arcsine', 'facility',
'cleaners', 'bobwhite', 'Hawkins', 'stockade', 'provisional',
'tenants', 'forearms', 'Knowlton', 'commit', 'scornful',
'pediatrician', 'greets', 'clenches', 'trowels', 'accepts',
'Carboloy', 'Glenn', 'Leigh', 'enroll', 'Madison', 'Macon', 'oiling',
'entertainingly', 'super', 'propositional', 'pliers', 'beneficiary',
'hospitable', 'emigration', 'sift', 'sensor', 'reserved',
'colonization', 'shrilled', 'momentously', 'stevedore', 'Shanghaiing',
'schoolmasters', 'shaken', 'biology', 'inclination', 'immoderate',
'stem', 'allegory', 'economical', 'daytime', 'Newell', 'Moscow',
'archeology', 'ported', 'scandals', 'Blackfoot', 'leery', 'kilobit',
'empire', 'obliviousness', 'productions', 'sacrificed', 'ideals',
'enrolling', 'certainties', 'Capsicum', 'Brookdale', 'Markism',
'unkind', 'dyers', 'legislates', 'grotesquely', 'megawords',
'arbitrary', 'laughing', 'wildcats', 'thrower', 'sex', 'devils',
'Wehr', 'ablates', 'consume', 'gossips', 'doorways', 'Shari',
'advanced', 'enumerable', 'existentially', 'stunt', 'auctioneers',
'scheduler', 'blanching', 'petulance', 'perceptibly', 'vapors',
'progressed', 'rains', 'intercom', 'emergency', 'increased',
'fluctuating', 'Krishna', 'silken', 'reformed', 'transformation',
'easter', 'fares', 'comprehensible', 'trespasses', 'hallmark',
'tormenter', 'breastworks', 'brassiere', 'bladders', 'civet', 'death',
'transformer', 'tolerably', 'bugle', 'clergy', 'mantels', 'satin',
'Boswellizes', 'Bloomington', 'notifier', 'Filippo', 'circling',
'unassigned', 'dumbness', 'sentries', 'representativeness', 'souped',
'Klux', 'Kingstown', 'gerund', 'Russell', 'splices', 'bellow',
'bandies', 'beefers', 'cameramen', 'appalled', 'Ionian', 'butterball',
'Portland', 'pleaded', 'admiringly', 'pricks', 'hearty', 'corer',
'deliverable', 'accountably', 'mentors', 'accorded',
'acknowledgement', 'Lawrenceville', 'morphology', 'eucalyptus',
'Rena', 'enchanting', 'tighter', 'scholars', 'graduations', 'edges',
'Latinization', 'proficiency', 'monolithic', 'parenthesizing', 'defy',
'shames', 'enjoyment', 'Purdue', 'disagrees', 'barefoot', 'maims',
'flabbergast', 'dishonorable', 'interpolation', 'fanatics', 'dickens',
'abysses', 'adverse', 'components', 'bowl', 'belong', 'Pipestone',
'trainees', 'paw', 'pigtail', 'feed', 'whore', 'conditioner',
'Volstead', 'voices', 'strain', 'inhabits', 'Edwin', 'discourses',
'deigns', 'cruiser', 'biconvex', 'biking', 'depreciation', 'Harrison',
'Persian', 'stunning', 'agar', 'rope', 'wagoner', 'elections',
'reticulately', 'Cruz', 'pulpits', 'wilt', 'peels', 'plants',
'administerings', 'deepen', 'rubs', 'hence', 'dissension', 'implored',
'bereavement', 'abyss', 'Pennsylvania', 'benevolent', 'corresponding',
'Poseidon', 'inactive', 'butchers', 'Mach', 'woke', 'loading',
'utilizing', 'Hoosier', 'undo', 'Semitization', 'trigger', 'Mouthe',
'mark', 'disgracefully', 'copier', 'futility', 'gondola', 'algebraic',
'lecturers', 'sponged', 'instigators', 'looted', 'ether', 'trust',
'feeblest', 'sequencer', 'disjointness', 'congresses', 'Vicksburg',
'incompatibilities', 'commend', 'Luxembourg', 'reticulation',
'instructively', 'reconstructs', 'bricks', 'attache', 'Englishman',
'provocation', 'roughen', 'cynic', 'plugged', 'scrawls', 'antipode',
'injected', 'Daedalus', 'Burnsides', 'asker', 'confronter',
'merriment', 'disdain', 'thicket', 'stinker', 'great', 'tiers',
'oust', 'antipodes', 'Macintosh', 'tented', 'packages',
'Mediterraneanize', 'hurts', 'orthodontist', 'seeder', 'readying',
'babying', 'Florida', 'Sri', 'buckets', 'complementary',
'cartographer', 'chateaus', 'shaves', 'thinkable', 'Tehran',
'Gordian', 'Angles', 'arguable', 'bureau', 'smallest', 'fans',
'navigated', 'dipole', 'bootleg', 'distinctive', 'minimization',
'absorbed', 'surmised', 'Malawi', 'absorbent', 'close', 'conciseness',
'hopefully', 'declares', 'descent', 'trick', 'portend', 'unable',
'mildly', 'Morse', 'reference', 'scours', 'Caribbean', 'battlers',
'astringency', 'likelier', 'Byronizes', 'econometric', 'grad',
'steak', 'Austrian', 'ban', 'voting', 'Darlington', 'bison', 'Cetus',
'proclaim', 'Gilbertson', 'evictions', 'submittal', 'bearings',
'Gothicizer', 'settings', 'McMahon', 'densities', 'determinants',
'period', 'DeKastere', 'swindle', 'promptness', 'enablers', 'wordy',
'during', 'tables', 'responder', 'baffle', 'phosgene', 'muttering',
'limiters', 'custodian', 'prevented', 'Stouffer', 'waltz', 'Videotex',
'brainstorms', 'alcoholism', 'jab', 'shouldering', 'screening',
'explicitly', 'earner', 'commandment', 'French', 'scrutinizing',
'Gemma', 'capacitive', 'sheriff', 'herbivore', 'Betsey', 'Formosa',
'scorcher', 'font', 'damming', 'soldiers', 'flack', 'Marks',
'unlinking', 'serenely', 'rotating', 'converge', 'celebrities',
'unassailable', 'bawling', 'wording', 'silencing', 'scotch',
'coincided', 'masochists', 'graphs', 'pernicious', 'disease',
'depreciates', 'later', 'torus', 'interject', 'mutated', 'causer',
'messy', 'Bechtel', 'redundantly', 'profoundest', 'autopsy',
'philosophic', 'iterate', 'Poisson', 'horridly', 'silversmith',
'millennium', 'plunder', 'salmon', 'missioner', 'advances', 'provers',
'earthliness', 'manor', 'resurrectors', 'Dahl', 'canto', 'gangrene',
'gabler', 'ashore', 'frictionless', 'expansionism', 'emphasis',
'preservations', 'Duane', 'descend', 'isolated', 'firmware',
'dynamites', 'scrawled', 'cavemen', 'ponder', 'prosperity', 'squaw',
'vulnerable', 'opthalmic', 'Simms', 'unite', 'totallers', 'Waring',
'enforced', 'bridge', 'collecting', 'sublime', 'Moore', 'gobble',
'criticizes', 'daydreams', 'sedate', 'apples', 'Concordia',
'subsequence', 'distill', 'Allan', 'seizure', 'Isadore', 'Lancashire',
'spacings', 'corresponded', 'hobble', 'Boonton', 'genuineness',
'artifact', 'gratuities', 'interviewee', 'Vladimir', 'mailable',
'Bini', 'Kowalewski', 'interprets', 'bereave', 'evacuated', 'friend',
'tourists', 'crunched', 'soothsayer', 'fleetly', 'Romanizations',
'Medicaid', 'persevering', 'flimsy', 'doomsday', 'trillion',
'carcasses', 'guess', 'seersucker', 'ripping', 'affliction',
'wildest', 'spokes', 'sheaths', 'procreate', 'rusticates', 'Schapiro',
'thereafter', 'mistakenly', 'shelf', 'ruination', 'bushel',
'assuredly', 'corrupting', 'federation', 'portmanteau', 'wading',
'incendiary', 'thing', 'wanderers', 'messages', 'Paso', 'reexamined',
'freeings', 'denture', 'potting', 'disturber', 'laborer', 'comrade',
'intercommunicating', 'Pelham', 'reproach', 'Fenton', 'Alva', 'oasis',
'attending', 'cockpit', 'scout', 'Jude', 'gagging', 'jailed',
'crustaceans', 'dirt', 'exquisitely', 'Internet', 'blocker', 'smock',
'Troutman', 'neighboring', 'surprise', 'midscale', 'impart',
'badgering', 'fountain', 'Essen', 'societies', 'redresses',
'afterwards', 'puckering', 'silks', 'Blakey', 'sequel', 'greet',
'basements', 'Aubrey', 'helmsman', 'album', 'wheelers', 'easternmost',
'flock', 'ambassadors', 'astatine', 'supplant', 'gird', 'clockwork',
'foxes', 'rerouting', 'divisional', 'bends', 'spacer',
'physiologically', 'exquisite', 'concerts', 'unbridled', 'crossing',
'rock', 'leatherneck', 'Fortescue', 'reloading', 'Laramie', 'Tim',
'forlorn', 'revert', 'scarcer', 'spigot', 'equality', 'paranormal',
'aggrieves', 'pegs', 'committeewomen', 'documented', 'interrupt',
'emerald', 'Battelle', 'reconverted', 'anticipated', 'prejudices',
'drowsiness', 'trivialities', 'food', 'blackberries', 'Cyclades',
'tourist', 'branching', 'nugget', 'Asilomar', 'repairmen', 'Cowan',
'receptacles', 'nobler', 'Nebraskan', 'territorial', 'chickadee',
'bedbug', 'darted', 'vigilance', 'Octavia', 'summands', 'policemen',
'twirls', 'style', 'outlawing', 'specifiable', 'pang', 'Orpheus',
'epigram', 'Babel', 'butyrate', 'wishing', 'fiendish', 'accentuate',
'much', 'pulsed', 'adorned', 'arbiters', 'counted', 'Afrikaner',
'parameterizes', 'agenda', 'Americanism', 'referenda', 'derived',
'liquidity', 'trembling', 'lordly', 'Agway', 'Dillon', 'propellers',
'statement', 'stickiest', 'thankfully', 'autograph', 'parallel',
'impulse', 'Hamey', 'stylistic', 'disproved', 'inquirer', 'hoisting',
'residues', 'variant', 'colonials', 'dequeued', 'especial', 'Samoa',
'Polaris', 'dismisses', 'surpasses', 'prognosis', 'urinates',
'leaguers', 'ostriches', 'calculative', 'digested', 'divided',
'reconfigurer', 'Lakewood', 'illegalities', 'redundancy',
'approachability', 'masterly', 'cookery', 'crystallized', 'Dunham',
'exclaims', 'mainline', 'Australianizes', 'nationhood', 'pusher',
'ushers', 'paranoia', 'workstations', 'radiance', 'impedes',
'Minotaur', 'cataloging', 'bites', 'fashioning', 'Alsop', 'servants',
'Onondaga', 'paragraph', 'leadings', 'clients', 'Latrobe',
'Cornwallis', 'excitingly', 'calorimetric', 'savior', 'tandem',
'antibiotics', 'excuse', 'brushy', 'selfish', 'naive', 'becomes',
'towers', 'popularizes', 'engender', 'introducing', 'possession',
'slaughtered', 'marginally', 'Packards', 'parabola', 'utopia',
'automata', 'deterrent', 'chocolates', 'objectives', 'clannish',
'aspirin', 'ferociousness', 'primarily', 'armpit', 'handfuls',
'dangle', 'Manila', 'enlivened', 'decrease', 'phylum', 'hardy',
'objectively', 'baskets', 'chaired', 'Sepoy', 'deputy', 'blizzard',
'shootings', 'breathtaking', 'sticking', 'initials', 'epitomized',
'Forrest', 'cellular', 'amatory', 'radioed', 'horrified', 'Neva',
'simultaneous', 'delimiter', 'expulsion', 'Himmler', 'contradiction',
'Remus', 'Franklinizations', 'luggage', 'moisture', 'Jews',
'comptroller', 'brevity', 'contradictions', 'Ohio', 'active',
'babysit', 'China', 'youngest', 'superstition', 'clawing', 'raccoons',
'chose', 'shoreline', 'helmets', 'Jeffersonian', 'papered',
'kindergarten', 'reply', 'succinct', 'split', 'wriggle', 'suitcases',
'nonce', 'grinders', 'anthem', 'showcase', 'maimed', 'blue', 'obeys',
'unreported', 'perusing', 'recalculate', 'rancher', 'demonic',
'Lilliputianize', 'approximation', 'repents', 'yellowness',
'irritates', 'Ferber', 'flashlights', 'booty', 'Neanderthal',
'someday', 'foregoes', 'lingering', 'cloudiness', 'guy', 'consumer',
'Berkowitz', 'relics', 'interpolating', 'reappearing', 'advisements',
'Nolan', 'turrets', 'skeletal', 'skills', 'mammas', 'Winsett',
'wheelings', 'stiffen', 'monkeys', 'plainness', 'braziers', 'Leary',
'advisee', 'jack', 'verb', 'reinterpret', 'geometrical', 'trolleys',
'arboreal', 'overpowered', 'Cuzco', 'poetical', 'admirations',
'Hobbes', 'phonemes', 'Newsweek', 'agitator', 'finally', 'prophets',
'environment', 'easterners', 'precomputed', 'faults', 'rankly',
'swallowing', 'crawl', 'trolley', 'spreading', 'resourceful', 'go',
'demandingly', 'broader', 'spiders', 'Marsha', 'debris', 'operates',
'Dundee', 'alleles', 'crunchier', 'quizzical', 'hanging', 'Fisk']
wordsd = {}
for word in words:
wordsd[word] = 1
def collect_options(args, jobs, options):
while args:
arg = args.pop(0)
if arg.startswith('-'):
name = arg[1:]
if name == 'options':
fname = args.pop(0)
d = {}
execfile(fname, d)
collect_options(list(d['options']), jobs, options)
elif options.has_key(name):
v = args.pop(0)
if options[name] != None:
raise ValueError(
"Duplicate values for %s, %s and %s"
% (name, v, options[name])
)
options[name] = v
elif name == 'setup':
options['setup'] = 1
elif globals().has_key(name.capitalize()+'Job'):
job = name
kw = {}
while args and args[0].find("=") > 0:
arg = args.pop(0).split('=')
name, v = arg[0], '='.join(arg[1:])
if kw.has_key(name):
raise ValueError(
"Duplicate parameter %s for job %s"
% (name, job)
)
kw[name]=v
if kw.has_key('frequency'):
frequency = kw['frequency']
del kw['frequency']
else:
frequency = 1
if kw.has_key('sleep'):
sleep = float(kw['sleep'])
del kw['sleep']
else:
sleep = 0.0001
if kw.has_key('repeat'):
repeatp = float(kw['repeat'])
del kw['repeat']
else:
repeatp = 0
jobs.append((job, kw, frequency, sleep, repeatp))
else:
raise ValueError("not an option or job", name)
else:
raise ValueError("Expected an option", arg)
def find_lib_python():
for b in os.getcwd(), os.path.split(sys.argv[0])[0]:
for i in range(6):
d = ['..']*i + ['lib', 'python']
p = os.path.join(b, *d)
if os.path.isdir(p):
return p
raise ValueError("Couldn't find lib/python")
def main(args=None):
lib_python = find_lib_python()
sys.path.insert(0, lib_python)
if args is None:
args = sys.argv[1:]
if not args:
print __doc__
sys.exit(0)
print args
random.seed(hash(tuple(args))) # always use the same for the given args
options = {"mbox": None, "threads": None}
jobdefs = []
collect_options(args, jobdefs, options)
mboxes = {}
if options["mbox"]:
mboxes[options["mbox"]] = MBox(options["mbox"])
# Perform a ZConfig-based Zope initialization:
zetup(os.path.join(lib_python, '..', '..', 'etc', 'zope.conf'))
if options.has_key('setup'):
setup(lib_python)
else:
import Zope2
Zope2.startup()
#from ThreadedAsync.LoopCallback import loop
#threading.Thread(target=loop, args=(), name='asyncore').start()
jobs = JobProducer()
for job, kw, frequency, sleep, repeatp in jobdefs:
Job = globals()[job.capitalize()+'Job']
if getattr(Job, 'needs_mbox', 0):
if not kw.has_key("mbox"):
if not options["mbox"]:
raise ValueError(
"no mailbox (mbox option) file specified")
kw['mbox'] = mboxes[options["mbox"]]
else:
if not mboxes.has_key[kw["mbox"]]:
mboxes[kw['mbox']] = MBox[kw['mbox']]
kw["mbox"] = mboxes[kw['mbox']]
jobs.add(Job(**kw), frequency, sleep, repeatp)
if not jobs:
print "No jobs to execute"
return
threads = int(options['threads'] or '0')
if threads > 1:
threads = [threading.Thread(target=run, args=(jobs, i), name=str(i))
for i in range(threads)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
else:
run(jobs)
def zetup(configfile_name):
from Zope.Startup.options import ZopeOptions
from Zope.Startup import handlers as h
from App import config
opts = ZopeOptions()
opts.configfile = configfile_name
opts.realize(args=[])
h.handleConfig(opts.configroot, opts.confighandlers)
config.setConfiguration(opts.configroot)
from Zope.Startup import dropPrivileges
dropPrivileges(opts.configroot)
if __name__ == '__main__':
main()
ZODB/src/scripts/migrate.py 0000750 0001750 0000144 00000025761 10057366465 015473 0 ustar ctheune users #!/usr/bin/env python2.3
##############################################################################
#
# Copyright (c) 2001, 2002, 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""A script to gather statistics while doing a storage migration.
This is very similar to a standard storage's copyTransactionsFrom() method,
except that it's geared to run as a script, and it collects useful pieces of
information as it's working. This script can be used to stress test a storage
since it blasts transactions at it as fast as possible. You can get a good
sense of the performance of a storage by running this script.
Actually it just counts the size of pickles in the transaction via the
iterator protocol, so storage overheads aren't counted.
Usage: %(PROGRAM)s [options] [source-storage-args] [destination-storage-args]
Options:
-S sourcetype
--stype=sourcetype
This is the name of a recognized type for the source database. Use -T
to print out the known types. Defaults to "file".
-D desttype
--dtype=desttype
This is the name of the recognized type for the destination database.
Use -T to print out the known types. Defaults to "file".
-o filename
--output=filename
Print results in filename, otherwise stdout.
-m txncount
--max=txncount
Stop after committing txncount transactions.
-k txncount
--skip=txncount
Skip the first txncount transactions.
-p/--profile
Turn on specialized profiling.
-t/--timestamps
Print tids as timestamps.
-T/--storage_types
Print all the recognized storage types and exit.
-v/--verbose
Turns on verbose output. Multiple -v options increase the verbosity.
-h/--help
Print this message and exit.
Positional arguments:
source-storage-args:
Semicolon separated list of arguments for the source storage, as
key=val pairs. E.g. "file_name=Data.fs;read_only=1"
destination-storage-args:
Comma separated list of arguments for the source storage, as key=val
pairs. E.g. "name=full;frequency=3600"
"""
import re
import sys
import time
import getopt
import marshal
import profile
from ZODB import utils
from ZODB import StorageTypes
from ZODB.TimeStamp import TimeStamp
PROGRAM = sys.argv[0]
ZERO = '\0'*8
try:
True, False
except NameError:
True = 1
False = 0
def usage(code, msg=''):
print >> sys.stderr, __doc__ % globals()
if msg:
print >> sys.stderr, msg
sys.exit(code)
def error(code, msg):
print >> sys.stderr, msg
print "use --help for usage message"
sys.exit(code)
def main():
try:
opts, args = getopt.getopt(
sys.argv[1:],
'hvo:pm:k:D:S:Tt',
['help', 'verbose',
'output=', 'profile', 'storage_types',
'max=', 'skip=', 'dtype=', 'stype=', 'timestamps'])
except getopt.error, msg:
error(2, msg)
class Options:
stype = 'FileStorage'
dtype = 'FileStorage'
verbose = 0
outfile = None
profilep = False
maxtxn = -1
skiptxn = -1
timestamps = False
options = Options()
for opt, arg in opts:
if opt in ('-h', '--help'):
usage(0)
elif opt in ('-v', '--verbose'):
options.verbose += 1
elif opt in ('-T', '--storage_types'):
print_types()
sys.exit(0)
elif opt in ('-S', '--stype'):
options.stype = arg
elif opt in ('-D', '--dtype'):
options.dtype = arg
elif opt in ('-o', '--output'):
options.outfile = arg
elif opt in ('-p', '--profile'):
options.profilep = True
elif opt in ('-m', '--max'):
options.maxtxn = int(arg)
elif opt in ('-k', '--skip'):
options.skiptxn = int(arg)
elif opt in ('-t', '--timestamps'):
options.timestamps = True
if len(args) > 2:
error(2, "too many arguments")
srckws = {}
if len(args) > 0:
srcargs = args[0]
for kv in re.split(r';\s*', srcargs):
key, val = kv.split('=')
srckws[key] = val
destkws = {}
if len(args) > 1:
destargs = args[1]
for kv in re.split(r';\s*', destargs):
key, val = kv.split('=')
destkws[key] = val
if options.stype not in StorageTypes.storage_types.keys():
usage(2, 'Source database type must be provided')
if options.dtype not in StorageTypes.storage_types.keys():
usage(2, 'Destination database type must be provided')
# Open the output file
if options.outfile is None:
options.outfp = sys.stdout
options.outclosep = False
else:
options.outfp = open(options.outfile, 'w')
options.outclosep = True
if options.verbose > 0:
print 'Opening source database...'
modname, sconv = StorageTypes.storage_types[options.stype]
kw = sconv(**srckws)
__import__(modname)
sclass = getattr(sys.modules[modname], options.stype)
srcdb = sclass(**kw)
if options.verbose > 0:
print 'Opening destination database...'
modname, dconv = StorageTypes.storage_types[options.dtype]
kw = dconv(**destkws)
__import__(modname)
dclass = getattr(sys.modules[modname], options.dtype)
dstdb = dclass(**kw)
try:
t0 = time.time()
doit(srcdb, dstdb, options)
t1 = time.time()
if options.verbose > 0:
print 'Migration time: %8.3f' % (t1-t0)
finally:
# Done
srcdb.close()
dstdb.close()
if options.outclosep:
options.outfp.close()
def doit(srcdb, dstdb, options):
outfp = options.outfp
profilep = options.profilep
verbose = options.verbose
# some global information
largest_pickle = 0
largest_txn_in_size = 0
largest_txn_in_objects = 0
total_pickle_size = 0L
total_object_count = 0
# Ripped from BaseStorage.copyTransactionsFrom()
ts = None
ok = True
prevrevids = {}
counter = 0
skipper = 0
if options.timestamps:
print "%4s. %26s %6s %8s %5s %5s %5s %5s %5s" % (
"NUM", "TID AS TIMESTAMP", "OBJS", "BYTES",
# Does anybody know what these times mean?
"t4-t0", "t1-t0", "t2-t1", "t3-t2", "t4-t3")
else:
print "%4s. %20s %6s %8s %6s %6s %6s %6s %6s" % (
"NUM", "TRANSACTION ID", "OBJS", "BYTES",
# Does anybody know what these times mean?
"t4-t0", "t1-t0", "t2-t1", "t3-t2", "t4-t3")
for txn in srcdb.iterator():
skipper += 1
if skipper <= options.skiptxn:
continue
counter += 1
if counter > options.maxtxn >= 0:
break
tid = txn.tid
if ts is None:
ts = TimeStamp(tid)
else:
t = TimeStamp(tid)
if t <= ts:
if ok:
print >> sys.stderr, \
'Time stamps are out of order %s, %s' % (ts, t)
ok = False
ts = t.laterThan(ts)
tid = `ts`
else:
ts = t
if not ok:
print >> sys.stderr, \
'Time stamps are back in order %s' % t
ok = True
if verbose > 1:
print ts
prof = None
if profilep and (counter % 100) == 0:
prof = profile.Profile()
objects = 0
size = 0
newrevids = RevidAccumulator()
t0 = time.time()
dstdb.tpc_begin(txn, tid, txn.status)
t1 = time.time()
for r in txn:
oid = r.oid
objects += 1
thissize = len(r.data)
size += thissize
if thissize > largest_pickle:
largest_pickle = thissize
if verbose > 1:
if not r.version:
vstr = 'norev'
else:
vstr = r.version
print utils.U64(oid), vstr, len(r.data)
oldrevid = prevrevids.get(oid, ZERO)
result = dstdb.store(oid, oldrevid, r.data, r.version, txn)
newrevids.store(oid, result)
t2 = time.time()
result = dstdb.tpc_vote(txn)
t3 = time.time()
newrevids.tpc_vote(result)
prevrevids.update(newrevids.get_dict())
# Profile every 100 transactions
if prof:
prof.runcall(dstdb.tpc_finish, txn)
else:
dstdb.tpc_finish(txn)
t4 = time.time()
# record the results
if objects > largest_txn_in_objects:
largest_txn_in_objects = objects
if size > largest_txn_in_size:
largest_txn_in_size = size
if options.timestamps:
tidstr = str(TimeStamp(tid))
format = "%4d. %26s %6d %8d %5.3f %5.3f %5.3f %5.3f %5.3f"
else:
tidstr = utils.U64(tid)
format = "%4d. %20s %6d %8d %6.4f %6.4f %6.4f %6.4f %6.4f"
print >> outfp, format % (skipper, tidstr, objects, size,
t4-t0, t1-t0, t2-t1, t3-t2, t4-t3)
total_pickle_size += size
total_object_count += objects
if prof:
prof.create_stats()
fp = open('profile-%02d.txt' % (counter / 100), 'wb')
marshal.dump(prof.stats, fp)
fp.close()
print >> outfp, "Largest pickle: %8d" % largest_pickle
print >> outfp, "Largest transaction: %8d" % largest_txn_in_size
print >> outfp, "Largest object count: %8d" % largest_txn_in_objects
print >> outfp, "Total pickle size: %14d" % total_pickle_size
print >> outfp, "Total object count: %8d" % total_object_count
# helper to deal with differences between old-style store() return and
# new-style store() return that supports ZEO
import types
class RevidAccumulator:
def __init__(self):
self.data = {}
def _update_from_list(self, list):
for oid, serial in list:
if not isinstance(serial, types.StringType):
raise serial
self.data[oid] = serial
def store(self, oid, result):
if isinstance(result, types.StringType):
self.data[oid] = result
elif result is not None:
self._update_from_list(result)
def tpc_vote(self, result):
if result is not None:
self._update_from_list(result)
def get_dict(self):
return self.data
if __name__ == '__main__':
main()
ZODB/src/scripts/fsrefs.py 0000640 0001750 0000144 00000013523 10252101267 015302 0 ustar ctheune users #!/usr/bin/env python2.3
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Check FileStorage for dangling references.
usage: fsrefs.py [-v] data.fs
fsrefs.py checks object sanity by trying to load the current revision of
every object O in the database, and also verifies that every object
directly reachable from each such O exists in the database.
It's hard to explain exactly what it does because it relies on undocumented
features in Python's cPickle module: many of the crucial steps of loading
an object are taken, but application objects aren't actually created. This
saves a lot of time, and allows fsrefs to be run even if the code
implementing the object classes isn't available.
A read-only connection to the specified FileStorage is made, but it is not
recommended to run fsrefs against a live FileStorage. Because a live
FileStorage is mutating while fsrefs runs, it's not possible for fsrefs to
get a wholly consistent view of the database across the entire time fsrefs
is running; spurious error messages may result.
fsrefs doesn't normally produce any output. If an object fails to load, the
oid of the object is given in a message saying so, and if -v was specified
then the traceback corresponding to the load failure is also displayed
(this is the only effect of the -v flag).
Three other kinds of errors are also detected, when an object O loads OK,
and directly refers to a persistent object P but there's a problem with P:
- If P doesn't exist in the database, a message saying so is displayed.
The unsatisifiable reference to P is often called a "dangling
reference"; P is called "missing" in the error output.
- If the current state of the database is such that P's creation has
been undone, then P can't be loaded either. This is also a kind of
dangling reference, but is identified as "object creation was undone".
- If P can't be loaded (but does exist in the database), a message saying
that O refers to an object that can't be loaded is displayed.
fsrefs also (indirectly) checks that the .index file is sane, because
fsrefs uses the index to get its idea of what constitutes "all the objects
in the database".
Note these limitations: because fsrefs only looks at the current revision
of objects, it does not attempt to load objects in versions, or non-current
revisions of objects; therefore fsrefs cannot find problems in versions or
in non-current revisions.
"""
import traceback
import types
from ZODB.FileStorage import FileStorage
from ZODB.TimeStamp import TimeStamp
from ZODB.utils import u64, oid_repr, get_pickle_metadata
from ZODB.serialize import get_refs
from ZODB.POSException import POSKeyError
VERBOSE = 0
# There's a problem with oid. 'data' is its pickle, and 'serial' its
# serial number. 'missing' is a list of (oid, class, reason) triples,
# explaining what the problem(s) is(are).
def report(oid, data, serial, missing):
from_mod, from_class = get_pickle_metadata(data)
if len(missing) > 1:
plural = "s"
else:
plural = ""
ts = TimeStamp(serial)
print "oid %s %s.%s" % (hex(u64(oid)), from_mod, from_class)
print "last updated: %s, tid=%s" % (ts, hex(u64(serial)))
print "refers to invalid object%s:" % plural
for oid, info, reason in missing:
if isinstance(info, types.TupleType):
description = "%s.%s" % info
else:
description = str(info)
print "\toid %s %s: %r" % (oid_repr(oid), reason, description)
print
def main(path):
fs = FileStorage(path, read_only=1)
# Set of oids in the index that failed to load due to POSKeyError.
# This is what happens if undo is applied to the transaction creating
# the object (the oid is still in the index, but its current data
# record has a backpointer of 0, and POSKeyError is raised then
# because of that backpointer).
undone = {}
# Set of oids that were present in the index but failed to load.
# This does not include oids in undone.
noload = {}
for oid in fs._index.keys():
try:
data, serial = fs.load(oid, "")
except (KeyboardInterrupt, SystemExit):
raise
except POSKeyError:
undone[oid] = 1
except:
if VERBOSE:
traceback.print_exc()
noload[oid] = 1
inactive = noload.copy()
inactive.update(undone)
for oid in fs._index.keys():
if oid in inactive:
continue
data, serial = fs.load(oid, "")
refs = get_refs(data)
missing = [] # contains 3-tuples of oid, klass-metadata, reason
for info in refs:
ref, klass = info
if klass is None:
# failed to unpack
ref = info
klass = ''
if ref not in fs._index:
missing.append((ref, klass, "missing"))
if ref in noload:
missing.append((ref, klass, "failed to load"))
if ref in undone:
missing.append((ref, klass, "object creation was undone"))
if missing:
report(oid, data, serial, missing)
if __name__ == "__main__":
import sys
import getopt
opts, args = getopt.getopt(sys.argv[1:], "v")
for k, v in opts:
if k == "-v":
VERBOSE += 1
path, = args
main(path)
ZODB/src/scripts/simul.py 0000750 0001750 0000144 00000163215 10276710420 015154 0 ustar ctheune users #! /usr/bin/env python
##############################################################################
#
# Copyright (c) 2001-2005 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Cache simulation.
Usage: simul.py [-s size] tracefile
Options:
-s size: cache size in MB (default 20 MB)
"""
import sys
import time
import getopt
import struct
import math
import bisect
from sets import Set
from ZODB.utils import z64
def usage(msg):
print >> sys.stderr, msg
print >> sys.stderr, __doc__
def main():
# Parse options.
MB = 1024**2
cachelimit = 20*MB
simclass = CircularCacheSimulation
try:
opts, args = getopt.getopt(sys.argv[1:], "bflyz2cOaTUs:")
except getopt.error, msg:
usage(msg)
return 2
for o, a in opts:
if o == '-b':
simclass = BuddyCacheSimulation
elif o == '-f':
simclass = SimpleCacheSimulation
elif o == '-l':
simclass = LRUCacheSimulation
elif o == '-y':
simclass = AltZEOCacheSimulation
elif o == '-z':
simclass = ZEOCacheSimulation
elif o == '-s':
cachelimit = int(float(a)*MB)
elif o == '-2':
simclass = TwoQSimluation
elif o == '-c':
simclass = CircularCacheSimulation
elif o == '-O':
simclass = OracleSimulation
elif o == '-a':
simclass = ARCCacheSimulation
elif o == '-T':
simclass = ThorSimulation
elif o == '-U':
simclass = UnboundedSimulation
else:
assert False, (o, a)
if len(args) != 1:
usage("exactly one file argument required")
return 2
filename = args[0]
# Open file.
if filename.endswith(".gz"):
# Open gzipped file.
try:
import gzip
except ImportError:
print >> sys.stderr, "can't read gzipped files (no module gzip)"
return 1
try:
f = gzip.open(filename, "rb")
except IOError, msg:
print >> sys.stderr, "can't open %s: %s" % (filename, msg)
return 1
elif filename == "-":
# Read from stdin.
f = sys.stdin
else:
# Open regular file.
try:
f = open(filename, "rb")
except IOError, msg:
print >> sys.stderr, "can't open %s: %s" % (filename, msg)
return 1
# Create simulation object.
if simclass is OracleSimulation:
sim = simclass(cachelimit, filename)
else:
sim = simclass(cachelimit)
# Print output header.
sim.printheader()
# Read trace file, simulating cache behavior.
f_read = f.read
unpack = struct.unpack
FMT = ">iiH8s8s"
FMT_SIZE = struct.calcsize(FMT)
assert FMT_SIZE == 26
while 1:
# Read a record and decode it.
r = f_read(FMT_SIZE)
if len(r) < FMT_SIZE:
break
ts, code, oidlen, start_tid, end_tid = unpack(FMT, r)
if ts == 0:
# Must be a misaligned record caused by a crash; skip 8 bytes
# and try again. Why 8? Lost in the mist of history.
f.seek(f.tell() - FMT_SIZE + 8)
continue
oid = f_read(oidlen)
if len(oid) < oidlen:
break
# Decode the code.
dlen, version, code = (code & 0x7fffff00,
code & 0x80,
code & 0x7e)
# And pass it to the simulation.
sim.event(ts, dlen, version, code, oid, start_tid, end_tid)
f.close()
# Finish simulation.
sim.finish()
# Exit code from main().
return 0
class Simulation(object):
"""Base class for simulations.
The driver program calls: event(), printheader(), finish().
The standard event() method calls these additional methods:
write(), load(), inval(), report(), restart(); the standard
finish() method also calls report().
"""
def __init__(self, cachelimit):
self.cachelimit = cachelimit
# Initialize global statistics.
self.epoch = None
self.total_loads = 0
self.total_hits = 0 # subclass must increment
self.total_invals = 0 # subclass must increment
self.total_writes = 0
if not hasattr(self, "extras"):
self.extras = (self.extraname,)
self.format = self.format + " %7s" * len(self.extras)
# Reset per-run statistics and set up simulation data.
self.restart()
def restart(self):
# Reset per-run statistics.
self.loads = 0
self.hits = 0 # subclass must increment
self.invals = 0 # subclass must increment
self.writes = 0
self.ts0 = None
def event(self, ts, dlen, _version, code, oid,
start_tid, end_tid):
# Record first and last timestamp seen.
if self.ts0 is None:
self.ts0 = ts
if self.epoch is None:
self.epoch = ts
self.ts1 = ts
# Simulate cache behavior. Caution: the codes in the trace file
# record whether the actual cache missed or hit on each load, but
# that bears no necessary relationship to whether the simulated cache
# will hit or miss. Relatedly, if the actual cache needed to store
# an object, the simulated cache may not need to (it may already
# have the data).
action = code & 0x70
if action == 0x20:
# Load.
self.loads += 1
self.total_loads += 1
# Asserting that dlen is 0 iff it's a load miss.
# assert (dlen == 0) == (code in (0x20, 0x24))
self.load(oid, dlen, start_tid)
elif action == 0x50:
# Store.
assert dlen
self.write(oid, dlen, start_tid, end_tid)
elif action == 0x10:
# Invalidate.
self.inval(oid, start_tid)
elif action == 0x00:
# Restart.
self.report()
self.restart()
else:
raise ValueError("unknown trace code 0x%x" % code)
def write(self, oid, size, start_tid, end_tid):
pass
def load(self, oid, size, start_tid):
# Must increment .hits and .total_hits as appropriate.
pass
def inval(self, oid, start_tid):
# Must increment .invals and .total_invals as appropriate.
pass
format = "%12s %9s %8s %8s %6s %6s %7s"
# Subclass should override extraname to name known instance variables;
# if extraname is 'foo', both self.foo and self.total_foo must exist:
extraname = "*** please override ***"
def printheader(self):
print "%s, cache size %s bytes" % (self.__class__.__name__,
addcommas(self.cachelimit))
self.extraheader()
extranames = tuple([s.upper() for s in self.extras])
args = ("START TIME", "DURATION", "LOADS", "HITS",
"INVALS", "WRITES", "HITRATE") + extranames
print self.format % args
def extraheader(self):
pass
nreports = 0
def report(self, extratext=''):
if self.loads:
self.nreports += 1
args = (time.ctime(self.ts0)[4:-8],
duration(self.ts1 - self.ts0),
self.loads, self.hits, self.invals, self.writes,
hitrate(self.loads, self.hits))
args += tuple([getattr(self, name) for name in self.extras])
print self.format % args, extratext
def finish(self):
# Make sure that the last line of output ends with "OVERALL". This
# makes it much easier for another program parsing the output to
# find summary statistics.
if self.nreports < 2:
self.report('OVERALL')
else:
self.report()
args = (
time.ctime(self.epoch)[4:-8],
duration(self.ts1 - self.epoch),
self.total_loads,
self.total_hits,
self.total_invals,
self.total_writes,
hitrate(self.total_loads, self.total_hits))
args += tuple([getattr(self, "total_" + name)
for name in self.extras])
print (self.format + " OVERALL") % args
# For use in CircularCacheSimulation.
class CircularCacheEntry(object):
__slots__ = (# object key: an (oid, start_tid) pair, where
# start_tid is the tid of the transaction that created
# this revision of oid
'key',
# tid of transaction that created the next revision;
# z64 iff this is the current revision
'end_tid',
# Offset from start of file to the object's data
# record; this includes all overhead bytes (status
# byte, size bytes, etc).
'offset',
)
def __init__(self, key, end_tid, offset):
self.key = key
self.end_tid = end_tid
self.offset = offset
from ZEO.cache import ZEC3_HEADER_SIZE
class CircularCacheSimulation(Simulation):
"""Simulate the ZEO 3.0 cache."""
# The cache is managed as a single file with a pointer that
# goes around the file, circularly, forever. New objects
# are written at the current pointer, evicting whatever was
# there previously.
extras = "evicts", "inuse"
def __init__(self, cachelimit):
from ZEO import cache
Simulation.__init__(self, cachelimit)
self.total_evicts = 0 # number of cache evictions
# Current offset in file.
self.offset = ZEC3_HEADER_SIZE
# Map offset in file to (size, CircularCacheEntry) pair, or to
# (size, None) if the offset starts a free block.
self.filemap = {ZEC3_HEADER_SIZE: (self.cachelimit - ZEC3_HEADER_SIZE,
None)}
# Map key to CircularCacheEntry. A key is an (oid, tid) pair.
self.key2entry = {}
# Map oid to tid of current revision.
self.current = {}
# Map oid to list of (start_tid, end_tid) pairs in sorted order.
# Used to find matching key for load of non-current data.
self.noncurrent = {}
# The number of overhead bytes needed to store an object pickle
# on disk (all bytes beyond those needed for the object pickle).
self.overhead = (cache.Object.TOTAL_FIXED_SIZE +
cache.OBJECT_HEADER_SIZE)
def restart(self):
Simulation.restart(self)
self.evicts = 0
def load(self, oid, size, tid):
if tid == z64:
# Trying to load current revision.
if oid in self.current: # else it's a cache miss
self.hits += 1
self.total_hits += 1
return
# May or may not be trying to load current revision.
cur_tid = self.current.get(oid)
if cur_tid == tid:
self.hits += 1
self.total_hits += 1
return
# It's a load for non-current data. Do we know about this oid?
L = self.noncurrent.get(oid)
if L is None:
return # cache miss
i = bisect.bisect_left(L, (tid, None))
if i == 0:
# This tid is smaller than any we know about -- miss.
return
lo, hi = L[i-1]
assert lo < tid
if tid > hi:
# No data in the right tid range -- miss.
return
# Cache hit.
self.hits += 1
self.total_hits += 1
# (oid, tid) is in the cache. Remove it: take it out of key2entry,
# and in `filemap` mark the space it occupied as being free. The
# caller is responsible for removing it from `current` or `noncurrent`.
def _remove(self, oid, tid):
key = oid, tid
e = self.key2entry.pop(key)
pos = e.offset
size, _e = self.filemap[pos]
assert e is _e
self.filemap[pos] = size, None
def _remove_noncurrent_revisions(self, oid):
noncurrent_list = self.noncurrent.get(oid)
if noncurrent_list:
self.invals += len(noncurrent_list)
self.total_invals += len(noncurrent_list)
for start_tid, end_tid in noncurrent_list:
self._remove(oid, start_tid)
del self.noncurrent[oid]
def inval(self, oid, tid):
if tid == z64:
# This is part of startup cache verification: forget everything
# about this oid.
self._remove_noncurrent_revisions(oid)
cur_tid = self.current.get(oid)
if cur_tid is None:
# We don't have current data, so nothing more to do.
return
# We had current data for oid, but no longer.
self.invals += 1
self.total_invals += 1
del self.current[oid]
if tid == z64:
# Startup cache verification: forget this oid entirely.
self._remove(oid, cur_tid)
return
# Our current data becomes non-current data.
# Add the validity range to the list of non-current data for oid.
assert cur_tid < tid
L = self.noncurrent.setdefault(oid, [])
bisect.insort_left(L, (cur_tid, tid))
# Update the end of oid's validity range in its CircularCacheEntry.
e = self.key2entry[oid, cur_tid]
assert e.end_tid == z64
e.end_tid = tid
def write(self, oid, size, start_tid, end_tid):
if end_tid == z64:
# Storing current revision.
if oid in self.current: # we already have it in cache
return
self.current[oid] = start_tid
self.writes += 1
self.total_writes += 1
self.add(oid, size, start_tid)
return
# Storing non-current revision.
L = self.noncurrent.setdefault(oid, [])
p = start_tid, end_tid
if p in L:
return # we already have it in cache
bisect.insort_left(L, p)
self.writes += 1
self.total_writes += 1
self.add(oid, size, start_tid, end_tid)
# Add `oid` to the cache, evicting objects as needed to make room.
# This updates `filemap` and `key2entry`; it's the caller's
# responsibilty to update `current` or `noncurrent` appropriately.
def add(self, oid, size, start_tid, end_tid=z64):
key = oid, start_tid
assert key not in self.key2entry
size += self.overhead
avail = self.makeroom(size)
e = CircularCacheEntry(key, end_tid, self.offset)
self.filemap[self.offset] = size, e
self.key2entry[key] = e
self.offset += size
# All the space made available must be accounted for in filemap.
excess = avail - size
if excess:
self.filemap[self.offset] = excess, None
# Evict enough objects to make at least `need` contiguous bytes, starting
# at `self.offset`, available. Evicted objects are removed from
# `filemap`, `key2entry`, `current` and `noncurrent`. The caller is
# responsible for adding new entries to `filemap` to account for all
# the freed bytes, and for advancing `self.offset`. The number of bytes
# freed is the return value, and will be >= need.
def makeroom(self, need):
if self.offset + need > self.cachelimit:
self.offset = ZEC3_HEADER_SIZE
pos = self.offset
while need > 0:
assert pos < self.cachelimit
size, e = self.filemap.pop(pos)
if e: # there is an object here (else it's already free space)
self.evicts += 1
self.total_evicts += 1
assert pos == e.offset
_e = self.key2entry.pop(e.key)
assert e is _e
oid, start_tid = e.key
if e.end_tid == z64:
del self.current[oid]
else:
L = self.noncurrent[oid]
L.remove((start_tid, e.end_tid))
need -= size
pos += size
return pos - self.offset # total number of bytes freed
def report(self):
self.check()
free = used = total = 0
for size, e in self.filemap.itervalues():
total += size
if e:
used += size
else:
free += size
self.inuse = round(100.0 * used / total, 1)
self.total_inuse = self.inuse
Simulation.report(self)
def check(self):
oidcount = 0
pos = ZEC3_HEADER_SIZE
while pos < self.cachelimit:
size, e = self.filemap[pos]
if e:
oidcount += 1
assert self.key2entry[e.key].offset == pos
pos += size
assert oidcount == len(self.key2entry)
assert pos == self.cachelimit
def dump(self):
print len(self.filemap)
L = list(self.filemap)
L.sort()
for k in L:
v = self.filemap[k]
print k, v[0], repr(v[1])
#############################################################################
# CAUTION: It's most likely that none of the simulators below this
# point work anymore. A great many changes were needed to teach
# CircularCacheSimulation (above) about MVCC, including method signature
# changes and changes in cache file format, and none of the other simulator
# classes were changed.
#############################################################################
class ZEOCacheSimulation(Simulation):
"""Simulate the ZEO 1.0 and 2.0 cache behavior.
This assumes the cache is not persistent (we don't know how to
simulate cache validation.)
"""
extraname = "flips"
def __init__(self, cachelimit):
# Initialize base class
Simulation.__init__(self, cachelimit)
# Initialize additional global statistics
self.total_flips = 0
def restart(self):
# Reset base class
Simulation.restart(self)
# Reset additional per-run statistics
self.flips = 0
# Set up simulation
self.filesize = [4, 4] # account for magic number
self.fileoids = [{}, {}]
self.current = 0 # index into filesize, fileoids
def load(self, oid, size):
if (self.fileoids[self.current].get(oid) or
self.fileoids[1 - self.current].get(oid)):
self.hits += 1
self.total_hits += 1
else:
self.write(oid, size)
def write(self, oid, size):
# Fudge because size is rounded up to multiples of 256. (31
# is header overhead per cache record; 127 is to compensate
# for rounding up to multiples of 256.)
size = size + 31 - 127
if self.filesize[self.current] + size > self.cachelimit / 2:
# Cache flip
self.flips += 1
self.total_flips += 1
self.current = 1 - self.current
self.filesize[self.current] = 4
self.fileoids[self.current] = {}
self.filesize[self.current] += size
self.fileoids[self.current][oid] = 1
def inval(self, oid):
if self.fileoids[self.current].get(oid):
self.invals += 1
self.total_invals += 1
del self.fileoids[self.current][oid]
elif self.fileoids[1 - self.current].get(oid):
self.invals += 1
self.total_invals += 1
del self.fileoids[1 - self.current][oid]
class AltZEOCacheSimulation(ZEOCacheSimulation):
"""A variation of the ZEO cache that copies to the current file.
When a hit is found in the non-current cache file, it is copied to
the current cache file. Exception: when the copy would cause a
cache flip, we don't copy (this is part laziness, part concern
over causing extraneous flips).
"""
def load(self, oid, size):
if self.fileoids[self.current].get(oid):
self.hits += 1
self.total_hits += 1
elif self.fileoids[1 - self.current].get(oid):
self.hits += 1
self.total_hits += 1
# Simulate a write, unless it would cause a flip
size = size + 31 - 127
if self.filesize[self.current] + size <= self.cachelimit / 2:
self.filesize[self.current] += size
self.fileoids[self.current][oid] = 1
del self.fileoids[1 - self.current][oid]
else:
self.write(oid, size)
class LRUCacheSimulation(Simulation):
extraname = "evicts"
def __init__(self, cachelimit):
# Initialize base class
Simulation.__init__(self, cachelimit)
# Initialize additional global statistics
self.total_evicts = 0
def restart(self):
# Reset base class
Simulation.restart(self)
# Reset additional per-run statistics
self.evicts = 0
# Set up simulation
self.cache = {}
self.size = 0
self.head = Node(None, None)
self.head.linkbefore(self.head)
def load(self, oid, size):
node = self.cache.get(oid)
if node is not None:
self.hits += 1
self.total_hits += 1
node.linkbefore(self.head)
else:
self.write(oid, size)
def write(self, oid, size):
node = self.cache.get(oid)
if node is not None:
node.unlink()
assert self.head.next is not None
self.size -= node.size
node = Node(oid, size)
self.cache[oid] = node
node.linkbefore(self.head)
self.size += size
# Evict LRU nodes
while self.size > self.cachelimit:
self.evicts += 1
self.total_evicts += 1
node = self.head.next
assert node is not self.head
node.unlink()
assert self.head.next is not None
del self.cache[node.oid]
self.size -= node.size
def inval(self, oid):
node = self.cache.get(oid)
if node is not None:
assert node.oid == oid
self.invals += 1
self.total_invals += 1
node.unlink()
assert self.head.next is not None
del self.cache[oid]
self.size -= node.size
assert self.size >= 0
class Node(object):
"""Node in a doubly-linked list, storing oid and size as payload.
A node can be linked or unlinked; in the latter case, next and
prev are None. Initially a node is unlinked.
"""
__slots__ = ['prev', 'next', 'oid', 'size']
def __init__(self, oid, size):
self.oid = oid
self.size = size
self.prev = self.next = None
def unlink(self):
prev = self.prev
next = self.next
if prev is not None:
assert next is not None
assert prev.next is self
assert next.prev is self
prev.next = next
next.prev = prev
self.prev = self.next = None
else:
assert next is None
def linkbefore(self, next):
self.unlink()
prev = next.prev
if prev is None:
assert next.next is None
prev = next
self.prev = prev
self.next = next
prev.next = next.prev = self
am = object()
a1in = object()
a1out = object()
class Node2Q(Node):
__slots__ = ["kind", "hits"]
def __init__(self, oid, size, kind=None):
Node.__init__(self, oid, size)
self.kind = kind
self.hits = 0
def linkbefore(self, next):
if next.kind != self.kind:
self.kind = next.kind
Node.linkbefore(self, next)
class TwoQSimluation(Simulation):
# The original 2Q algorithm is page based and the authors offer
# tuning guidlines based on a page-based cache. Our cache is
# object based, so, for example, it's hard to compute the number
# of oids to store in a1out based on the size of a1in.
extras = "evicts", "hothit", "am_add"
NodeClass = Node2Q
def __init__(self, cachelimit, outlen=10000, threshold=0):
Simulation.__init__(self, cachelimit)
# The promotion threshold: If a hit occurs in a1out, it is
# promoted to am if the number of hits on the object while it
# was in a1in is at least threshold. The standard 2Q scheme
# uses a threshold of 0.
self.threshold = threshold
self.am_limit = 3 * self.cachelimit / 4
self.a1in_limit = self.cachelimit / 4
self.cache = {}
self.am_size = 0
self.a1in_size = 0
self.a1out_size = 0
self.total_evicts = 0
self.total_hothit = 0
self.total_am_add = 0
self.a1out_limit = outlen
# An LRU queue of hot objects
self.am = self.NodeClass(None, None, am)
self.am.linkbefore(self.am)
# A FIFO queue of recently referenced objects. It's purpose
# is to absorb references to objects that are accessed a few
# times in short order, then forgotten about.
self.a1in = self.NodeClass(None, None, a1in)
self.a1in.linkbefore(self.a1in)
# A FIFO queue of recently reference oids.
# This queue only stores the oids, not any data. If we get a
# hit in this queue, promote the object to am.
self.a1out = self.NodeClass(None, None, a1out)
self.a1out.linkbefore(self.a1out)
def makespace(self, size):
for space in 0, size:
if self.enoughspace(size):
return
self.evict_a1in(space)
if self.enoughspace(size):
return
self.evict_am(space)
def enoughspace(self, size):
totalsize = self.a1in_size + self.am_size
return totalsize + size < self.cachelimit
def evict_a1in(self, extra):
while self.a1in_size + extra > self.a1in_limit:
if self.a1in.next is self.a1in:
return
assert self.a1in.next is not None
node = self.a1in.next
self.evicts += 1
self.total_evicts += 1
node.linkbefore(self.a1out)
self.a1out_size += 1
self.a1in_size -= node.size
if self.a1out_size > self.a1out_limit:
assert self.a1out.next is not None
node = self.a1out.next
node.unlink()
del self.cache[node.oid]
self.a1out_size -= 1
def evict_am(self, extra):
while self.am_size + extra > self.am_limit:
if self.am.next is self.am:
return
assert self.am.next is not None
node = self.am.next
self.evicts += 1
self.total_evicts += 1
# This node hasn't been accessed in a while, so just
# forget about it.
node.unlink()
del self.cache[node.oid]
self.am_size -= node.size
def write(self, oid, size):
# A write always follows a read (ZODB doesn't allow blind writes).
# So this write must have followed a recent read of the object.
# Don't change it's position, but do update the size.
# XXX For now, don't evict pages if the new version of the object
# is big enough to require eviction.
node = self.cache.get(oid)
if node is None or node.kind is a1out:
return
if node.kind is am:
self.am_size = self.am_size - node.size + size
node.size = size
else:
self.a1in_size = self.a1in_size - node.size + size
node.size = size
def load(self, oid, size):
node = self.cache.get(oid)
if node is not None:
if node.kind is am:
self.hits += 1
self.total_hits += 1
self.hothit += 1
self.total_hothit += 1
node.hits += 1
node.linkbefore(self.am)
elif node.kind is a1in:
self.hits += 1
self.total_hits += 1
node.hits += 1
elif node.kind is a1out:
self.a1out_size -= 1
if node.hits >= self.threshold:
self.makespace(node.size)
self.am_size += node.size
node.linkbefore(self.am)
self.cache[oid] = node
self.am_add += 1
self.total_am_add += 1
else:
node.unlink()
self.insert(oid, size)
else:
self.insert(oid, size)
def insert(self, oid, size):
# New objects enter the cache via a1in. If they
# are frequently used over a long enough time, they
# will be promoted to am -- but only via a1out.
self.makespace(size)
node = self.NodeClass(oid, size, a1in)
node.linkbefore(self.a1in)
self.cache[oid] = node
self.a1in_size += node.size
def inval(self, oid):
# The original 2Q algorithm didn't have to deal with
# invalidations. My own solution: Move it to the head of
# a1out.
node = self.cache.get(oid)
if node is None:
return
self.invals += 1
self.total_invals += 1
# XXX Should an invalidation to a1out count?
if node.kind is a1out:
return
node.linkbefore(self.a1out)
if node.kind is am:
self.am_size -= node.size
else:
self.a1in_size -= node.size
def restart(self):
Simulation.restart(self)
self.evicts = 0
self.hothit = 0
self.am_add = 0
lruT = object()
lruB = object()
fifoT = object()
fifoB = object()
class ARCCacheSimulation(Simulation):
# Based on the paper ARC: A Self-Tuning, Low Overhead Replacement
# Cache by Nimrod Megiddo and Dharmendra S. Modha, USENIX FAST
# 2003. The paper describes a block-based cache. A lot of the
# details need to be fiddled to work with an object-based cache.
# For size issues, the key insight ended up being conditions
# A.1-A.5 rather than the details of the algorithm in Fig. 4.
extras = "lruThits", "evicts", "p"
def __init__(self, cachelimit):
Simulation.__init__(self, cachelimit)
# There are two pairs of linked lists. Each pair has a top and
# bottom half. The bottom half contains metadata, but not actual
# objects.
# LRU list of frequently used objects
self.lruT = Node2Q(None, None, lruT)
self.lruT.linkbefore(self.lruT)
self.lruT_len = 0
self.lruT_size = 0
self.lruB = Node2Q(None, None, lruB)
self.lruB.linkbefore(self.lruB)
self.lruB_len = 0
self.lruB_size = 0
# FIFO list of objects seen once
self.fifoT = Node2Q(None, None, fifoT)
self.fifoT.linkbefore(self.fifoT)
self.fifoT_len = 0
self.fifoT_size = 0
self.fifoB = Node2Q(None, None, fifoB)
self.fifoB.linkbefore(self.fifoB)
self.fifoB_len = 0
self.fifoB_size = 0
# maps oid to node
self.cache = {}
# The paper says that p should be adjust be 1 as the minimum:
# "The compound effect of such small increments and decrements
# to p s quite profound as we will demonstrated in the next
# section." Not really, as far as I can tell. In my traces
# with a very small cache, it was taking far too long to adjust
# towards favoring some FIFO component. I would guess that the
# chief difference is that our caches are much bigger than the
# ones they experimented with. Their biggest cache had 512K
# entries, while our smallest cache will have 40 times that many
# entries.
self.p = 0
# XXX multiply computed adjustments to p by walk_factor
self.walk_factor = 500
# statistics
self.total_hits = 0
self.total_lruThits = 0
self.total_fifoThits = 0
self.total_evicts = 0
def restart(self):
Simulation.restart(self)
self.hits = 0
self.lruThits = 0
self.fifoThits = 0
self.evicts = 0
def write(self, oid, size):
pass
def replace(self, lruB=False):
self.evicts += 1
self.total_evicts += 1
if self.fifoT_size > self.p or (lruB and self.fifoT_size == self.p):
node = self.fifoT.next
if node is self.fifoT:
return 0
assert node is not self.fifoT, self.stats()
node.linkbefore(self.fifoB)
self.fifoT_len -= 1
self.fifoT_size -= node.size
self.fifoB_len += 1
self.fifoB_size += node.size
else:
node = self.lruT.next
if node is self.lruT:
return 0
assert node is not self.lruT, self.stats()
node.linkbefore(self.lruB)
self.lruT_len -= 1
self.lruT_size -= node.size
self.lruB_len += 1
self.lruB_size += node.size
return node.size
def stats(self):
self.totalsize = self.lruT_size + self.fifoT_size
self.allsize = self.totalsize + self.lruB_size + self.fifoB_size
print "cachelimit = %s totalsize=%s allsize=%s" % (
addcommas(self.cachelimit),
addcommas(self.totalsize),
addcommas(self.allsize))
fmt = (
"p=%(p)d\n"
"lruT = %(lruT_len)5d / %(lruT_size)8d / %(lruThits)d\n"
"fifoT = %(fifoT_len)5d / %(fifoT_size)8d / %(fifoThits)d\n"
"lruB = %(lruB_len)5d / %(lruB_size)8d\n"
"fifoB = %(fifoB_len)5d / %(fifoB_size)8d\n"
"loads=%(loads)d hits=%(hits)d evicts=%(evicts)d\n"
)
print fmt % self.__dict__
def report(self):
self.total_p = self.p
Simulation.report(self)
## self.stats()
def load(self, oid, size):
## maybe(self.stats, p=0.002)
node = self.cache.get(oid)
if node is None:
# cache miss: We're going to insert a new object in fifoT.
# If fifo is full, we'll need to evict something to make
# room for it.
prev = need = size
while need > 0:
if size + self.fifoT_size + self.fifoB_size >= self.cachelimit:
if need + self.fifoT_size >= self.cachelimit:
node = self.fifoB.next
assert node is not self.fifoB, self.stats()
node.unlink()
del self.cache[node.oid]
self.fifoB_size -= node.size
self.fifoB_len -= 1
self.evicts += 1
self.total_evicts += 1
else:
node = self.fifoB.next
assert node is not self.fifoB, self.stats()
node.unlink()
del self.cache[node.oid]
self.fifoB_size -= node.size
self.fifoB_len -= 1
if self.fifoT_size + self.lruT_size > self.cachelimit:
need -= self.replace()
else:
incache_size = self.fifoT_size + self.lruT_size + need
total_size = (incache_size + self.fifoB_size
+ self.lruB_size)
if total_size >= self.cachelimit * 2:
node = self.lruB.next
if node is self.lruB:
break
assert node is not self.lruB
node.unlink()
del self.cache[node.oid]
self.lruB_size -= node.size
self.lruB_len -= 1
elif incache_size > self.cachelimit:
need -= self.replace()
else:
break
if need == prev:
# XXX hack, apparently we can't get rid of anything else
break
prev = need
node = Node2Q(oid, size)
node.linkbefore(self.fifoT)
self.fifoT_len += 1
self.fifoT_size += size
self.cache[oid] = node
else:
# a cache hit, but possibly in a bottom list that doesn't
# actually hold the object
if node.kind is lruT:
node.linkbefore(self.lruT)
self.hits += 1
self.total_hits += 1
self.lruThits += 1
self.total_lruThits += 1
elif node.kind is fifoT:
node.linkbefore(self.lruT)
self.fifoT_len -= 1
self.lruT_len += 1
self.fifoT_size -= node.size
self.lruT_size += node.size
self.hits += 1
self.total_hits += 1
self.fifoThits += 1
self.total_fifoThits += 1
elif node.kind is fifoB:
node.linkbefore(self.lruT)
self.fifoB_len -= 1
self.lruT_len += 1
self.fifoB_size -= node.size
self.lruT_size += node.size
# XXX need a better min than 1?
## print "adapt+", max(1, self.lruB_size // self.fifoB_size)
delta = max(1, self.lruB_size / max(1, self.fifoB_size))
self.p += delta * self.walk_factor
if self.p > self.cachelimit:
self.p = self.cachelimit
need = node.size
if self.lruT_size + self.fifoT_size + need > self.cachelimit:
while need > 0:
r = self.replace()
if not r:
break
need -= r
elif node.kind is lruB:
node.linkbefore(self.lruT)
self.lruB_len -= 1
self.lruT_len += 1
self.lruB_size -= node.size
self.lruT_size += node.size
# XXX need a better min than 1?
## print "adapt-", max(1, self.fifoB_size // self.lruB_size)
delta = max(1, self.fifoB_size / max(1, self.lruB_size))
self.p -= delta * self.walk_factor
if self.p < 0:
self.p = 0
need = node.size
if self.lruT_size + self.fifoT_size + need > self.cachelimit:
while need > 0:
r = self.replace(lruB=True)
if not r:
break
need -= r
def inval(self, oid):
pass
def extraheader(self):
pass
class OracleSimulation(LRUCacheSimulation):
# Not sure how to implement this yet. This is a cache where I
# cheat to see how good we could actually do. The cache
# replacement problem for multi-size caches is NP-hard, so we're
# not going to have an optimal solution.
# At the moment, the oracle is mostly blind. It knows which
# objects will be referenced more than once, so that it can
# ignore objects referenced only once. In most traces, these
# objects account for about 20% of references.
def __init__(self, cachelimit, filename):
LRUCacheSimulation.__init__(self, cachelimit)
self.count = {}
self.scan(filename)
def load(self, oid, size):
node = self.cache.get(oid)
if node is not None:
self.hits += 1
self.total_hits += 1
node.linkbefore(self.head)
else:
if oid in self.count:
self.write(oid, size)
def scan(self, filename):
# scan the file in advance to figure out which objects will
# be referenced more than once.
f = open(filename, "rb")
struct_unpack = struct.unpack
f_read = f.read
offset = 0
while 1:
# Read a record and decode it
r = f_read(8)
if len(r) < 8:
break
offset += 8
ts, code = struct_unpack(">ii", r)
if ts == 0:
# Must be a misaligned record caused by a crash
##print "Skipping 8 bytes at offset", offset-8
continue
r = f_read(16)
if len(r) < 16:
break
offset += 16
oid, serial = struct_unpack(">8s8s", r)
if code & 0x70 == 0x20:
# only look at loads
self.count[oid] = self.count.get(oid, 0) + 1
all = len(self.count)
# Now remove everything with count == 1
once = [oid for oid, count in self.count.iteritems()
if count == 1]
for oid in once:
del self.count[oid]
print "Scanned file, %d unique oids, %d repeats" % (
all, len(self.count))
class BuddyCacheSimulation(LRUCacheSimulation):
def __init__(self, cachelimit):
LRUCacheSimulation.__init__(self, roundup(cachelimit))
def restart(self):
LRUCacheSimulation.restart(self)
self.allocator = self.allocatorFactory(self.cachelimit)
def allocatorFactory(self, size):
return BuddyAllocator(size)
# LRUCacheSimulation.load() is just fine
def write(self, oid, size):
node = self.cache.get(oid)
if node is not None:
node.unlink()
assert self.head.next is not None
self.size -= node.size
self.allocator.free(node)
while 1:
node = self.allocator.alloc(size)
if node is not None:
break
# Failure to allocate. Evict something and try again.
node = self.head.next
assert node is not self.head
self.evicts += 1
self.total_evicts += 1
node.unlink()
assert self.head.next is not None
del self.cache[node.oid]
self.size -= node.size
self.allocator.free(node)
node.oid = oid
self.cache[oid] = node
node.linkbefore(self.head)
self.size += node.size
def inval(self, oid):
node = self.cache.get(oid)
if node is not None:
assert node.oid == oid
self.invals += 1
self.total_invals += 1
node.unlink()
assert self.head.next is not None
del self.cache[oid]
self.size -= node.size
assert self.size >= 0
self.allocator.free(node)
class SimpleCacheSimulation(BuddyCacheSimulation):
def allocatorFactory(self, size):
return SimpleAllocator(size)
def finish(self):
BuddyCacheSimulation.finish(self)
self.allocator.report()
MINSIZE = 256
class BuddyAllocator:
def __init__(self, cachelimit):
cachelimit = roundup(cachelimit)
self.cachelimit = cachelimit
self.avail = {} # Map rounded-up sizes to free list node heads
self.nodes = {} # Map address to node
k = MINSIZE
while k <= cachelimit:
self.avail[k] = n = Node(None, None) # Not BlockNode; has no addr
n.linkbefore(n)
k += k
node = BlockNode(None, cachelimit, 0)
self.nodes[0] = node
node.linkbefore(self.avail[cachelimit])
def alloc(self, size):
size = roundup(size)
k = size
while k <= self.cachelimit:
head = self.avail[k]
node = head.next
if node is not head:
break
k += k
else:
return None # Store is full, or block is too large
node.unlink()
size2 = node.size
while size2 > size:
size2 = size2 / 2
assert size2 >= size
node.size = size2
buddy = BlockNode(None, size2, node.addr + size2)
self.nodes[buddy.addr] = buddy
buddy.linkbefore(self.avail[size2])
node.oid = 1 # Flag as in-use
return node
def free(self, node):
assert node is self.nodes[node.addr]
assert node.prev is node.next is None
node.oid = None # Flag as free
while node.size < self.cachelimit:
buddy_addr = node.addr ^ node.size
buddy = self.nodes[buddy_addr]
assert buddy.addr == buddy_addr
if buddy.oid is not None or buddy.size != node.size:
break
# Merge node with buddy
buddy.unlink()
if buddy.addr < node.addr: # buddy prevails
del self.nodes[node.addr]
node = buddy
else: # node prevails
del self.nodes[buddy.addr]
node.size *= 2
assert node is self.nodes[node.addr]
node.linkbefore(self.avail[node.size])
def dump(self, msg=""):
if msg:
print msg,
size = MINSIZE
blocks = bytes = 0
while size <= self.cachelimit:
head = self.avail[size]
node = head.next
count = 0
while node is not head:
count += 1
node = node.next
if count:
print "%d:%d" % (size, count),
blocks += count
bytes += count*size
size += size
print "-- %d, %d" % (bytes, blocks)
def roundup(size):
k = MINSIZE
while k < size:
k += k
return k
class SimpleAllocator:
def __init__(self, arenasize):
self.arenasize = arenasize
self.avail = BlockNode(None, 0, 0) # Weird: empty block as list head
self.rover = self.avail
node = BlockNode(None, arenasize, 0)
node.linkbefore(self.avail)
self.taglo = {0: node}
self.taghi = {arenasize: node}
# Allocator statistics
self.nallocs = 0
self.nfrees = 0
self.allocloops = 0
self.freebytes = arenasize
self.freeblocks = 1
self.allocbytes = 0
self.allocblocks = 0
def report(self):
print ("NA=%d AL=%d NF=%d ABy=%d ABl=%d FBy=%d FBl=%d" %
(self.nallocs, self.allocloops,
self.nfrees,
self.allocbytes, self.allocblocks,
self.freebytes, self.freeblocks))
def alloc(self, size):
self.nallocs += 1
# First fit algorithm
rover = stop = self.rover
while 1:
self.allocloops += 1
if rover.size >= size:
break
rover = rover.next
if rover is stop:
return None # We went round the list without finding space
if rover.size == size:
self.rover = rover.next
rover.unlink()
del self.taglo[rover.addr]
del self.taghi[rover.addr + size]
self.freeblocks -= 1
self.allocblocks += 1
self.freebytes -= size
self.allocbytes += size
return rover
# Take space from the beginning of the roving pointer
assert rover.size > size
node = BlockNode(None, size, rover.addr)
del self.taglo[rover.addr]
rover.size -= size
rover.addr += size
self.taglo[rover.addr] = rover
#self.freeblocks += 0 # No change here
self.allocblocks += 1
self.freebytes -= size
self.allocbytes += size
return node
def free(self, node):
self.nfrees += 1
self.freeblocks += 1
self.allocblocks -= 1
self.freebytes += node.size
self.allocbytes -= node.size
node.linkbefore(self.avail)
self.taglo[node.addr] = node
self.taghi[node.addr + node.size] = node
x = self.taghi.get(node.addr)
if x is not None:
# Merge x into node
x.unlink()
self.freeblocks -= 1
del self.taglo[x.addr]
del self.taghi[x.addr + x.size]
del self.taglo[node.addr]
node.addr = x.addr
node.size += x.size
self.taglo[node.addr] = node
x = self.taglo.get(node.addr + node.size)
if x is not None:
# Merge x into node
x.unlink()
self.freeblocks -= 1
del self.taglo[x.addr]
del self.taghi[x.addr + x.size]
del self.taghi[node.addr + node.size]
node.size += x.size
self.taghi[node.addr + node.size] = node
# It's possible that either one of the merges above invalidated
# the rover.
# It's simplest to simply reset the rover to the newly freed block.
self.rover = node
def dump(self, msg=""):
if msg:
print msg,
count = 0
bytes = 0
node = self.avail.next
while node is not self.avail:
bytes += node.size
count += 1
node = node.next
print count, "free blocks,", bytes, "free bytes"
self.report()
class BlockNode(Node):
__slots__ = ['addr']
def __init__(self, oid, size, addr):
Node.__init__(self, oid, size)
self.addr = addr
def testallocator(factory=BuddyAllocator):
# Run one of Knuth's experiments as a test
import random
import heapq # This only runs with Python 2.3, folks :-)
reportfreq = 100
cachelimit = 2**17
cache = factory(cachelimit)
queue = []
T = 0
blocks = 0
while T < 5000:
while queue and queue[0][0] <= T:
time, node = heapq.heappop(queue)
assert time == T
##print "free addr=%d, size=%d" % (node.addr, node.size)
cache.free(node)
blocks -= 1
size = random.randint(100, 2000)
lifetime = random.randint(1, 100)
node = cache.alloc(size)
if node is None:
print "out of mem"
cache.dump("T=%4d: %d blocks;" % (T, blocks))
break
else:
##print "alloc addr=%d, size=%d" % (node.addr, node.size)
blocks += 1
heapq.heappush(queue, (T + lifetime, node))
T = T+1
if T % reportfreq == 0:
cache.dump("T=%4d: %d blocks;" % (T, blocks))
def hitrate(loads, hits):
return "%5.1f%%" % (100.0 * hits / max(1, loads))
def duration(secs):
mm, ss = divmod(secs, 60)
hh, mm = divmod(mm, 60)
if hh:
return "%d:%02d:%02d" % (hh, mm, ss)
if mm:
return "%d:%02d" % (mm, ss)
return "%d" % ss
def addcommas(n):
sign, s = '', str(n)
if s[0] == '-':
sign, s = '-', s[1:]
i = len(s) - 3
while i > 0:
s = s[:i] + ',' + s[i:]
i -= 3
return sign + s
import random
def maybe(f, p=0.5):
if random.random() < p:
f()
#############################################################################
# Thor-like eviction scheme.
#
# The cache keeps a list of all objects, and uses a travelling pointer
# to decay the worth of objects over time.
class ThorNode(Node):
__slots__ = ['worth']
def __init__(self, oid, size, worth=None):
Node.__init__(self, oid, size)
self.worth = worth
class ThorListHead(Node):
def __init__(self):
Node.__init__(self, 0, 0)
self.next = self.prev = self
class ThorSimulation(Simulation):
extras = "evicts", "trips"
def __init__(self, cachelimit):
Simulation.__init__(self, cachelimit)
# Maximum total of object sizes we keep in cache.
self.maxsize = cachelimit
# Current total of object sizes in cache.
self.currentsize = 0
# A worth byte maps to a set of all objects with that worth.
# This is cheap to keep updated, and makes finding low-worth
# objects for eviction trivial (just march over the worthsets
# list, in order).
self.worthsets = [Set() for dummy in range(256)]
# We keep a circular list of all objects in cache. currentobj
# walks around it forever. Each time _tick() is called, the
# worth of currentobj is decreased, basically by shifting
# right 1, and currentobj moves on to the next object. When
# an object is first inserted, it enters the list right before
# currentobj. When an object is accessed, its worth is
# increased by or'ing in 0x80. This scheme comes from the
# Thor system, and is an inexpensive way to account for both
# recency and frequency of access: recency is reflected in
# the leftmost bit set, and frequency by how many bits are
# set.
#
# Note: because evictions are interleaved with ticks,
# unlinking an object is tricky, lest we evict currentobj. The
# class _unlink method takes care of this properly.
self.listhead = ThorListHead()
self.currentobj = self.listhead
# Map an object.oid to its ThorNode.
self.oid2object = {}
self.total_evicts = self.total_trips = 0
# Unlink object from the circular list, taking care not to lose
# track of the current object. Always call this instead of
# invoking obj.unlink() directly.
def _unlink(self, obj):
assert obj is not self.listhead
if obj is self.currentobj:
self.currentobj = obj.next
obj.unlink()
# Change obj.worth to newworth, maintaining invariants.
def _change_worth(self, obj, newworth):
if obj.worth != newworth:
self.worthsets[obj.worth].remove(obj)
obj.worth = newworth
self.worthsets[newworth].add(obj)
def add(self, object):
assert object.oid not in self.oid2object
self.oid2object[object.oid] = object
newsize = self.currentsize + object.size
if newsize > self.maxsize:
self._evictbytes(newsize - self.maxsize)
self.currentsize += object.size
object.linkbefore(self.currentobj)
if object.worth is None:
# Give smaller objects higher initial worth. This favors kicking
# out unreferenced large objects before kicking out unreferenced
# small objects. On real life traces, this is a significant
# win for the hit rate.
object.worth = 32 - int(round(math.log(object.size, 2)))
self.worthsets[object.worth].add(object)
# Decrease the worth of the current object, and advance the
# current object.
def _tick(self):
c = self.currentobj
if c is self.listhead:
c = c.next
if c is self.listhead: # list is empty
return
self.total_trips += 1
self.trips += 1
self._change_worth(c, (c.worth + 1) >> 1)
self.currentobj = c.next
def access(self, oid):
self._tick()
obj = self.oid2object.get(oid)
if obj is None:
return None
self._change_worth(obj, obj.worth | 0x80)
return obj
# Evict objects of least worth first, until at least nbytes bytes
# have been freed.
def _evictbytes(self, nbytes):
for s in self.worthsets:
while s:
if nbytes <= 0:
return
obj = s.pop()
nbytes -= obj.size
self._evictobj(obj)
def _evictobj(self, obj):
self.currentsize -= obj.size
self.worthsets[obj.worth].discard(obj)
del self.oid2object[obj.oid]
self._unlink(obj)
self.evicts += 1
self.total_evicts += 1
def _evict_without_bumping_evict_stats(self, obj):
self._evictobj(obj)
self.evicts -= 1
self.total_evicts -= 1
# Simulator overrides from here on.
def restart(self):
# Reset base class
Simulation.restart(self)
# Reset additional per-run statistics
self.evicts = self.trips = 0
def write(self, oid, size):
obj = self.oid2object.get(oid)
worth = None
if obj is not None:
worth = obj.worth
self._evict_without_bumping_evict_stats(obj)
self.add(ThorNode(oid, size, worth))
def load(self, oid, size):
if self.access(oid) is not None:
self.hits += 1
self.total_hits += 1
else:
self.write(oid, size)
def inval(self, oid):
obj = self.oid2object.get(oid)
if obj is not None:
self.invals += 1
self.total_invals += 1
self._evict_without_bumping_evict_stats(obj)
# Take the "x" off to see additional stats after each restart period.
def xreport(self):
Simulation.report(self)
print 'non-empty worth sets', sum(map(bool, self.worthsets)),
print 'objects', len(self.oid2object),
print 'size', self.currentsize
#############################################################################
# Perfection: What if the cache were unbounded, and never forgot anything?
# This simulator answers that question directly; the cache size parameter
# isn't used.
class UnboundedSimulation(Simulation):
extraname = 'evicts' # for some reason we *have* to define >= 1 extra
def __init__(self, cachelimit):
Simulation.__init__(self, cachelimit)
self.oids = Set()
self.evicts = self.total_evicts = 0
def write(self, oid, size):
self.oids.add(oid)
def load(self, oid, size):
if oid in self.oids:
self.hits += 1
self.total_hits += 1
else:
self.oids.add(oid)
def inval(self, oid):
if oid in self.oids:
self.invals += 1
self.total_invals += 1
self.oids.remove(oid)
if __name__ == "__main__":
sys.exit(main())
ZODB/src/scripts/space.py 0000640 0001750 0000144 00000003037 10144554637 015121 0 ustar ctheune users #!/usr/bin/env python2.3
"""Report on the space used by objects in a storage.
usage: space.py data.fs
The current implementation only supports FileStorage.
Current limitations / simplifications: Ignores revisions and versions.
"""
from ZODB.FileStorage import FileStorage
from ZODB.utils import U64, get_pickle_metadata
def run(path, v=0):
fs = FileStorage(path, read_only=1)
# break into the file implementation
if hasattr(fs._index, 'iterkeys'):
iter = fs._index.iterkeys()
else:
iter = fs._index.keys()
totals = {}
for oid in iter:
data, serialno = fs.load(oid, '')
mod, klass = get_pickle_metadata(data)
key = "%s.%s" % (mod, klass)
bytes, count = totals.get(key, (0, 0))
bytes += len(data)
count += 1
totals[key] = bytes, count
if v:
print "%8s %5d %s" % (U64(oid), len(data), key)
L = totals.items()
L.sort(lambda a, b: cmp(a[1], b[1]))
L.reverse()
print "Totals per object class:"
for key, (bytes, count) in L:
print "%8d %8d %s" % (count, bytes, key)
def main():
import sys
import getopt
try:
opts, args = getopt.getopt(sys.argv[1:], "v")
except getopt.error, msg:
print msg
print "usage: space.py [-v] Data.fs"
sys.exit(2)
if len(args) != 1:
print "usage: space.py [-v] Data.fs"
sys.exit(2)
v = 0
for o, a in opts:
if o == "-v":
v += 1
path = args[0]
run(path, v)
if __name__ == "__main__":
main()
ZODB/src/scripts/parsezeolog.py 0000640 0001750 0000144 00000006476 10051215041 016346 0 ustar ctheune users #!/usr/bin/env python2.3
"""Parse the BLATHER logging generated by ZEO2.
An example of the log format is:
2002-04-15T13:05:29 BLATHER(-100) ZEO Server storea(3235680, [714], 235339406490168806) ('10.0.26.30', 45514)
"""
import re
import time
rx_time = re.compile('(\d\d\d\d-\d\d-\d\d)T(\d\d:\d\d:\d\d)')
def parse_time(line):
"""Return the time portion of a zLOG line in seconds or None."""
mo = rx_time.match(line)
if mo is None:
return None
date, time_ = mo.group(1, 2)
date_l = [int(elt) for elt in date.split('-')]
time_l = [int(elt) for elt in time_.split(':')]
return int(time.mktime(date_l + time_l + [0, 0, 0]))
rx_meth = re.compile("zrpc:\d+ calling (\w+)\((.*)")
def parse_method(line):
pass
def parse_line(line):
"""Parse a log entry and return time, method info, and client."""
t = parse_time(line)
if t is None:
return None, None
mo = rx_meth.search(line)
if mo is None:
return None, None
meth_name = mo.group(1)
meth_args = mo.group(2).strip()
if meth_args.endswith(')'):
meth_args = meth_args[:-1]
meth_args = [s.strip() for s in meth_args.split(",")]
m = meth_name, tuple(meth_args)
return t, m
class TStats:
counter = 1
def __init__(self):
self.id = TStats.counter
TStats.counter += 1
fields = ("time", "vote", "done", "user", "path")
fmt = "%-24s %5s %5s %-15s %s"
hdr = fmt % fields
def report(self):
"""Print a report about the transaction"""
t = time.ctime(self.begin)
if hasattr(self, "vote"):
d_vote = self.vote - self.begin
else:
d_vote = "*"
if hasattr(self, "finish"):
d_finish = self.finish - self.begin
else:
d_finish = "*"
print self.fmt % (time.ctime(self.begin), d_vote, d_finish,
self.user, self.url)
class TransactionParser:
def __init__(self):
self.txns = {}
self.skipped = 0
def parse(self, line):
t, m = parse_line(line)
if t is None:
return
name = m[0]
meth = getattr(self, name, None)
if meth is not None:
meth(t, m[1])
def tpc_begin(self, time, args):
t = TStats()
t.begin = time
t.user = args[1]
t.url = args[2]
t.objects = []
tid = eval(args[0])
self.txns[tid] = t
def get_txn(self, args):
tid = eval(args[0])
try:
return self.txns[tid]
except KeyError:
print "uknown tid", repr(tid)
return None
def tpc_finish(self, time, args):
t = self.get_txn(args)
if t is None:
return
t.finish = time
def vote(self, time, args):
t = self.get_txn(args)
if t is None:
return
t.vote = time
def get_txns(self):
L = [(t.id, t) for t in self.txns.values()]
L.sort()
return [t for (id, t) in L]
if __name__ == "__main__":
import fileinput
p = TransactionParser()
i = 0
for line in fileinput.input():
i += 1
try:
p.parse(line)
except:
print "line", i
raise
print "Transaction: %d" % len(p.txns)
print TStats.hdr
for txn in p.get_txns():
txn.report()
ZODB/src/scripts/zeopack.py 0000750 0001750 0000144 00000006161 10305413212 015443 0 ustar ctheune users #!/usr/bin/env python2.3
"""Connect to a ZEO server and ask it to pack.
Usage: zeopack.py [options]
Options:
-p port -- port to connect to
-h host -- host to connect to (default is current host)
-U path -- Unix-domain socket to connect to
-S name -- storage name (default is '1')
-d days -- pack objects more than days old
-1 -- Connect to a ZEO 1 server
-W -- wait for server to come up. Normally the script tries to
connect for 10 seconds, then exits with an error. The -W
option is only supported with ZEO 1.
You must specify either -p and -h or -U.
"""
import getopt
import socket
import sys
import time
from ZEO.ClientStorage import ClientStorage
WAIT = 10 # wait no more than 10 seconds for client to connect
def connect(storage):
# The connect-on-startup logic that ZEO provides isn't too useful
# for this script. We'd like to client to attempt to startup, but
# fail if it can't get through to the server after a reasonable
# amount of time. There's no external support for this, so we'll
# expose the ZEO 1.0 internals. (consenting adults only)
t0 = time.time()
while t0 + WAIT > time.time():
storage._call.connect()
if storage._connected:
return
raise RuntimeError("Unable to connect to ZEO server")
def pack1(addr, storage, days, wait):
cs = ClientStorage(addr, storage=storage,
wait_for_server_on_startup=wait)
if wait:
# _startup() is an artifact of the way ZEO 1.0 works. The
# ClientStorage doesn't get fully initialized until registerDB()
# is called. The only thing we care about, though, is that
# registerDB() calls _startup().
cs._startup()
else:
connect(cs)
cs.invalidator = None
cs.pack(wait=1, days=days)
cs.close()
def pack2(addr, storage, days):
cs = ClientStorage(addr, storage=storage, wait=1, read_only=1)
cs.pack(wait=1, days=days)
cs.close()
def usage(exit=1):
print __doc__
print " ".join(sys.argv)
sys.exit(exit)
def main():
host = None
port = None
unix = None
storage = '1'
days = 0
wait = 0
zeoversion = 2
try:
opts, args = getopt.getopt(sys.argv[1:], 'p:h:U:S:d:W1')
for o, a in opts:
if o == '-p':
port = int(a)
elif o == '-h':
host = a
elif o == '-U':
unix = a
elif o == '-S':
storage = a
elif o == '-d':
days = int(a)
elif o == '-W':
wait = 1
elif o == '-1':
zeoversion = 1
except Exception, err:
print err
usage()
if unix is not None:
addr = unix
else:
if host is None:
host = socket.gethostname()
if port is None:
usage()
addr = host, port
if zeoversion == 1:
pack1(addr, storage, days, wait)
else:
pack2(addr, storage, days)
if __name__ == "__main__":
try:
main()
except Exception, err:
print err
sys.exit(1)
ZODB/src/scripts/fsdump.py 0000640 0001750 0000144 00000000306 10051215041 015274 0 ustar ctheune users #!/usr/bin/env python2.3
"""Print a text summary of the contents of a FileStorage."""
from ZODB.FileStorage.fsdump import fsdump
if __name__ == "__main__":
import sys
fsdump(sys.argv[1])
ZODB/src/scripts/stats.py 0000750 0001750 0000144 00000030365 10276710420 015160 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Trace file statistics analyzer.
Usage: stats.py [-h] [-i interval] [-q] [-s] [-S] [-v] [-X] tracefile
-h: print histogram of object load frequencies
-i: summarizing interval in minutes (default 15; max 60)
-q: quiet; don't print summaries
-s: print histogram of object sizes
-S: don't print statistics
-v: verbose; print each record
-X: enable heuristic checking for misaligned records: oids > 2**32
will be rejected; this requires the tracefile to be seekable
"""
"""File format:
Each record is 26 bytes, plus a variable number of bytes to store an oid,
with the following layout. Numbers are big-endian integers.
Offset Size Contents
0 4 timestamp (seconds since 1/1/1970)
4 3 data size, in 256-byte increments, rounded up
7 1 code (see below)
8 2 object id length
10 8 start tid
18 8 end tid
26 variable object id
The code at offset 7 packs three fields:
Mask bits Contents
0x80 1 set if there was a non-empty version string
0x7e 6 function and outcome code
0x01 1 current cache file (0 or 1)
The "current cache file" bit is no longer used; it refers to a 2-file
cache scheme used before ZODB 3.3.
The function and outcome codes are documented in detail at the end of
this file in the 'explain' dictionary. Note that the keys there (and
also the arguments to _trace() in ClientStorage.py) are 'code & 0x7e',
i.e. the low bit is always zero.
"""
import sys
import time
import getopt
import struct
from types import StringType
def usage(msg):
print >> sys.stderr, msg
print >> sys.stderr, __doc__
def main():
# Parse options
verbose = False
quiet = False
dostats = True
print_size_histogram = False
print_histogram = False
interval = 15*60 # Every 15 minutes
heuristic = False
try:
opts, args = getopt.getopt(sys.argv[1:], "hi:qsSvX")
except getopt.error, msg:
usage(msg)
return 2
for o, a in opts:
if o == '-h':
print_histogram = True
elif o == "-i":
interval = int(60 * float(a))
if interval <= 0:
interval = 60
elif interval > 3600:
interval = 3600
elif o == "-q":
quiet = True
verbose = False
elif o == "-s":
print_size_histogram = True
elif o == "-S":
dostats = False
elif o == "-v":
verbose = True
elif o == '-X':
heuristic = True
else:
assert False, (o, opt)
if len(args) != 1:
usage("exactly one file argument required")
return 2
filename = args[0]
# Open file
if filename.endswith(".gz"):
# Open gzipped file
try:
import gzip
except ImportError:
print >> sys.stderr, "can't read gzipped files (no module gzip)"
return 1
try:
f = gzip.open(filename, "rb")
except IOError, msg:
print >> sys.stderr, "can't open %s: %s" % (filename, msg)
return 1
elif filename == '-':
# Read from stdin
f = sys.stdin
else:
# Open regular file
try:
f = open(filename, "rb")
except IOError, msg:
print >> sys.stderr, "can't open %s: %s" % (filename, msg)
return 1
rt0 = time.time()
bycode = {} # map code to count of occurrences
byinterval = {} # map code to count in current interval
records = 0 # number of trace records read
versions = 0 # number of trace records with versions
datarecords = 0 # number of records with dlen set
datasize = 0L # sum of dlen across records with dlen set
oids = {} # map oid to number of times it was loaded
bysize = {} # map data size to number of loads
bysizew = {} # map data size to number of writes
total_loads = 0
t0 = None # first timestamp seen
te = None # most recent timestamp seen
h0 = None # timestamp at start of current interval
he = None # timestamp at end of current interval
thisinterval = None # generally te//interval
f_read = f.read
unpack = struct.unpack
FMT = ">iiH8s8s"
FMT_SIZE = struct.calcsize(FMT)
assert FMT_SIZE == 26
# Read file, gathering statistics, and printing each record if verbose.
try:
while 1:
r = f_read(FMT_SIZE)
if len(r) < FMT_SIZE:
break
ts, code, oidlen, start_tid, end_tid = unpack(FMT, r)
if ts == 0:
# Must be a misaligned record caused by a crash.
if not quiet:
print "Skipping 8 bytes at offset", f.tell() - FMT_SIZE
f.seek(f.tell() - FMT_SIZE + 8)
continue
oid = f_read(oidlen)
if len(oid) < oidlen:
break
records += 1
if t0 is None:
t0 = ts
thisinterval = t0 // interval
h0 = he = ts
te = ts
if ts // interval != thisinterval:
if not quiet:
dumpbyinterval(byinterval, h0, he)
byinterval = {}
thisinterval = ts // interval
h0 = ts
he = ts
dlen, code = code & 0x7fffff00, code & 0xff
if dlen:
datarecords += 1
datasize += dlen
if code & 0x80:
version = 'V'
versions += 1
else:
version = '-'
code &= 0x7e
bycode[code] = bycode.get(code, 0) + 1
byinterval[code] = byinterval.get(code, 0) + 1
if dlen:
if code & 0x70 == 0x20: # All loads
bysize[dlen] = d = bysize.get(dlen) or {}
d[oid] = d.get(oid, 0) + 1
elif code & 0x70 == 0x50: # All stores
bysizew[dlen] = d = bysizew.get(dlen) or {}
d[oid] = d.get(oid, 0) + 1
if verbose:
print "%s %02x %s %016x %016x %c %s" % (
time.ctime(ts)[4:-5],
code,
oid_repr(oid),
U64(start_tid),
U64(end_tid),
version,
dlen and str(dlen) or "")
if code & 0x70 == 0x20:
oids[oid] = oids.get(oid, 0) + 1
total_loads += 1
elif code == 0x00: # restart
if not quiet:
dumpbyinterval(byinterval, h0, he)
byinterval = {}
thisinterval = ts // interval
h0 = he = ts
if not quiet:
print time.ctime(ts)[4:-5],
print '='*20, "Restart", '='*20
except KeyboardInterrupt:
print "\nInterrupted. Stats so far:\n"
end_pos = f.tell()
f.close()
rte = time.time()
if not quiet:
dumpbyinterval(byinterval, h0, he)
# Error if nothing was read
if not records:
print >> sys.stderr, "No records processed"
return 1
# Print statistics
if dostats:
print
print "Read %s trace records (%s bytes) in %.1f seconds" % (
addcommas(records), addcommas(end_pos), rte-rt0)
print "Versions: %s records used a version" % addcommas(versions)
print "First time: %s" % time.ctime(t0)
print "Last time: %s" % time.ctime(te)
print "Duration: %s seconds" % addcommas(te-t0)
print "Data recs: %s (%.1f%%), average size %.1f KB" % (
addcommas(datarecords),
100.0 * datarecords / records,
datasize / 1024.0 / datarecords)
print "Hit rate: %.1f%% (load hits / loads)" % hitrate(bycode)
print
codes = bycode.keys()
codes.sort()
print "%13s %4s %s" % ("Count", "Code", "Function (action)")
for code in codes:
print "%13s %02x %s" % (
addcommas(bycode.get(code, 0)),
code,
explain.get(code) or "*** unknown code ***")
# Print histogram.
if print_histogram:
print
print "Histogram of object load frequency"
total = len(oids)
print "Unique oids: %s" % addcommas(total)
print "Total loads: %s" % addcommas(total_loads)
s = addcommas(total)
width = max(len(s), len("objects"))
fmt = "%5d %" + str(width) + "s %5.1f%% %5.1f%% %5.1f%%"
hdr = "%5s %" + str(width) + "s %6s %6s %6s"
print hdr % ("loads", "objects", "%obj", "%load", "%cum")
cum = 0.0
for binsize, count in histogram(oids):
obj_percent = 100.0 * count / total
load_percent = 100.0 * count * binsize / total_loads
cum += load_percent
print fmt % (binsize, addcommas(count),
obj_percent, load_percent, cum)
# Print size histogram.
if print_size_histogram:
print
print "Histograms of object sizes"
print
dumpbysize(bysizew, "written", "writes")
dumpbysize(bysize, "loaded", "loads")
def dumpbysize(bysize, how, how2):
print
print "Unique sizes %s: %s" % (how, addcommas(len(bysize)))
print "%10s %6s %6s" % ("size", "objs", how2)
sizes = bysize.keys()
sizes.sort()
for size in sizes:
loads = 0
for n in bysize[size].itervalues():
loads += n
print "%10s %6d %6d" % (addcommas(size),
len(bysize.get(size, "")),
loads)
def dumpbyinterval(byinterval, h0, he):
loads = hits = 0
for code in byinterval:
if code & 0x70 == 0x20:
n = byinterval[code]
loads += n
if code in (0x22, 0x26):
hits += n
if not loads:
return
if loads:
hr = 100.0 * hits / loads
else:
hr = 0.0
print "%s-%s %10s loads, %10s hits,%5.1f%% hit rate" % (
time.ctime(h0)[4:-8], time.ctime(he)[14:-8],
addcommas(loads), addcommas(hits), hr)
def hitrate(bycode):
loads = hits = 0
for code in bycode:
if code & 0x70 == 0x20:
n = bycode[code]
loads += n
if code in (0x22, 0x26):
hits += n
if loads:
return 100.0 * hits / loads
else:
return 0.0
def histogram(d):
bins = {}
for v in d.itervalues():
bins[v] = bins.get(v, 0) + 1
L = bins.items()
L.sort()
return L
def U64(s):
return struct.unpack(">Q", s)[0]
def oid_repr(oid):
if isinstance(oid, StringType) and len(oid) == 8:
return '%16x' % U64(oid)
else:
return repr(oid)
def addcommas(n):
sign, s = '', str(n)
if s[0] == '-':
sign, s = '-', s[1:]
i = len(s) - 3
while i > 0:
s = s[:i] + ',' + s[i:]
i -= 3
return sign + s
explain = {
# The first hex digit shows the operation, the second the outcome.
# If the second digit is in "02468" then it is a 'miss'.
# If it is in "ACE" then it is a 'hit'.
0x00: "_setup_trace (initialization)",
0x10: "invalidate (miss)",
0x1A: "invalidate (hit, version)",
0x1C: "invalidate (hit, saving non-current)",
# 0x1E can occur during startup verification.
0x1E: "invalidate (hit, discarding current or non-current)",
0x20: "load (miss)",
0x22: "load (hit)",
0x24: "load (non-current, miss)",
0x26: "load (non-current, hit)",
0x50: "store (version)",
0x52: "store (current, non-version)",
0x54: "store (non-current)",
}
if __name__ == "__main__":
sys.exit(main())
ZODB/src/scripts/netspace.py 0000640 0001750 0000144 00000006324 10214421637 015621 0 ustar ctheune users #!/usr/bin/env python2.3
"""Report on the net size of objects counting subobjects.
usage: netspace.py [-P | -v] data.fs
-P: do a pack first
-v: print info for all objects, even if a traversal path isn't found
"""
import ZODB
from ZODB.FileStorage import FileStorage
from ZODB.utils import U64, get_pickle_metadata
from ZODB.referencesf import referencesf
def find_paths(root, maxdist):
"""Find Python attribute traversal paths for objects to maxdist distance.
Starting at a root object, traverse attributes up to distance levels
from the root, looking for persistent objects. Return a dict
mapping oids to traversal paths.
TODO: Assumes that the keys of the root are not themselves
persistent objects.
TODO: Doesn't traverse containers.
"""
paths = {}
# Handle the root as a special case because it's a dict
objs = []
for k, v in root.items():
oid = getattr(v, '_p_oid', None)
objs.append((k, v, oid, 0))
for path, obj, oid, dist in objs:
if oid is not None:
paths[oid] = path
if dist < maxdist:
getattr(obj, 'foo', None) # unghostify
try:
items = obj.__dict__.items()
except AttributeError:
continue
for k, v in items:
oid = getattr(v, '_p_oid', None)
objs.append(("%s.%s" % (path, k), v, oid, dist + 1))
return paths
def main(path):
fs = FileStorage(path, read_only=1)
if PACK:
fs.pack()
db = ZODB.DB(fs)
rt = db.open().root()
paths = find_paths(rt, 3)
def total_size(oid):
cache = {}
cache_size = 1000
def _total_size(oid, seen):
v = cache.get(oid)
if v is not None:
return v
data, serialno = fs.load(oid, '')
size = len(data)
for suboid in referencesf(data):
if seen.has_key(suboid):
continue
seen[suboid] = 1
size += _total_size(suboid, seen)
cache[oid] = size
if len(cache) == cache_size:
cache.popitem()
return size
return _total_size(oid, {})
keys = fs._index.keys()
keys.sort()
keys.reverse()
if not VERBOSE:
# If not running verbosely, don't print an entry for an object
# unless it has an entry in paths.
keys = filter(paths.has_key, keys)
fmt = "%8s %5d %8d %s %s.%s"
for oid in keys:
data, serialno = fs.load(oid, '')
mod, klass = get_pickle_metadata(data)
refs = referencesf(data)
path = paths.get(oid, '-')
print fmt % (U64(oid), len(data), total_size(oid), path, mod, klass)
if __name__ == "__main__":
import sys
import getopt
PACK = 0
VERBOSE = 0
try:
opts, args = getopt.getopt(sys.argv[1:], 'Pv')
path, = args
except getopt.error, err:
print err
print __doc__
sys.exit(2)
except ValueError:
print "expected one argument, got", len(args)
print __doc__
sys.exit(2)
for o, v in opts:
if o == '-P':
PACK = 1
if o == '-v':
VERBOSE += 1
main(path)
ZODB/src/scripts/zeoctl.py 0000750 0001750 0000144 00000001402 10307651713 015314 0 ustar ctheune users #!/usr/bin/env python2.3
##############################################################################
#
# Copyright (c) 2005 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Wrapper script for zdctl.py that causes it to use the ZEO schema."""
from ZEO.zeoctl import main
main()
ZODB/src/scripts/zeoqueue.py 0000750 0001750 0000144 00000025535 10214427545 015674 0 ustar ctheune users #!/usr/bin/env python2.3
"""Report on the number of currently waiting clients in the ZEO queue.
Usage: %(PROGRAM)s [options] logfile
Options:
-h / --help
Print this help text and exit.
-v / --verbose
Verbose output
-f file
--file file
Use the specified file to store the incremental state as a pickle. If
not given, %(STATEFILE)s is used.
-r / --reset
Reset the state of the tool. This blows away any existing state
pickle file and then exits -- it does not parse the file. Use this
when you rotate log files so that the next run will parse from the
beginning of the file.
"""
import os
import re
import sys
import time
import errno
import getopt
import cPickle as pickle
COMMASPACE = ', '
STATEFILE = 'zeoqueue.pck'
PROGRAM = sys.argv[0]
try:
True, False
except NameError:
True = 1
False = 0
tcre = re.compile(r"""
(?P
\d{4}- # year
\d{2}- # month
\d{2}) # day
T # separator
(?P
\d{2}: # hour
\d{2}: # minute
\d{2}) # second
""", re.VERBOSE)
ccre = re.compile(r"""
zrpc-conn:(?P\d+.\d+.\d+.\d+:\d+)\s+
calling\s+
(?P
\w+) # the method
\( # args open paren
\' # string quote start
(?P
\S+) # first argument -- usually the tid
\' # end of string
(?P
.*) # rest of line
""", re.VERBOSE)
wcre = re.compile(r'Clients waiting: (?P\d+)')
def parse_time(line):
"""Return the time portion of a zLOG line in seconds or None."""
mo = tcre.match(line)
if mo is None:
return None
date, time_ = mo.group('ymd', 'hms')
date_l = [int(elt) for elt in date.split('-')]
time_l = [int(elt) for elt in time_.split(':')]
return int(time.mktime(date_l + time_l + [0, 0, 0]))
class Txn:
"""Track status of single transaction."""
def __init__(self, tid):
self.tid = tid
self.hint = None
self.begin = None
self.vote = None
self.abort = None
self.finish = None
self.voters = []
def isactive(self):
if self.begin and not (self.abort or self.finish):
return True
else:
return False
class Status:
"""Track status of ZEO server by replaying log records.
We want to keep track of several events:
- The last committed transaction.
- The last committed or aborted transaction.
- The last transaction that got the lock but didn't finish.
- The client address doing the first vote of a transaction.
- The number of currently active transactions.
- The number of reported queued transactions.
- Client restarts.
- Number of current connections (but this might not be useful).
We can observe these events by reading the following sorts of log
entries:
2002-12-16T06:16:05 BLATHER(-100) zrpc:12649 calling
tpc_begin('\x03I\x90((\xdbp\xd5', '', 'QueueCatal...
2002-12-16T06:16:06 BLATHER(-100) zrpc:12649 calling
vote('\x03I\x90((\xdbp\xd5')
2002-12-16T06:16:06 BLATHER(-100) zrpc:12649 calling
tpc_finish('\x03I\x90((\xdbp\xd5')
2002-12-16T10:46:10 INFO(0) ZSS:12649:1 Transaction blocked waiting
for storage. Clients waiting: 1.
2002-12-16T06:15:57 BLATHER(-100) zrpc:12649 connect from
('10.0.26.54', 48983):
2002-12-16T10:30:09 INFO(0) ZSS:12649:1 disconnected
"""
def __init__(self):
self.lineno = 0
self.pos = 0
self.reset()
def reset(self):
self.commit = None
self.commit_or_abort = None
self.last_unfinished = None
self.n_active = 0
self.n_blocked = 0
self.n_conns = 0
self.t_restart = None
self.txns = {}
def iscomplete(self):
# The status report will always be complete if we encounter an
# explicit restart.
if self.t_restart is not None:
return True
# If we haven't seen a restart, assume that seeing a finished
# transaction is good enough.
return self.commit is not None
def process_file(self, fp):
if self.pos:
if VERBOSE:
print 'seeking to file position', self.pos
fp.seek(self.pos)
while True:
line = fp.readline()
if not line:
break
self.lineno += 1
self.process(line)
self.pos = fp.tell()
def process(self, line):
if line.find("calling") != -1:
self.process_call(line)
elif line.find("connect") != -1:
self.process_connect(line)
# test for "locked" because word may start with "B" or "b"
elif line.find("locked") != -1:
self.process_block(line)
elif line.find("Starting") != -1:
self.process_start(line)
def process_call(self, line):
mo = ccre.search(line)
if mo is None:
return
called_method = mo.group('method')
# Exit early if we've got zeoLoad, because it's the most
# frequently called method and we don't use it.
if called_method == "zeoLoad":
return
t = parse_time(line)
meth = getattr(self, "call_%s" % called_method, None)
if meth is None:
return
client = mo.group('addr')
tid = mo.group('tid')
rest = mo.group('rest')
meth(t, client, tid, rest)
def process_connect(self, line):
pass
def process_block(self, line):
mo = wcre.search(line)
if mo is None:
# assume that this was a restart message for the last blocked
# transaction.
self.n_blocked = 0
else:
self.n_blocked = int(mo.group('num'))
def process_start(self, line):
if line.find("Starting ZEO server") != -1:
self.reset()
self.t_restart = parse_time(line)
def call_tpc_begin(self, t, client, tid, rest):
txn = Txn(tid)
txn.begin = t
if rest[0] == ',':
i = 1
while rest[i].isspace():
i += 1
rest = rest[i:]
txn.hint = rest
self.txns[tid] = txn
self.n_active += 1
self.last_unfinished = txn
def call_vote(self, t, client, tid, rest):
txn = self.txns.get(tid)
if txn is None:
print "Oops!"
txn = self.txns[tid] = Txn(tid)
txn.vote = t
txn.voters.append(client)
def call_tpc_abort(self, t, client, tid, rest):
txn = self.txns.get(tid)
if txn is None:
print "Oops!"
txn = self.txns[tid] = Txn(tid)
txn.abort = t
txn.voters = []
self.n_active -= 1
if self.commit_or_abort:
# delete the old transaction
try:
del self.txns[self.commit_or_abort.tid]
except KeyError:
pass
self.commit_or_abort = txn
def call_tpc_finish(self, t, client, tid, rest):
txn = self.txns.get(tid)
if txn is None:
print "Oops!"
txn = self.txns[tid] = Txn(tid)
txn.finish = t
txn.voters = []
self.n_active -= 1
if self.commit:
# delete the old transaction
try:
del self.txns[self.commit.tid]
except KeyError:
pass
if self.commit_or_abort:
# delete the old transaction
try:
del self.txns[self.commit_or_abort.tid]
except KeyError:
pass
self.commit = self.commit_or_abort = txn
def report(self):
print "Blocked transactions:", self.n_blocked
if not VERBOSE:
return
if self.t_restart:
print "Server started:", time.ctime(self.t_restart)
if self.commit is not None:
t = self.commit_or_abort.finish
if t is None:
t = self.commit_or_abort.abort
print "Last finished transaction:", time.ctime(t)
# the blocked transaction should be the first one that calls vote
L = [(txn.begin, txn) for txn in self.txns.values()]
L.sort()
for x, txn in L:
if txn.isactive():
began = txn.begin
if txn.voters:
print "Blocked client (first vote):", txn.voters[0]
print "Blocked transaction began at:", time.ctime(began)
print "Hint:", txn.hint
print "Idle time: %d sec" % int(time.time() - began)
break
def usage(code, msg=''):
print >> sys.stderr, __doc__ % globals()
if msg:
print >> sys.stderr, msg
sys.exit(code)
def main():
global VERBOSE
VERBOSE = 0
file = STATEFILE
reset = False
# -0 is a secret option used for testing purposes only
seek = True
try:
opts, args = getopt.getopt(sys.argv[1:], 'vhf:r0',
['help', 'verbose', 'file=', 'reset'])
except getopt.error, msg:
usage(1, msg)
for opt, arg in opts:
if opt in ('-h', '--help'):
usage(0)
elif opt in ('-v', '--verbose'):
VERBOSE += 1
elif opt in ('-f', '--file'):
file = arg
elif opt in ('-r', '--reset'):
reset = True
elif opt == '-0':
seek = False
if reset:
# Blow away the existing state file and exit
try:
os.unlink(file)
if VERBOSE:
print 'removing pickle state file', file
except OSError, e:
if e.errno <> errno.ENOENT:
raise
return
if not args:
usage(1, 'logfile is required')
if len(args) > 1:
usage(1, 'too many arguments: %s' % COMMASPACE.join(args))
path = args[0]
# Get the previous status object from the pickle file, if it is available
# and if the --reset flag wasn't given.
status = None
try:
statefp = open(file, 'rb')
try:
status = pickle.load(statefp)
if VERBOSE:
print 'reading status from file', file
finally:
statefp.close()
except IOError, e:
if e.errno <> errno.ENOENT:
raise
if status is None:
status = Status()
if VERBOSE:
print 'using new status'
if not seek:
status.pos = 0
fp = open(path, 'rb')
try:
status.process_file(fp)
finally:
fp.close()
# Save state
statefp = open(file, 'wb')
pickle.dump(status, statefp, 1)
statefp.close()
# Print the report and return the number of blocked clients in the exit
# status code.
status.report()
sys.exit(status.n_blocked)
if __name__ == "__main__":
main()
ZODB/src/scripts/zeoup.py 0000750 0001750 0000144 00000010012 10375443702 015155 0 ustar ctheune users #!/usr/bin/env python2.3
"""Make sure a ZEO server is running.
usage: zeoup.py [options]
The test will connect to a ZEO server, load the root object, and attempt to
update the zeoup counter in the root. It will report success if it updates
the counter or if it gets a ConflictError. A ConflictError is considered a
success, because the client was able to start a transaction.
Options:
-p port -- port to connect to
-h host -- host to connect to (default is current host)
-S storage -- storage name (default '1')
-U path -- Unix-domain socket to connect to
--nowrite -- Do not update the zeoup counter.
-1 -- Connect to a ZEO 1.0 server.
You must specify either -p and -h or -U.
"""
import getopt
import logging
import socket
import sys
import time
from persistent.mapping import PersistentMapping
import transaction
import ZODB
from ZODB.POSException import ConflictError
from ZODB.tests.MinPO import MinPO
from ZEO.ClientStorage import ClientStorage
ZEO_VERSION = 2
def setup_logging():
# Set up logging to stderr which will show messages originating
# at severity ERROR or higher.
root = logging.getLogger()
root.setLevel(logging.ERROR)
fmt = logging.Formatter(
"------\n%(asctime)s %(levelname)s %(name)s %(message)s",
"%Y-%m-%dT%H:%M:%S")
handler = logging.StreamHandler()
handler.setFormatter(fmt)
root.addHandler(handler)
def check_server(addr, storage, write):
t0 = time.time()
if ZEO_VERSION == 2:
# TODO: should do retries w/ exponential backoff.
cs = ClientStorage(addr, storage=storage, wait=0,
read_only=(not write))
else:
cs = ClientStorage(addr, storage=storage, debug=1,
wait_for_server_on_startup=1)
# _startup() is an artifact of the way ZEO 1.0 works. The
# ClientStorage doesn't get fully initialized until registerDB()
# is called. The only thing we care about, though, is that
# registerDB() calls _startup().
if write:
db = ZODB.DB(cs)
cn = db.open()
root = cn.root()
try:
# We store the data in a special `monitor' dict under the root,
# where other tools may also store such heartbeat and bookkeeping
# type data.
monitor = root.get('monitor')
if monitor is None:
monitor = root['monitor'] = PersistentMapping()
obj = monitor['zeoup'] = monitor.get('zeoup', MinPO(0))
obj.value += 1
transaction.commit()
except ConflictError:
pass
cn.close()
db.close()
else:
data, serial = cs.load("\0\0\0\0\0\0\0\0", "")
cs.close()
t1 = time.time()
print "Elapsed time: %.2f" % (t1 - t0)
def usage(exit=1):
print __doc__
print " ".join(sys.argv)
sys.exit(exit)
def main():
host = None
port = None
unix = None
write = 1
storage = '1'
try:
opts, args = getopt.getopt(sys.argv[1:], 'p:h:U:S:1',
['nowrite'])
for o, a in opts:
if o == '-p':
port = int(a)
elif o == '-h':
host = a
elif o == '-U':
unix = a
elif o == '-S':
storage = a
elif o == '--nowrite':
write = 0
elif o == '-1':
ZEO_VERSION = 1
except Exception, err:
s = str(err)
if s:
s = ": " + s
print err.__class__.__name__ + s
usage()
if unix is not None:
addr = unix
else:
if host is None:
host = socket.gethostname()
if port is None:
usage()
addr = host, port
setup_logging()
check_server(addr, storage, write)
if __name__ == "__main__":
try:
main()
except SystemExit:
raise
except Exception, err:
s = str(err)
if s:
s = ": " + s
print err.__class__.__name__ + s
sys.exit(1)
ZODB/src/scripts/analyze.py 0000750 0001750 0000144 00000010156 10110730025 015447 0 ustar ctheune users #!/usr/bin/env python2.3
# Based on a transaction analyzer by Matt Kromer.
import pickle
import re
import sys
import types
from ZODB.FileStorage import FileStorage
class Report:
def __init__(self):
self.OIDMAP = {}
self.TYPEMAP = {}
self.TYPESIZE = {}
self.FREEMAP = {}
self.USEDMAP = {}
self.TIDS = 0
self.OIDS = 0
self.DBYTES = 0
self.COIDS = 0
self.CBYTES = 0
self.FOIDS = 0
self.FBYTES = 0
def shorten(s, n):
l = len(s)
if l <= n:
return s
while len(s) + 3 > n: # account for ...
i = s.find(".")
if i == -1:
# In the worst case, just return the rightmost n bytes
return s[-n:]
else:
s = s[i + 1:]
l = len(s)
return "..." + s
def report(rep):
print "Processed %d records in %d transactions" % (rep.OIDS, rep.TIDS)
print "Average record size is %7.2f bytes" % (rep.DBYTES * 1.0 / rep.OIDS)
print ("Average transaction size is %7.2f bytes" %
(rep.DBYTES * 1.0 / rep.TIDS))
print "Types used:"
fmt = "%-46s %7s %9s %6s %7s"
fmtp = "%-46s %7d %9d %5.1f%% %7.2f" # per-class format
fmts = "%46s %7d %8dk %5.1f%% %7.2f" # summary format
print fmt % ("Class Name", "Count", "TBytes", "Pct", "AvgSize")
print fmt % ('-'*46, '-'*7, '-'*9, '-'*5, '-'*7)
typemap = rep.TYPEMAP.keys()
typemap.sort()
cumpct = 0.0
for t in typemap:
pct = rep.TYPESIZE[t] * 100.0 / rep.DBYTES
cumpct += pct
print fmtp % (shorten(t, 46), rep.TYPEMAP[t], rep.TYPESIZE[t],
pct, rep.TYPESIZE[t] * 1.0 / rep.TYPEMAP[t])
print fmt % ('='*46, '='*7, '='*9, '='*5, '='*7)
print "%46s %7d %9s %6s %6.2fk" % ('Total Transactions', rep.TIDS, ' ',
' ', rep.DBYTES * 1.0 / rep.TIDS / 1024.0)
print fmts % ('Total Records', rep.OIDS, rep.DBYTES / 1024.0, cumpct,
rep.DBYTES * 1.0 / rep.OIDS)
print fmts % ('Current Objects', rep.COIDS, rep.CBYTES / 1024.0,
rep.CBYTES * 100.0 / rep.DBYTES,
rep.CBYTES * 1.0 / rep.COIDS)
if rep.FOIDS:
print fmts % ('Old Objects', rep.FOIDS, rep.FBYTES / 1024.0,
rep.FBYTES * 100.0 / rep.DBYTES,
rep.FBYTES * 1.0 / rep.FOIDS)
def analyze(path):
fs = FileStorage(path, read_only=1)
fsi = fs.iterator()
report = Report()
for txn in fsi:
analyze_trans(report, txn)
return report
def analyze_trans(report, txn):
report.TIDS += 1
for rec in txn:
analyze_rec(report, rec)
def get_type(record):
try:
classinfo = pickle.loads(record.data)[0]
except SystemError, err:
s = str(err)
mo = re.match('Failed to import class (\S+) from module (\S+)', s)
if mo is None:
raise
else:
klass, mod = mo.group(1, 2)
return "%s.%s" % (mod, klass)
if isinstance(classinfo, types.TupleType):
mod, klass = classinfo
return "%s.%s" % (mod, klass)
else:
return str(classinfo)
def analyze_rec(report, record):
oid = record.oid
report.OIDS += 1
if record.data is None:
# No pickle -- aborted version or undo of object creation.
return
try:
size = len(record.data) # Ignores various overhead
report.DBYTES += size
if oid not in report.OIDMAP:
type = get_type(record)
report.OIDMAP[oid] = type
report.USEDMAP[oid] = size
report.COIDS += 1
report.CBYTES += size
else:
type = report.OIDMAP[oid]
fsize = report.USEDMAP[oid]
report.FREEMAP[oid] = report.FREEMAP.get(oid, 0) + fsize
report.USEDMAP[oid] = size
report.FOIDS += 1
report.FBYTES += fsize
report.CBYTES += size - fsize
report.TYPEMAP[type] = report.TYPEMAP.get(type, 0) + 1
report.TYPESIZE[type] = report.TYPESIZE.get(type, 0) + size
except Exception, err:
print err
if __name__ == "__main__":
path = sys.argv[1]
report(analyze(path))
ZODB/src/scripts/SETUP.cfg 0000640 0001750 0000144 00000000014 10277166665 015034 0 ustar ctheune users script *.py
ZODB/src/scripts/zeoreplay.py 0000640 0001750 0000144 00000020457 10051215041 016021 0 ustar ctheune users #!/usr/bin/env python2.3
"""Parse the BLATHER logging generated by ZEO, and optionally replay it.
Usage: zeointervals.py [options]
Options:
--help / -h
Print this message and exit.
--replay=storage
-r storage
Replay the parsed transactions through the new storage
--maxtxn=count
-m count
Parse no more than count transactions.
--report / -p
Print a report as we're parsing.
Unlike parsezeolog.py, this script generates timestamps for each transaction,
and sub-command in the transaction. We can use this to compare timings with
synthesized data.
"""
import re
import sys
import time
import getopt
import operator
# ZEO logs measure wall-clock time so for consistency we need to do the same
#from time import clock as now
from time import time as now
from ZODB.FileStorage import FileStorage
#from BDBStorage.BDBFullStorage import BDBFullStorage
#from Standby.primary import PrimaryStorage
#from Standby.config import RS_PORT
from ZODB.Transaction import Transaction
from ZODB.utils import p64
datecre = re.compile('(\d\d\d\d-\d\d-\d\d)T(\d\d:\d\d:\d\d)')
methcre = re.compile("ZEO Server (\w+)\((.*)\) \('(.*)', (\d+)")
class StopParsing(Exception):
pass
def usage(code, msg=''):
print __doc__
if msg:
print msg
sys.exit(code)
def parse_time(line):
"""Return the time portion of a zLOG line in seconds or None."""
mo = datecre.match(line)
if mo is None:
return None
date, time_ = mo.group(1, 2)
date_l = [int(elt) for elt in date.split('-')]
time_l = [int(elt) for elt in time_.split(':')]
return int(time.mktime(date_l + time_l + [0, 0, 0]))
def parse_line(line):
"""Parse a log entry and return time, method info, and client."""
t = parse_time(line)
if t is None:
return None, None, None
mo = methcre.search(line)
if mo is None:
return None, None, None
meth_name = mo.group(1)
meth_args = mo.group(2)
meth_args = [s.strip() for s in meth_args.split(',')]
m = meth_name, tuple(meth_args)
c = mo.group(3), mo.group(4)
return t, m, c
class StoreStat:
def __init__(self, when, oid, size):
self.when = when
self.oid = oid
self.size = size
# Crufty
def __getitem__(self, i):
if i == 0: return self.oid
if i == 1: return self.size
raise IndexError
class TxnStat:
def __init__(self):
self._begintime = None
self._finishtime = None
self._aborttime = None
self._url = None
self._objects = []
def tpc_begin(self, when, args, client):
self._begintime = when
# args are txnid, user, description (looks like it's always a url)
self._url = args[2]
def storea(self, when, args, client):
oid = int(args[0])
# args[1] is "[numbytes]"
size = int(args[1][1:-1])
s = StoreStat(when, oid, size)
self._objects.append(s)
def tpc_abort(self, when):
self._aborttime = when
def tpc_finish(self, when):
self._finishtime = when
# Mapping oid -> revid
_revids = {}
class ReplayTxn(TxnStat):
def __init__(self, storage):
self._storage = storage
self._replaydelta = 0
TxnStat.__init__(self)
def replay(self):
ZERO = '\0'*8
t0 = now()
t = Transaction()
self._storage.tpc_begin(t)
for obj in self._objects:
oid = obj.oid
revid = _revids.get(oid, ZERO)
# BAW: simulate a pickle of the given size
data = 'x' * obj.size
# BAW: ignore versions for now
newrevid = self._storage.store(p64(oid), revid, data, '', t)
_revids[oid] = newrevid
if self._aborttime:
self._storage.tpc_abort(t)
origdelta = self._aborttime - self._begintime
else:
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
origdelta = self._finishtime - self._begintime
t1 = now()
# Shows how many seconds behind (positive) or ahead (negative) of the
# original reply our local update took
self._replaydelta = t1 - t0 - origdelta
class ZEOParser:
def __init__(self, maxtxns=-1, report=1, storage=None):
self.__txns = []
self.__curtxn = {}
self.__skipped = 0
self.__maxtxns = maxtxns
self.__finishedtxns = 0
self.__report = report
self.__storage = storage
def parse(self, line):
t, m, c = parse_line(line)
if t is None:
# Skip this line
return
name = m[0]
meth = getattr(self, name, None)
if meth is not None:
meth(t, m[1], c)
def tpc_begin(self, when, args, client):
txn = ReplayTxn(self.__storage)
self.__curtxn[client] = txn
meth = getattr(txn, 'tpc_begin', None)
if meth is not None:
meth(when, args, client)
def storea(self, when, args, client):
txn = self.__curtxn.get(client)
if txn is None:
self.__skipped += 1
return
meth = getattr(txn, 'storea', None)
if meth is not None:
meth(when, args, client)
def tpc_finish(self, when, args, client):
txn = self.__curtxn.get(client)
if txn is None:
self.__skipped += 1
return
meth = getattr(txn, 'tpc_finish', None)
if meth is not None:
meth(when)
if self.__report:
self.report(txn)
self.__txns.append(txn)
self.__curtxn[client] = None
self.__finishedtxns += 1
if self.__maxtxns > 0 and self.__finishedtxns >= self.__maxtxns:
raise StopParsing
def report(self, txn):
"""Print a report about the transaction"""
if txn._objects:
bytes = reduce(operator.add, [size for oid, size in txn._objects])
else:
bytes = 0
print '%s %s %4d %10d %s %s' % (
txn._begintime, txn._finishtime - txn._begintime,
len(txn._objects),
bytes,
time.ctime(txn._begintime),
txn._url)
def replay(self):
for txn in self.__txns:
txn.replay()
# How many fell behind?
slower = []
faster = []
for txn in self.__txns:
if txn._replaydelta > 0:
slower.append(txn)
else:
faster.append(txn)
print len(slower), 'laggards,', len(faster), 'on-time or faster'
# Find some averages
if slower:
sum = reduce(operator.add,
[txn._replaydelta for txn in slower], 0)
print 'average slower txn was:', float(sum) / len(slower)
if faster:
sum = reduce(operator.add,
[txn._replaydelta for txn in faster], 0)
print 'average faster txn was:', float(sum) / len(faster)
def main():
try:
opts, args = getopt.getopt(
sys.argv[1:],
'hr:pm:',
['help', 'replay=', 'report', 'maxtxns='])
except getopt.error, e:
usage(1, e)
if args:
usage(1)
replay = 0
maxtxns = -1
report = 0
storagefile = None
for opt, arg in opts:
if opt in ('-h', '--help'):
usage(0)
elif opt in ('-r', '--replay'):
replay = 1
storagefile = arg
elif opt in ('-p', '--report'):
report = 1
elif opt in ('-m', '--maxtxns'):
try:
maxtxns = int(arg)
except ValueError:
usage(1, 'Bad -m argument: %s' % arg)
if replay:
storage = FileStorage(storagefile)
#storage = BDBFullStorage(storagefile)
#storage = PrimaryStorage('yyz', storage, RS_PORT)
t0 = now()
p = ZEOParser(maxtxns, report, storage)
i = 0
while 1:
line = sys.stdin.readline()
if not line:
break
i += 1
try:
p.parse(line)
except StopParsing:
break
except:
print 'input file line:', i
raise
t1 = now()
print 'total parse time:', t1-t0
t2 = now()
if replay:
p.replay()
t3 = now()
print 'total replay time:', t3-t2
print 'total time:', t3-t0
if __name__ == '__main__':
main()
ZODB/src/scripts/fstest.py 0000640 0001750 0000144 00000015026 10305413212 015315 0 ustar ctheune users #!/usr/bin/env python2.3
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Simple consistency checker for FileStorage.
usage: fstest.py [-v] data.fs
The fstest tool will scan all the data in a FileStorage and report an
error if it finds any corrupt transaction data. The tool will print a
message when the first error is detected, then exit.
The tool accepts one or more -v arguments. If a single -v is used, it
will print a line of text for each transaction record it encounters.
If two -v arguments are used, it will also print a line of text for
each object. The objects for a transaction will be printed before the
transaction itself.
Note: It does not check the consistency of the object pickles. It is
possible for the damage to occur only in the part of the file that
stores object pickles. Those errors will go undetected.
"""
# The implementation is based closely on the read_index() function in
# ZODB.FileStorage. If anything about the FileStorage layout changes,
# this file will need to be udpated.
import string
import struct
import sys
class FormatError(ValueError):
"""There is a problem with the format of the FileStorage."""
class Status:
checkpoint = 'c'
undone = 'u'
packed_version = 'FS21'
TREC_HDR_LEN = 23
DREC_HDR_LEN = 42
VERBOSE = 0
def hexify(s):
"""Format an 8-bite string as hex"""
l = []
for c in s:
h = hex(ord(c))
if h[:2] == '0x':
h = h[2:]
if len(h) == 1:
l.append("0")
l.append(h)
return "0x" + string.join(l, '')
def chatter(msg, level=1):
if VERBOSE >= level:
sys.stdout.write(msg)
def U64(v):
"""Unpack an 8-byte string as a 64-bit long"""
h, l = struct.unpack(">II", v)
if h:
return (h << 32) + l
else:
return l
def check(path):
file = open(path, 'rb')
file.seek(0, 2)
file_size = file.tell()
if file_size == 0:
raise FormatError("empty file")
file.seek(0)
if file.read(4) != packed_version:
raise FormatError("invalid file header")
pos = 4L
tid = '\000' * 8 # lowest possible tid to start
i = 0
while pos:
_pos = pos
pos, tid = check_trec(path, file, pos, tid, file_size)
if tid is not None:
chatter("%10d: transaction tid %s #%d \n" %
(_pos, hexify(tid), i))
i = i + 1
def check_trec(path, file, pos, ltid, file_size):
"""Read an individual transaction record from file.
Returns the pos of the next transaction and the transaction id.
It also leaves the file pointer set to pos. The path argument is
used for generating error messages.
"""
h = file.read(TREC_HDR_LEN)
if not h:
return None, None
if len(h) != TREC_HDR_LEN:
raise FormatError("%s truncated at %s" % (path, pos))
tid, stl, status, ul, dl, el = struct.unpack(">8s8scHHH", h)
tmeta_len = TREC_HDR_LEN + ul + dl + el
if tid <= ltid:
raise FormatError("%s time-stamp reduction at %s: %s <= %s" %
(path, pos, hexify(tid), hexify(ltid)))
ltid = tid
tl = U64(stl) # transaction record length - 8
if pos + tl + 8 > file_size:
raise FormatError("%s truncated possibly because of"
" damaged records at %s" % (path, pos))
if status == Status.checkpoint:
raise FormatError("%s checkpoint flag was not cleared at %s"
% (path, pos))
if status not in ' up':
raise FormatError("%s has invalid status '%s' at %s" %
(path, status, pos))
if tmeta_len > tl:
raise FormatError("%s has an invalid transaction header"
" at %s" % (path, pos))
tpos = pos
tend = tpos + tl
if status != Status.undone:
pos = tpos + tmeta_len
file.read(ul + dl + el) # skip transaction metadata
i = 0
while pos < tend:
_pos = pos
pos, oid = check_drec(path, file, pos, tpos, tid)
if pos > tend:
raise FormatError("%s has data records that extend beyond"
" the transaction record; end at %s" %
(path, pos))
chatter("%10d: object oid %s #%d\n" % (_pos, hexify(oid), i),
level=2)
i = i + 1
file.seek(tend)
rtl = file.read(8)
if rtl != stl:
raise FormatError("%s has inconsistent transaction length"
" for undone transaction at %s" % (path, pos))
pos = tend + 8
return pos, tid
def check_drec(path, file, pos, tpos, tid):
"""Check a data record for the current transaction record"""
h = file.read(DREC_HDR_LEN)
if len(h) != DREC_HDR_LEN:
raise FormatError("%s truncated at %s" % (path, pos))
oid, serial, _prev, _tloc, vlen, _plen = (
struct.unpack(">8s8s8s8sH8s", h))
prev = U64(_prev)
tloc = U64(_tloc)
plen = U64(_plen)
dlen = DREC_HDR_LEN + (plen or 8)
if vlen:
dlen = dlen + 16 + vlen
file.seek(8, 1)
pv = U64(file.read(8))
file.seek(vlen, 1) # skip the version data
if tloc != tpos:
raise FormatError("%s data record exceeds transaction record "
"at %s: tloc %d != tpos %d" %
(path, pos, tloc, tpos))
pos = pos + dlen
if plen:
file.seek(plen, 1)
else:
file.seek(8, 1)
# _loadBack() ?
return pos, oid
def usage():
print __doc__
sys.exit(-1)
if __name__ == "__main__":
import getopt
try:
opts, args = getopt.getopt(sys.argv[1:], 'v')
if len(args) != 1:
raise ValueError("expected one argument")
for k, v in opts:
if k == '-v':
VERBOSE = VERBOSE + 1
except (getopt.error, ValueError):
usage()
try:
check(args[0])
except FormatError, msg:
print msg
sys.exit(-1)
chatter("no errors detected")
ZODB/src/scripts/zeopasswd.py 0000750 0001750 0000144 00000001306 10276710420 016032 0 ustar ctheune users #!python
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import sys
from ZEO.zeopasswd import main
main(sys.argv[1:])
ZODB/src/scripts/repozo.py 0000750 0001750 0000144 00000041211 10214421637 015331 0 ustar ctheune users #!/usr/bin/env python2.3
# repozo.py -- incremental and full backups of a Data.fs file.
#
# Originally written by Anthony Baxter
# Significantly modified by Barry Warsaw
"""repozo.py -- incremental and full backups of a Data.fs file.
Usage: %(program)s [options]
Where:
Exactly one of -B or -R must be specified:
-B / --backup
Backup current ZODB file.
-R / --recover
Restore a ZODB file from a backup.
-v / --verbose
Verbose mode.
-h / --help
Print this text and exit.
-r dir
--repository=dir
Repository directory containing the backup files. This argument
is required. The directory must already exist. You should not
edit the files in this directory, or add your own files to it.
Options for -B/--backup:
-f file
--file=file
Source Data.fs file. This argument is required.
-F / --full
Force a full backup. By default, an incremental backup is made
if possible (e.g., if a pack has occurred since the last
incremental backup, a full backup is necessary).
-Q / --quick
Verify via md5 checksum only the last incremental written. This
significantly reduces the disk i/o at the (theoretical) cost of
inconsistency. This is a probabilistic way of determining whether
a full backup is necessary.
-z / --gzip
Compress with gzip the backup files. Uses the default zlib
compression level. By default, gzip compression is not used.
Options for -R/--recover:
-D str
--date=str
Recover state as of this date. Specify UTC (not local) time.
yyyy-mm-dd[-hh[-mm[-ss]]]
By default, current time is used.
-o filename
--output=filename
Write recovered ZODB to given file. By default, the file is
written to stdout.
"""
import os
import sys
import md5
import gzip
import time
import errno
import getopt
from ZODB.FileStorage import FileStorage
program = sys.argv[0]
BACKUP = 1
RECOVER = 2
COMMASPACE = ', '
READCHUNK = 16 * 1024
VERBOSE = False
def usage(code, msg=''):
outfp = sys.stderr
if code == 0:
outfp = sys.stdout
print >> outfp, __doc__ % globals()
if msg:
print >> outfp, msg
sys.exit(code)
def log(msg, *args):
if VERBOSE:
# Use stderr here so that -v flag works with -R and no -o
print >> sys.stderr, msg % args
def parseargs():
global VERBOSE
try:
opts, args = getopt.getopt(sys.argv[1:], 'BRvhf:r:FD:o:Qz',
['backup', 'recover', 'verbose', 'help',
'file=', 'repository=', 'full', 'date=',
'output=', 'quick', 'gzip'])
except getopt.error, msg:
usage(1, msg)
class Options:
mode = None # BACKUP or RECOVER
file = None # name of input Data.fs file
repository = None # name of directory holding backups
full = False # True forces full backup
date = None # -D argument, if any
output = None # where to write recovered data; None = stdout
quick = False # -Q flag state
gzip = False # -z flag state
options = Options()
for opt, arg in opts:
if opt in ('-h', '--help'):
usage(0)
elif opt in ('-v', '--verbose'):
VERBOSE = True
elif opt in ('-R', '--recover'):
if options.mode is not None:
usage(1, '-B and -R are mutually exclusive')
options.mode = RECOVER
elif opt in ('-B', '--backup'):
if options.mode is not None:
usage(1, '-B and -R are mutually exclusive')
options.mode = BACKUP
elif opt in ('-Q', '--quick'):
options.quick = True
elif opt in ('-f', '--file'):
options.file = arg
elif opt in ('-r', '--repository'):
options.repository = arg
elif opt in ('-F', '--full'):
options.full = True
elif opt in ('-D', '--date'):
options.date = arg
elif opt in ('-o', '--output'):
options.output = arg
elif opt in ('-z', '--gzip'):
options.gzip = True
else:
assert False, (opt, arg)
# Any other arguments are invalid
if args:
usage(1, 'Invalid arguments: ' + COMMASPACE.join(args))
# Sanity checks
if options.mode is None:
usage(1, 'Either --backup or --recover is required')
if options.repository is None:
usage(1, '--repository is required')
if options.mode == BACKUP:
if options.date is not None:
log('--date option is ignored in backup mode')
options.date = None
if options.output is not None:
log('--output option is ignored in backup mode')
options.output = None
else:
assert options.mode == RECOVER
if options.file is not None:
log('--file option is ignored in recover mode')
options.file = None
return options
# afile is a Python file object, or created by gzip.open(). The latter
# doesn't have a fileno() method, so to fsync it we need to reach into
# its underlying file object.
def fsync(afile):
afile.flush()
fileobject = getattr(afile, 'fileobj', afile)
os.fsync(fileobject.fileno())
# Read bytes (no more than n, or to EOF if n is None) in chunks from the
# current position in file fp. Pass each chunk as an argument to func().
# Return the total number of bytes read == the total number of bytes
# passed in all to func(). Leaves the file position just after the
# last byte read.
def dofile(func, fp, n=None):
bytesread = 0L
while n is None or n > 0:
if n is None:
todo = READCHUNK
else:
todo = min(READCHUNK, n)
data = fp.read(todo)
if not data:
break
func(data)
nread = len(data)
bytesread += nread
if n is not None:
n -= nread
return bytesread
def checksum(fp, n):
# Checksum the first n bytes of the specified file
sum = md5.new()
def func(data):
sum.update(data)
dofile(func, fp, n)
return sum.hexdigest()
def copyfile(options, dst, start, n):
# Copy bytes from file src, to file dst, starting at offset start, for n
# length of bytes. For robustness, we first write, flush and fsync
# to a temp file, then rename the temp file at the end.
sum = md5.new()
ifp = open(options.file, 'rb')
ifp.seek(start)
tempname = os.path.join(os.path.dirname(dst), 'tmp.tmp')
if options.gzip:
ofp = gzip.open(tempname, 'wb')
else:
ofp = open(tempname, 'wb')
def func(data):
sum.update(data)
ofp.write(data)
ndone = dofile(func, ifp, n)
assert ndone == n
ifp.close()
fsync(ofp)
ofp.close()
os.rename(tempname, dst)
return sum.hexdigest()
def concat(files, ofp=None):
# Concatenate a bunch of files from the repository, output to `outfile' if
# given. Return the number of bytes written and the md5 checksum of the
# bytes.
sum = md5.new()
def func(data):
sum.update(data)
if ofp:
ofp.write(data)
bytesread = 0
for f in files:
# Auto uncompress
if f.endswith('fsz'):
ifp = gzip.open(f, 'rb')
else:
ifp = open(f, 'rb')
bytesread += dofile(func, ifp)
ifp.close()
if ofp:
ofp.close()
return bytesread, sum.hexdigest()
def gen_filename(options, ext=None):
if ext is None:
if options.full:
ext = '.fs'
else:
ext = '.deltafs'
if options.gzip:
ext += 'z'
t = time.gmtime()[:6] + (ext,)
return '%04d-%02d-%02d-%02d-%02d-%02d%s' % t
# Return a list of files needed to reproduce state at time options.date.
# This is a list, in chronological order, of the .fs[z] and .deltafs[z]
# files, from the time of the most recent full backup preceding
# options.date, up to options.date.
import re
is_data_file = re.compile(r'\d{4}(?:-\d\d){5}\.(?:delta)?fsz?$').match
del re
def find_files(options):
when = options.date
if not when:
when = gen_filename(options, '')
log('looking for files between last full backup and %s...', when)
all = filter(is_data_file, os.listdir(options.repository))
all.sort()
all.reverse() # newest file first
# Find the last full backup before date, then include all the
# incrementals between that full backup and "when".
needed = []
for fname in all:
root, ext = os.path.splitext(fname)
if root <= when:
needed.append(fname)
if ext in ('.fs', '.fsz'):
break
# Make the file names relative to the repository directory
needed = [os.path.join(options.repository, f) for f in needed]
# Restore back to chronological order
needed.reverse()
if needed:
log('files needed to recover state as of %s:', when)
for f in needed:
log('\t%s', f)
else:
log('no files found')
return needed
# Scan the .dat file corresponding to the last full backup performed.
# Return
#
# filename, startpos, endpos, checksum
#
# of the last incremental. If there is no .dat file, or the .dat file
# is empty, return
#
# None, None, None, None
def scandat(repofiles):
fullfile = repofiles[0]
datfile = os.path.splitext(fullfile)[0] + '.dat'
fn = startpos = endpos = sum = None # assume .dat file missing or empty
try:
fp = open(datfile)
except IOError, e:
if e.errno <> errno.ENOENT:
raise
else:
# We only care about the last one.
lines = fp.readlines()
fp.close()
if lines:
fn, startpos, endpos, sum = lines[-1].split()
startpos = long(startpos)
endpos = long(endpos)
return fn, startpos, endpos, sum
def do_full_backup(options):
# Find the file position of the last completed transaction.
fs = FileStorage(options.file, read_only=True)
# Note that the FileStorage ctor calls read_index() which scans the file
# and returns "the position just after the last valid transaction record".
# getSize() then returns this position, which is exactly what we want,
# because we only want to copy stuff from the beginning of the file to the
# last valid transaction record.
pos = fs.getSize()
fs.close()
options.full = True
dest = os.path.join(options.repository, gen_filename(options))
if os.path.exists(dest):
print >> sys.stderr, 'Cannot overwrite existing file:', dest
sys.exit(2)
log('writing full backup: %s bytes to %s', pos, dest)
sum = copyfile(options, dest, 0, pos)
# Write the data file for this full backup
datfile = os.path.splitext(dest)[0] + '.dat'
fp = open(datfile, 'w')
print >> fp, dest, 0, pos, sum
fp.flush()
os.fsync(fp.fileno())
fp.close()
def do_incremental_backup(options, reposz, repofiles):
# Find the file position of the last completed transaction.
fs = FileStorage(options.file, read_only=True)
# Note that the FileStorage ctor calls read_index() which scans the file
# and returns "the position just after the last valid transaction record".
# getSize() then returns this position, which is exactly what we want,
# because we only want to copy stuff from the beginning of the file to the
# last valid transaction record.
pos = fs.getSize()
fs.close()
options.full = False
dest = os.path.join(options.repository, gen_filename(options))
if os.path.exists(dest):
print >> sys.stderr, 'Cannot overwrite existing file:', dest
sys.exit(2)
log('writing incremental: %s bytes to %s', pos-reposz, dest)
sum = copyfile(options, dest, reposz, pos - reposz)
# The first file in repofiles points to the last full backup. Use this to
# get the .dat file and append the information for this incrementatl to
# that file.
fullfile = repofiles[0]
datfile = os.path.splitext(fullfile)[0] + '.dat'
# This .dat file better exist. Let the exception percolate if not.
fp = open(datfile, 'a')
print >> fp, dest, reposz, pos, sum
fp.flush()
os.fsync(fp.fileno())
fp.close()
def do_backup(options):
repofiles = find_files(options)
# See if we need to do a full backup
if options.full or not repofiles:
log('doing a full backup')
do_full_backup(options)
return
srcsz = os.path.getsize(options.file)
if options.quick:
fn, startpos, endpos, sum = scandat(repofiles)
# If the .dat file was missing, or was empty, do a full backup
if (fn, startpos, endpos, sum) == (None, None, None, None):
log('missing or empty .dat file (full backup)')
do_full_backup(options)
return
# Has the file shrunk, possibly because of a pack?
if srcsz < endpos:
log('file shrunk, possibly because of a pack (full backup)')
do_full_backup(options)
return
# Now check the md5 sum of the source file, from the last
# incremental's start and stop positions.
srcfp = open(options.file, 'rb')
srcfp.seek(startpos)
srcsum = checksum(srcfp, endpos-startpos)
srcfp.close()
log('last incremental file: %s', fn)
log('last incremental checksum: %s', sum)
log('source checksum range: [%s..%s], sum: %s',
startpos, endpos, srcsum)
if sum == srcsum:
if srcsz == endpos:
log('No changes, nothing to do')
return
log('doing incremental, starting at: %s', endpos)
do_incremental_backup(options, endpos, repofiles)
return
else:
# This was is much slower, and more disk i/o intensive, but it's also
# more accurate since it checks the actual existing files instead of
# the information in the .dat file.
#
# See if we can do an incremental, based on the files that already
# exist. This call of concat() will not write an output file.
reposz, reposum = concat(repofiles)
log('repository state: %s bytes, md5: %s', reposz, reposum)
# Get the md5 checksum of the source file, up to two file positions:
# the entire size of the file, and up to the file position of the last
# incremental backup.
srcfp = open(options.file, 'rb')
srcsum = checksum(srcfp, srcsz)
srcfp.seek(0)
srcsum_backedup = checksum(srcfp, reposz)
srcfp.close()
log('current state : %s bytes, md5: %s', srcsz, srcsum)
log('backed up state : %s bytes, md5: %s', reposz, srcsum_backedup)
# Has nothing changed?
if srcsz == reposz and srcsum == reposum:
log('No changes, nothing to do')
return
# Has the file shrunk, probably because of a pack?
if srcsz < reposz:
log('file shrunk, possibly because of a pack (full backup)')
do_full_backup(options)
return
# The source file is larger than the repository. If the md5 checksums
# match, then we know we can do an incremental backup. If they don't,
# then perhaps the file was packed at some point (or a
# non-transactional undo was performed, but this is deprecated). Only
# do a full backup if forced to.
if reposum == srcsum_backedup:
log('doing incremental, starting at: %s', reposz)
do_incremental_backup(options, reposz, repofiles)
return
# The checksums don't match, meaning the front of the source file has
# changed. We'll need to do a full backup in that case.
log('file changed, possibly because of a pack (full backup)')
do_full_backup(options)
def do_recover(options):
# Find the first full backup at or before the specified date
repofiles = find_files(options)
if not repofiles:
if options.date:
log('No files in repository before %s', options.date)
else:
log('No files in repository')
return
if options.output is None:
log('Recovering file to stdout')
outfp = sys.stdout
else:
log('Recovering file to %s', options.output)
outfp = open(options.output, 'wb')
reposz, reposum = concat(repofiles, outfp)
if outfp <> sys.stdout:
outfp.close()
log('Recovered %s bytes, md5: %s', reposz, reposum)
def main():
options = parseargs()
if options.mode == BACKUP:
do_backup(options)
else:
assert options.mode == RECOVER
do_recover(options)
if __name__ == '__main__':
main()
ZODB/src/scripts/checkbtrees.py 0000750 0001750 0000144 00000006013 10051215041 016263 0 ustar ctheune users #!/usr/bin/env python2.3
"""Check the consistency of BTrees in a Data.fs
usage: checkbtrees.py data.fs
Try to find all the BTrees in a Data.fs, call their _check() methods,
and run them through BTrees.check.check().
"""
from types import IntType
import ZODB
from ZODB.FileStorage import FileStorage
from BTrees.check import check
# Set of oids we've already visited. Since the object structure is
# a general graph, this is needed to prevent unbounded paths in the
# presence of cycles. It's also helpful in eliminating redundant
# checking when a BTree is pointed to by many objects.
oids_seen = {}
# Append (obj, path) to L if and only if obj is a persistent object
# and we haven't seen it before.
def add_if_new_persistent(L, obj, path):
global oids_seen
getattr(obj, '_', None) # unghostify
if hasattr(obj, '_p_oid'):
oid = obj._p_oid
if not oids_seen.has_key(oid):
L.append((obj, path))
oids_seen[oid] = 1
def get_subobjects(obj):
getattr(obj, '_', None) # unghostify
sub = []
try:
attrs = obj.__dict__.items()
except AttributeError:
attrs = ()
for pair in attrs:
sub.append(pair)
# what if it is a mapping?
try:
items = obj.items()
except AttributeError:
items = ()
for k, v in items:
if not isinstance(k, IntType):
sub.append(("", k))
if not isinstance(v, IntType):
sub.append(("[%s]" % repr(k), v))
# what if it is a sequence?
i = 0
while 1:
try:
elt = obj[i]
except:
break
sub.append(("[%d]" % i, elt))
i += 1
return sub
def main(fname):
fs = FileStorage(fname, read_only=1)
cn = ZODB.DB(fs).open()
rt = cn.root()
todo = []
add_if_new_persistent(todo, rt, '')
found = 0
while todo:
obj, path = todo.pop(0)
found += 1
if not path:
print "", repr(obj)
else:
print path, repr(obj)
mod = str(obj.__class__.__module__)
if mod.startswith("BTrees"):
if hasattr(obj, "_check"):
try:
obj._check()
except AssertionError, msg:
print "*" * 60
print msg
print "*" * 60
try:
check(obj)
except AssertionError, msg:
print "*" * 60
print msg
print "*" * 60
if found % 100 == 0:
cn.cacheMinimize()
for k, v in get_subobjects(obj):
if k.startswith('['):
# getitem
newpath = "%s%s" % (path, k)
else:
newpath = "%s.%s" % (path, k)
add_if_new_persistent(todo, v, newpath)
print "total", len(fs._index), "found", found
if __name__ == "__main__":
import sys
try:
fname, = sys.argv[1:]
except:
print __doc__
sys.exit(2)
main(fname)
ZODB/src/scripts/fstail.py 0000640 0001750 0000144 00000002765 10057366465 015322 0 ustar ctheune users #!/usr/bin/env python2.3
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Tool to dump the last few transactions from a FileStorage."""
from ZODB.fstools import prev_txn
import binascii
import getopt
import sha
import sys
def main(path, ntxn):
f = open(path, "rb")
f.seek(0, 2)
th = prev_txn(f)
i = ntxn
while th and i > 0:
hash = sha.sha(th.get_raw_data()).digest()
l = len(str(th.get_timestamp())) + 1
th.read_meta()
print "%s: hash=%s" % (th.get_timestamp(),
binascii.hexlify(hash))
print ("user=%r description=%r length=%d"
% (th.user, th.descr, th.length))
print
th = th.prev_txn()
i -= 1
if __name__ == "__main__":
ntxn = 10
opts, args = getopt.getopt(sys.argv[1:], "n:")
path, = args
for k, v in opts:
if k == '-n':
ntxn = int(v)
main(path, ntxn)
ZODB/src/scripts/mkzeoinst.py 0000750 0001750 0000144 00000001333 10307343426 016041 0 ustar ctheune users #!python
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""ZEO instance home creation script."""
import ZEO.mkzeoinst
ZEO.mkzeoinst.main()
ZODB/src/scripts/runzeo.py 0000750 0001750 0000144 00000001253 10276710420 015336 0 ustar ctheune users #!python
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
from ZEO.runzeo import main
main()
ZODB/src/scripts/zeoserverlog.py 0000640 0001750 0000144 00000034730 10057366465 016563 0 ustar ctheune users #!/usr/bin/env python2.3
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Tools for analyzing ZEO Server logs.
This script contains a number of commands, implemented by command
functions. To run a command, give the command name and it's arguments
as arguments to this script.
Commands:
blocked_times file threshold
Output a summary of episodes where thransactions were blocked
when the episode lasted at least threshold seconds.
The file may be a file name or - to read from standard input.
The file may also be a command:
script blocked_times 'bunzip2 = 0
if blocking and waiting == 1:
t1 = time(line)
t2 = t1
if not blocking and last_blocking:
last_wait = 0
t2 = time(line)
cid = idre.search(line).group(1)
if waiting == 0:
d = sub(t1, time(line))
if d >= thresh:
print t1, sub(t1, t2), cid, d
t1 = t2 = cid = blocking = waiting = last_wait = max_wait = 0
last_blocking = blocking
connidre = re.compile(r' zrpc-conn:(\d+.\d+.\d+.\d+:\d+) ')
def time_calls(f):
f, thresh = f
if f == '-':
f = sys.stdin
else:
f = xopen(f)
thresh = float(thresh)
t1 = None
maxd = 0
for line in f:
line = line.strip()
if ' calling ' in line:
t1 = time(line)
elif ' returns ' in line and t1 is not None:
d = sub(t1, time(line))
if d >= thresh:
print t1, d, connidre.search(line).group(1)
maxd = max(maxd, d)
t1 = None
print maxd
def xopen(f):
if f == '-':
return sys.stdin
if ' ' in f:
return os.popen(f, 'r')
return open(f)
def time_tpc(f):
f, thresh = f
if f == '-':
f = sys.stdin
else:
f = xopen(f)
thresh = float(thresh)
transactions = {}
for line in f:
line = line.strip()
if ' calling vote(' in line:
cid = connidre.search(line).group(1)
transactions[cid] = time(line),
elif ' vote returns None' in line:
cid = connidre.search(line).group(1)
transactions[cid] += time(line), 'n'
elif ' vote() raised' in line:
cid = connidre.search(line).group(1)
transactions[cid] += time(line), 'e'
elif ' vote returns ' in line:
# delayed, skip
cid = connidre.search(line).group(1)
transactions[cid] += time(line), 'd'
elif ' calling tpc_abort(' in line:
cid = connidre.search(line).group(1)
if cid in transactions:
t1, t2, vs = transactions[cid]
t = time(line)
d = sub(t1, t)
if d >= thresh:
print 'a', t1, cid, sub(t1, t2), vs, sub(t2, t)
del transactions[cid]
elif ' calling tpc_finish(' in line:
if cid in transactions:
cid = connidre.search(line).group(1)
transactions[cid] += time(line),
elif ' tpc_finish returns ' in line:
if cid in transactions:
t1, t2, vs, t3 = transactions[cid]
t = time(line)
d = sub(t1, t)
if d >= thresh:
print 'c', t1, cid, sub(t1, t2), vs, sub(t2, t3), sub(t3, t)
del transactions[cid]
newobre = re.compile(r"storea\(.*, '\\x00\\x00\\x00\\x00\\x00")
def time_trans(f):
f, thresh = f
if f == '-':
f = sys.stdin
else:
f = xopen(f)
thresh = float(thresh)
transactions = {}
for line in f:
line = line.strip()
if ' calling tpc_begin(' in line:
cid = connidre.search(line).group(1)
transactions[cid] = time(line), [0, 0]
if ' calling storea(' in line:
cid = connidre.search(line).group(1)
if cid in transactions:
transactions[cid][1][0] += 1
if not newobre.search(line):
transactions[cid][1][1] += 1
elif ' calling vote(' in line:
cid = connidre.search(line).group(1)
if cid in transactions:
transactions[cid] += time(line),
elif ' vote returns None' in line:
cid = connidre.search(line).group(1)
if cid in transactions:
transactions[cid] += time(line), 'n'
elif ' vote() raised' in line:
cid = connidre.search(line).group(1)
if cid in transactions:
transactions[cid] += time(line), 'e'
elif ' vote returns ' in line:
# delayed, skip
cid = connidre.search(line).group(1)
if cid in transactions:
transactions[cid] += time(line), 'd'
elif ' calling tpc_abort(' in line:
cid = connidre.search(line).group(1)
if cid in transactions:
try:
t0, (stores, old), t1, t2, vs = transactions[cid]
except ValueError:
pass
else:
t = time(line)
d = sub(t1, t)
if d >= thresh:
print t1, cid, "%s/%s" % (stores, old), \
sub(t0, t1), sub(t1, t2), vs, \
sub(t2, t), 'abort'
del transactions[cid]
elif ' calling tpc_finish(' in line:
if cid in transactions:
cid = connidre.search(line).group(1)
transactions[cid] += time(line),
elif ' tpc_finish returns ' in line:
if cid in transactions:
t0, (stores, old), t1, t2, vs, t3 = transactions[cid]
t = time(line)
d = sub(t1, t)
if d >= thresh:
print t1, cid, "%s/%s" % (stores, old), \
sub(t0, t1), sub(t1, t2), vs, \
sub(t2, t3), sub(t3, t)
del transactions[cid]
def minute(f, slice=16, detail=1, summary=1):
f, = f
if f == '-':
f = sys.stdin
else:
f = xopen(f)
cols = ["time", "reads", "stores", "commits", "aborts", "txns"]
fmt = "%18s %6s %6s %7s %6s %6s"
print fmt % cols
print fmt % ["-"*len(col) for col in cols]
mlast = r = s = c = a = cl = None
rs = []
ss = []
cs = []
as = []
ts = []
cls = []
for line in f:
line = line.strip()
if (line.find('returns') > 0
or line.find('storea') > 0
or line.find('tpc_abort') > 0
):
client = connidre.search(line).group(1)
m = line[:slice]
if m != mlast:
if mlast:
if detail:
print fmt % (mlast, len(cl), r, s, c, a, a+c)
cls.append(len(cl))
rs.append(r)
ss.append(s)
cs.append(c)
as.append(a)
ts.append(c+a)
mlast = m
r = s = c = a = 0
cl = {}
if line.find('zeoLoad') > 0:
r += 1
cl[client] = 1
elif line.find('storea') > 0:
s += 1
cl[client] = 1
elif line.find('tpc_finish') > 0:
c += 1
cl[client] = 1
elif line.find('tpc_abort') > 0:
a += 1
cl[client] = 1
if mlast:
if detail:
print fmt % (mlast, len(cl), r, s, c, a, a+c)
cls.append(len(cl))
rs.append(r)
ss.append(s)
cs.append(c)
as.append(a)
ts.append(c+a)
if summary:
print
print 'Summary: \t', '\t'.join(('min', '10%', '25%', 'med',
'75%', '90%', 'max', 'mean'))
print "n=%6d\t" % len(cls), '-'*62
print 'Clients: \t', '\t'.join(map(str,stats(cls)))
print 'Reads: \t', '\t'.join(map(str,stats( rs)))
print 'Stores: \t', '\t'.join(map(str,stats( ss)))
print 'Commits: \t', '\t'.join(map(str,stats( cs)))
print 'Aborts: \t', '\t'.join(map(str,stats( as)))
print 'Trans: \t', '\t'.join(map(str,stats( ts)))
def stats(s):
s.sort()
min = s[0]
max = s[-1]
n = len(s)
out = [min]
ni = n + 1
for p in .1, .25, .5, .75, .90:
lp = ni*p
l = int(lp)
if lp < 1 or lp > n:
out.append('-')
elif abs(lp-l) < .00001:
out.append(s[l-1])
else:
out.append(int(s[l-1] + (lp - l) * (s[l] - s[l-1])))
mean = 0.0
for v in s:
mean += v
out.extend([max, int(mean/n)])
return out
def minutes(f):
minute(f, 16, detail=0)
def hour(f):
minute(f, 13)
def day(f):
minute(f, 10)
def hours(f):
minute(f, 13, detail=0)
def days(f):
minute(f, 10, detail=0)
new_connection_idre = re.compile(r"new connection \('(\d+.\d+.\d+.\d+)', (\d+)\):")
def verify(f):
f, = f
if f == '-':
f = sys.stdin
else:
f = xopen(f)
t1 = None
nv = {}
for line in f:
if line.find('new connection') > 0:
m = new_connection_idre.search(line)
cid = "%s:%s" % (m.group(1), m.group(2))
nv[cid] = [time(line), 0]
elif line.find('calling zeoVerify(') > 0:
cid = connidre.search(line).group(1)
nv[cid][1] += 1
elif line.find('calling endZeoVerify()') > 0:
cid = connidre.search(line).group(1)
t1, n = nv[cid]
if n:
d = sub(t1, time(line))
print cid, t1, n, d, n and (d*1000.0/n) or '-'
def recovery(f):
f, = f
if f == '-':
f = sys.stdin
else:
f = xopen(f)
last = ''
trans = []
n = 0
for line in f:
n += 1
if line.find('RecoveryServer') < 0:
continue
l = line.find('sending transaction ')
if l > 0 and last.find('sending transaction ') > 0:
trans.append(line[l+20:].strip())
else:
if trans:
if len(trans) > 1:
print " ... %s similar records skipped ..." % (
len(trans) - 1)
print n, last.strip()
trans=[]
print n, line.strip()
last = line
if len(trans) > 1:
print " ... %s similar records skipped ..." % (
len(trans) - 1)
print n, last.strip()
if __name__ == '__main__':
globals()[sys.argv[1]](sys.argv[2:])
ZODB/src/scripts/README.txt 0000640 0001750 0000144 00000011434 10111534045 015133 0 ustar ctheune users This directory contains a collection of utilities for managing ZODB
databases. Some are more useful than others. If you install ZODB
using distutils ("python setup.py install"), fsdump.py, fstest.py,
repozo.py, and zeopack.py will be installed in /usr/local/bin.
Unless otherwise noted, these scripts are invoked with the name of the
Data.fs file as their only argument. Example: checkbtrees.py data.fs.
analyze.py -- a transaction analyzer for FileStorage
Reports on the data in a FileStorage. The report is organized by
class. It shows total data, as well as separate reports for current
and historical revisions of objects.
checkbtrees.py -- checks BTrees in a FileStorage for corruption
Attempts to find all the BTrees contained in a Data.fs, calls their
_check() methods, and runs them through BTrees.check.check().
fsdump.py -- summarize FileStorage contents, one line per revision
Prints a report of FileStorage contents, with one line for each
transaction and one line for each data record in that transaction.
Includes time stamps, file positions, and class names.
fsoids.py -- trace all uses of specified oids in a FileStorage
For heavy debugging.
A set of oids is specified by text file listing and/or command line.
A report is generated showing all uses of these oids in the database:
all new-revision creation/modifications, all references from all
revisions of other objects, and all creation undos.
fstest.py -- simple consistency checker for FileStorage
usage: fstest.py [-v] data.fs
The fstest tool will scan all the data in a FileStorage and report an
error if it finds any corrupt transaction data. The tool will print a
message when the first error is detected an exit.
The tool accepts one or more -v arguments. If a single -v is used, it
will print a line of text for each transaction record it encounters.
If two -v arguments are used, it will also print a line of text for
each object. The objects for a transaction will be printed before the
transaction itself.
Note: It does not check the consistency of the object pickles. It is
possible for the damage to occur only in the part of the file that
stores object pickles. Those errors will go undetected.
space.py -- report space used by objects in a FileStorage
usage: space.py [-v] data.fs
This ignores revisions and versions.
netspace.py -- hackish attempt to report on size of objects
usage: netspace.py [-P | -v] data.fs
-P: do a pack first
-v: print info for all objects, even if a traversal path isn't found
Traverses objects from the database root and attempts to calculate
size of object, including all reachable subobjects.
parsezeolog.py -- parse BLATHER logs from ZEO server
This script may be obsolete. It has not been tested against the
current log output of the ZEO server.
Reports on the time and size of transactions committed by a ZEO
server, by inspecting log messages at BLATHER level.
repozo.py -- incremental backup utility for FileStorage
Run the script with the -h option to see usage details.
timeout.py -- script to test transaction timeout
usage: timeout.py address delay [storage-name]
This script connects to a storage, begins a transaction, calls store()
and tpc_vote(), and then sleeps forever. This should trigger the
transaction timeout feature of the server.
zeopack.py -- pack a ZEO server
The script connects to a server and calls pack() on a specific
storage. See the script for usage details.
zeoreplay.py -- experimental script to replay transactions from a ZEO log
Like parsezeolog.py, this may be obsolete because it was written
against an earlier version of the ZEO server. See the script for
usage details.
zeoup.py
usage: zeoup.py [options]
The test will connect to a ZEO server, load the root object, and
attempt to update the zeoup counter in the root. It will report
success if it updates to counter or if it gets a ConflictError. A
ConflictError is considered a success, because the client was able to
start a transaction.
See the script for details about the options.
zodbload.py -- exercise ZODB under a heavy synthesized Zope-like load
See the module docstring for details. Note that this script requires
Zope. New in ZODB3 3.1.4.
zeoserverlog.py -- analyze ZEO server log for performance statistics
See the module docstring for details; there are a large number of
options. New in ZODB3 3.1.4.
fsrefs.py -- check FileStorage for dangling references
fstail.py -- display the most recent transactions in a FileStorage
usage: fstail.py [-n nxtn] data.fs
The most recent ntxn transactions are displayed, to stdout.
Optional argument -n specifies ntxn, and defaults to 10.
migrate.py -- do a storage migration and gather statistics
See the module docstring for details.
zeoqueue.py -- report number of clients currently waiting in the ZEO queue
See the module docstring for details.
ZODB/src/scripts/timeout.py 0000750 0001750 0000144 00000003253 10051215041 015472 0 ustar ctheune users #!/usr/bin/env python2.3
"""Transaction timeout test script.
This script connects to a storage, begins a transaction, calls store()
and tpc_vote(), and then sleeps forever. This should trigger the
transaction timeout feature of the server.
usage: timeout.py address delay [storage-name]
"""
import sys
import time
from ZODB.Transaction import Transaction
from ZODB.tests.MinPO import MinPO
from ZODB.tests.StorageTestBase import zodb_pickle
from ZEO.ClientStorage import ClientStorage
ZERO = '\0'*8
def main():
if len(sys.argv) not in (3, 4):
sys.stderr.write("Usage: timeout.py address delay [storage-name]\n" %
sys.argv[0])
sys.exit(2)
hostport = sys.argv[1]
delay = float(sys.argv[2])
if sys.argv[3:]:
name = sys.argv[3]
else:
name = "1"
if "/" in hostport:
address = hostport
else:
if ":" in hostport:
i = hostport.index(":")
host, port = hostport[:i], hostport[i+1:]
else:
host, port = "", hostport
port = int(port)
address = (host, port)
print "Connecting to %s..." % repr(address)
storage = ClientStorage(address, name)
print "Connected. Now starting a transaction..."
oid = storage.new_oid()
version = ""
revid = ZERO
data = MinPO("timeout.py")
pickled_data = zodb_pickle(data)
t = Transaction()
t.user = "timeout.py"
storage.tpc_begin(t)
storage.store(oid, revid, pickled_data, version, t)
print "Stored. Now voting..."
storage.tpc_vote(t)
print "Voted; now sleeping %s..." % delay
time.sleep(delay)
print "Done."
if __name__ == "__main__":
main()
ZODB/src/scripts/fsstats.py 0000750 0001750 0000144 00000013037 10051215041 015474 0 ustar ctheune users #!/usr/bin/env python2.3
"""Print details statistics from fsdump output."""
import re
import sys
rx_txn = re.compile("tid=([0-9a-f]+).*size=(\d+)")
rx_data = re.compile("oid=([0-9a-f]+) class=(\S+) size=(\d+)")
def sort_byhsize(seq, reverse=False):
L = [(v.size(), k, v) for k, v in seq]
L.sort()
if reverse:
L.reverse()
return [(k, v) for n, k, v in L]
class Histogram(dict):
def add(self, size):
self[size] = self.get(size, 0) + 1
def size(self):
return sum(self.itervalues())
def mean(self):
product = sum([k * v for k, v in self.iteritems()])
return product / self.size()
def median(self):
# close enough?
n = self.size() / 2
L = self.keys()
L.sort()
L.reverse()
while 1:
k = L.pop()
if self[k] > n:
return k
n -= self[k]
def mode(self):
mode = 0
value = 0
for k, v in self.iteritems():
if v > value:
value = v
mode = k
return mode
def make_bins(self, binsize):
maxkey = max(self.iterkeys())
self.binsize = binsize
self.bins = [0] * (1 + maxkey / binsize)
for k, v in self.iteritems():
b = k / binsize
self.bins[b] += v
def report(self, name, binsize=50, usebins=False, gaps=True, skip=True):
if usebins:
# Use existing bins with whatever size they have
binsize = self.binsize
else:
# Make new bins
self.make_bins(binsize)
maxval = max(self.bins)
# Print up to 40 dots for a value
dot = max(maxval / 40, 1)
tot = sum(self.bins)
print name
print "Total", tot,
print "Median", self.median(),
print "Mean", self.mean(),
print "Mode", self.mode(),
print "Max", max(self)
print "One * represents", dot
gap = False
cum = 0
for i, n in enumerate(self.bins):
if gaps and (not n or (skip and not n / dot)):
if not gap:
print " ..."
gap = True
continue
gap = False
p = 100 * n / tot
cum += n
pc = 100 * cum / tot
print "%6d %6d %3d%% %3d%% %s" % (
i * binsize, n, p, pc, "*" * (n / dot))
print
def class_detail(class_size):
# summary of classes
fmt = "%5s %6s %6s %6s %-50.50s"
labels = ["num", "median", "mean", "mode", "class"]
print fmt % tuple(labels)
print fmt % tuple(["-" * len(s) for s in labels])
for klass, h in sort_byhsize(class_size.iteritems()):
print fmt % (h.size(), h.median(), h.mean(), h.mode(), klass)
print
# per class details
for klass, h in sort_byhsize(class_size.iteritems(), reverse=True):
h.make_bins(50)
if len(filter(None, h.bins)) == 1:
continue
h.report("Object size for %s" % klass, usebins=True)
def revision_detail(lifetimes, classes):
# Report per-class details for any object modified more than once
for name, oids in classes.iteritems():
h = Histogram()
keep = False
for oid in dict.fromkeys(oids, 1):
L = lifetimes.get(oid)
n = len(L)
h.add(n)
if n > 1:
keep = True
if keep:
h.report("Number of revisions for %s" % name, binsize=10)
def main(path):
txn_objects = Histogram() # histogram of txn size in objects
txn_bytes = Histogram() # histogram of txn size in bytes
obj_size = Histogram() # histogram of object size
n_updates = Histogram() # oid -> num updates
n_classes = Histogram() # class -> num objects
lifetimes = {} # oid -> list of tids
class_size = {} # class -> histogram of object size
classes = {} # class -> list of oids
MAX = 0
tid = None
f = open(path, "rb")
for i, line in enumerate(f):
if MAX and i > MAX:
break
if line.startswith(" data"):
m = rx_data.search(line)
if not m:
continue
oid, klass, size = m.groups()
size = int(size)
obj_size.add(size)
n_updates.add(oid)
n_classes.add(klass)
h = class_size.get(klass)
if h is None:
h = class_size[klass] = Histogram()
h.add(size)
L = lifetimes.setdefault(oid, [])
L.append(tid)
L = classes.setdefault(klass, [])
L.append(oid)
objects += 1
elif line.startswith("Trans"):
if tid is not None:
txn_objects.add(objects)
m = rx_txn.search(line)
if not m:
continue
tid, size = m.groups()
size = int(size)
objects = 0
txn_bytes.add(size)
f.close()
print "Summary: %d txns, %d objects, %d revisions" % (
txn_objects.size(), len(n_updates), n_updates.size())
print
txn_bytes.report("Transaction size (bytes)", binsize=1024)
txn_objects.report("Transaction size (objects)", binsize=10)
obj_size.report("Object size", binsize=128)
# object lifetime info
h = Histogram()
for k, v in lifetimes.items():
h.add(len(v))
h.report("Number of revisions", binsize=10, skip=False)
# details about revisions
revision_detail(lifetimes, classes)
class_detail(class_size)
if __name__ == "__main__":
main(sys.argv[1])
ZODB/src/scripts/fsoids.py 0000640 0001750 0000144 00000004463 10305413212 015277 0 ustar ctheune users #!/usr/bin/env python2.3
##############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""FileStorage oid-tracer.
usage: fsoids.py [-f oid_file] Data.fs [oid]...
Display information about all occurrences of specified oids in a FileStorage.
This is meant for heavy debugging.
This includes all revisions of the oids, all objects referenced by the
oids, and all revisions of all objects referring to the oids.
If specified, oid_file is an input text file, containing one oid per
line. oids are specified as integers, in any of Python's integer
notations (typically like 0x341a). One or more oids can also be specified
on the command line.
The output is grouped by oid, from smallest to largest, and sub-grouped
by transaction, from oldest to newest.
This will not alter the FileStorage, but running against a live FileStorage
is not recommended (spurious error messages may result).
See testfsoids.py for a tutorial doctest.
"""
import sys
from ZODB.FileStorage.fsoids import Tracer
def usage():
print __doc__
def main():
import getopt
try:
opts, args = getopt.getopt(sys.argv[1:], 'f:')
if not args:
usage()
raise ValueError("Must specify a FileStorage")
path = None
for k, v in opts:
if k == '-f':
path = v
except (getopt.error, ValueError):
usage()
raise
c = Tracer(args[0])
for oid in args[1:]:
as_int = int(oid, 0) # 0 == auto-detect base
c.register_oids(as_int)
if path is not None:
for line in open(path):
as_int = int(line, 0)
c.register_oids(as_int)
if not c.oids:
raise ValueError("no oids specified")
c.run()
c.report()
if __name__ == "__main__":
main()
ZODB/src/ZEO/ 0000751 0001750 0000144 00000000000 10424632373 012412 5 ustar ctheune users ZODB/src/ZEO/auth/ 0000751 0001750 0000144 00000000000 10424632373 013353 5 ustar ctheune users ZODB/src/ZEO/auth/auth_digest.py 0000640 0001750 0000144 00000012345 10305413212 016215 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Digest authentication for ZEO
This authentication mechanism follows the design of HTTP digest
authentication (RFC 2069). It is a simple challenge-response protocol
that does not send passwords in the clear, but does not offer strong
security. The RFC discusses many of the limitations of this kind of
protocol.
Guard the password database as if it contained plaintext passwords.
It stores the hash of a username and password. This does not expose
the plaintext password, but it is sensitive nonetheless. An attacker
with the hash can impersonate the real user. This is a limitation of
the simple digest scheme.
HTTP is a stateless protocol, and ZEO is a stateful protocol. The
security requirements are quite different as a result. The HTTP
protocol uses a nonce as a challenge. The ZEO protocol requires a
separate session key that is used for message authentication. We
generate a second nonce for this purpose; the hash of nonce and
user/realm/password is used as the session key.
TODO: I'm not sure if this is a sound approach; SRP would be preferred.
"""
import os
import random
import sha
import struct
import time
from ZEO.auth.base import Database, Client
from ZEO.StorageServer import ZEOStorage
from ZEO.Exceptions import AuthError
def get_random_bytes(n=8):
if os.path.exists("/dev/urandom"):
f = open("/dev/urandom")
s = f.read(n)
f.close()
else:
L = [chr(random.randint(0, 255)) for i in range(n)]
s = "".join(L)
return s
def hexdigest(s):
return sha.new(s).hexdigest()
class DigestDatabase(Database):
def __init__(self, filename, realm=None):
Database.__init__(self, filename, realm)
# Initialize a key used to build the nonce for a challenge.
# We need one key for the lifetime of the server, so it
# is convenient to store in on the database.
self.noncekey = get_random_bytes(8)
def _store_password(self, username, password):
dig = hexdigest("%s:%s:%s" % (username, self.realm, password))
self._users[username] = dig
def session_key(h_up, nonce):
# The hash itself is a bit too short to be a session key.
# HMAC wants a 64-byte key. We don't want to use h_up
# directly because it would never change over time. Instead
# use the hash plus part of h_up.
return sha.new("%s:%s" % (h_up, nonce)).digest() + h_up[:44]
class StorageClass(ZEOStorage):
def set_database(self, database):
assert isinstance(database, DigestDatabase)
self.database = database
self.noncekey = database.noncekey
def _get_time(self):
# Return a string representing the current time.
t = int(time.time())
return struct.pack("i", t)
def _get_nonce(self):
# RFC 2069 recommends a nonce of the form
# H(client-IP ":" time-stamp ":" private-key)
dig = sha.sha()
dig.update(str(self.connection.addr))
dig.update(self._get_time())
dig.update(self.noncekey)
return dig.hexdigest()
def auth_get_challenge(self):
"""Return realm, challenge, and nonce."""
self._challenge = self._get_nonce()
self._key_nonce = self._get_nonce()
return self.auth_realm, self._challenge, self._key_nonce
def auth_response(self, resp):
# verify client response
user, challenge, response = resp
# Since zrpc is a stateful protocol, we just store the nonce
# we sent to the client. It will need to generate a new
# nonce for a new connection anyway.
if self._challenge != challenge:
raise ValueError("invalid challenge")
# lookup user in database
h_up = self.database.get_password(user)
# regeneration resp from user, password, and nonce
check = hexdigest("%s:%s" % (h_up, challenge))
if check == response:
self.connection.setSessionKey(session_key(h_up, self._key_nonce))
return self.finish_auth(check == response)
extensions = [auth_get_challenge, auth_response]
class DigestClient(Client):
extensions = ["auth_get_challenge", "auth_response"]
def start(self, username, realm, password):
_realm, challenge, nonce = self.stub.auth_get_challenge()
if _realm != realm:
raise AuthError("expected realm %r, got realm %r"
% (_realm, realm))
h_up = hexdigest("%s:%s:%s" % (username, realm, password))
resp_dig = hexdigest("%s:%s" % (h_up, challenge))
result = self.stub.auth_response((username, challenge, resp_dig))
if result:
return session_key(h_up, nonce)
else:
return None
ZODB/src/ZEO/auth/__init__.py 0000640 0001750 0000144 00000002307 10305413212 015451 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
_auth_modules = {}
def get_module(name):
if name == 'sha':
from auth_sha import StorageClass, SHAClient, Database
return StorageClass, SHAClient, Database
elif name == 'digest':
from auth_digest import StorageClass, DigestClient, DigestDatabase
return StorageClass, DigestClient, DigestDatabase
else:
return _auth_modules.get(name)
def register_module(name, storage_class, client, db):
if _auth_modules.has_key(name):
raise TypeError("%s is already registred" % name)
_auth_modules[name] = storage_class, client, db
ZODB/src/ZEO/auth/hmac.py 0000640 0001750 0000144 00000005730 10305413212 014625 0 ustar ctheune users """HMAC (Keyed-Hashing for Message Authentication) Python module.
Implements the HMAC algorithm as described by RFC 2104.
"""
def _strxor(s1, s2):
"""Utility method. XOR the two strings s1 and s2 (must have same length).
"""
return "".join(map(lambda x, y: chr(ord(x) ^ ord(y)), s1, s2))
# The size of the digests returned by HMAC depends on the underlying
# hashing module used.
digest_size = None
class HMAC:
"""RFC2104 HMAC class.
This supports the API for Cryptographic Hash Functions (PEP 247).
"""
def __init__(self, key, msg = None, digestmod = None):
"""Create a new HMAC object.
key: key for the keyed hash object.
msg: Initial input for the hash, if provided.
digestmod: A module supporting PEP 247. Defaults to the md5 module.
"""
if digestmod is None:
import md5
digestmod = md5
self.digestmod = digestmod
self.outer = digestmod.new()
self.inner = digestmod.new()
self.digest_size = digestmod.digest_size
blocksize = 64
ipad = "\x36" * blocksize
opad = "\x5C" * blocksize
if len(key) > blocksize:
key = digestmod.new(key).digest()
key = key + chr(0) * (blocksize - len(key))
self.outer.update(_strxor(key, opad))
self.inner.update(_strxor(key, ipad))
if msg is not None:
self.update(msg)
## def clear(self):
## raise NotImplementedError("clear() method not available in HMAC.")
def update(self, msg):
"""Update this hashing object with the string msg.
"""
self.inner.update(msg)
def copy(self):
"""Return a separate copy of this hashing object.
An update to this copy won't affect the original object.
"""
other = HMAC("")
other.digestmod = self.digestmod
other.inner = self.inner.copy()
other.outer = self.outer.copy()
return other
def digest(self):
"""Return the hash value of this hashing object.
This returns a string containing 8-bit data. The object is
not altered in any way by this function; you can continue
updating the object after calling this function.
"""
h = self.outer.copy()
h.update(self.inner.digest())
return h.digest()
def hexdigest(self):
"""Like digest(), but returns a string of hexadecimal digits instead.
"""
return "".join([hex(ord(x))[2:].zfill(2)
for x in tuple(self.digest())])
def new(key, msg = None, digestmod = None):
"""Create a new hashing object and return it.
key: The starting key for the hash.
msg: if available, will immediately be hashed into the object's starting
state.
You can now feed arbitrary strings into the object using its update()
method, and can ask for the hash value at any time by calling its digest()
method.
"""
return HMAC(key, msg, digestmod)
ZODB/src/ZEO/auth/base.py 0000640 0001750 0000144 00000010232 10305413212 014620 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Base classes for defining an authentication protocol.
Database -- abstract base class for password database
Client -- abstract base class for authentication client
"""
import os
import sha
class Client:
# Subclass should override to list the names of methods that
# will be called on the server.
extensions = []
def __init__(self, stub):
self.stub = stub
for m in self.extensions:
setattr(self.stub, m, self.stub.extensionMethod(m))
def sort(L):
"""Sort a list in-place and return it."""
L.sort()
return L
class Database:
"""Abstracts a password database.
This class is used both in the authentication process (via
get_password()) and by client scripts that manage the password
database file.
The password file is a simple, colon-separated text file mapping
usernames to password hashes. The hashes are SHA hex digests
produced from the password string.
"""
realm = None
def __init__(self, filename, realm=None):
"""Creates a new Database
filename: a string containing the full pathname of
the password database file. Must be readable by the user
running ZEO. Must be writeable by any client script that
accesses the database.
realm: the realm name (a string)
"""
self._users = {}
self.filename = filename
self.load()
if realm:
if self.realm and self.realm != realm:
raise ValueError("Specified realm %r differs from database "
"realm %r" % (realm or '', self.realm))
else:
self.realm = realm
def save(self, fd=None):
filename = self.filename
if not fd:
fd = open(filename, 'w')
if self.realm:
print >> fd, "realm", self.realm
for username in sort(self._users.keys()):
print >> fd, "%s: %s" % (username, self._users[username])
def load(self):
filename = self.filename
if not filename:
return
if not os.path.exists(filename):
return
fd = open(filename)
L = fd.readlines()
if not L:
return
if L[0].startswith("realm "):
line = L.pop(0).strip()
self.realm = line[len("realm "):]
for line in L:
username, hash = line.strip().split(":", 1)
self._users[username] = hash.strip()
def _store_password(self, username, password):
self._users[username] = self.hash(password)
def get_password(self, username):
"""Returns password hash for specified username.
Callers must check for LookupError, which is raised in
the case of a non-existent user specified."""
if not self._users.has_key(username):
raise LookupError("No such user: %s" % username)
return self._users[username]
def hash(self, s):
return sha.new(s).hexdigest()
def add_user(self, username, password):
if self._users.has_key(username):
raise LookupError("User %s already exists" % username)
self._store_password(username, password)
def del_user(self, username):
if not self._users.has_key(username):
raise LookupError("No such user: %s" % username)
del self._users[username]
def change_password(self, username, password):
if not self._users.has_key(username):
raise LookupError("No such user: %s" % username)
self._store_password(username, password)
ZODB/src/ZEO/tests/ 0000751 0001750 0000144 00000000000 10424632373 013554 5 ustar ctheune users ZODB/src/ZEO/tests/test_cache.py 0000640 0001750 0000144 00000014014 10214421637 016224 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Basic unit tests for a multi-version client cache."""
import os
import tempfile
import unittest
import ZEO.cache
from ZODB.utils import p64
n1 = p64(1)
n2 = p64(2)
n3 = p64(3)
n4 = p64(4)
n5 = p64(5)
class CacheTests(unittest.TestCase):
def setUp(self):
# The default cache size is much larger than we need here. Since
# testSerialization reads the entire file into a string, it's not
# good to leave it that big.
self.cache = ZEO.cache.ClientCache(size=1024**2)
self.cache.open()
def tearDown(self):
if self.cache.path:
os.remove(self.cache.path)
def testLastTid(self):
self.assertEqual(self.cache.getLastTid(), None)
self.cache.setLastTid(n2)
self.assertEqual(self.cache.getLastTid(), n2)
self.cache.invalidate(None, "", n1)
self.assertEqual(self.cache.getLastTid(), n2)
self.cache.invalidate(None, "", n3)
self.assertEqual(self.cache.getLastTid(), n3)
self.assertRaises(ValueError, self.cache.setLastTid, n2)
def testLoad(self):
data1 = "data for n1"
self.assertEqual(self.cache.load(n1, ""), None)
self.assertEqual(self.cache.load(n1, "version"), None)
self.cache.store(n1, "", n3, None, data1)
self.assertEqual(self.cache.load(n1, ""), (data1, n3, ""))
# The cache doesn't know whether version exists, because it
# only has non-version data.
self.assertEqual(self.cache.load(n1, "version"), None)
self.assertEqual(self.cache.modifiedInVersion(n1), None)
def testInvalidate(self):
data1 = "data for n1"
self.cache.store(n1, "", n3, None, data1)
self.cache.invalidate(n1, "", n4)
self.cache.invalidate(n2, "", n2)
self.assertEqual(self.cache.load(n1, ""), None)
self.assertEqual(self.cache.loadBefore(n1, n4),
(data1, n3, n4))
def testVersion(self):
data1 = "data for n1"
data1v = "data for n1 in version"
self.cache.store(n1, "version", n3, None, data1v)
self.assertEqual(self.cache.load(n1, ""), None)
self.assertEqual(self.cache.load(n1, "version"),
(data1v, n3, "version"))
self.assertEqual(self.cache.load(n1, "random"), None)
self.assertEqual(self.cache.modifiedInVersion(n1), "version")
self.cache.invalidate(n1, "version", n4)
self.assertEqual(self.cache.load(n1, "version"), None)
def testNonCurrent(self):
data1 = "data for n1"
data2 = "data for n2"
self.cache.store(n1, "", n4, None, data1)
self.cache.store(n1, "", n2, n3, data2)
# can't say anything about state before n2
self.assertEqual(self.cache.loadBefore(n1, n2), None)
# n3 is the upper bound of non-current record n2
self.assertEqual(self.cache.loadBefore(n1, n3), (data2, n2, n3))
# no data for between n2 and n3
self.assertEqual(self.cache.loadBefore(n1, n4), None)
self.cache.invalidate(n1, "", n5)
self.assertEqual(self.cache.loadBefore(n1, n5), (data1, n4, n5))
self.assertEqual(self.cache.loadBefore(n2, n4), None)
def testException(self):
self.assertRaises(ValueError,
self.cache.store,
n1, "version", n2, n3, "data")
self.cache.store(n1, "", n2, None, "data")
self.assertRaises(ValueError,
self.cache.store,
n1, "", n3, None, "data")
def testEviction(self):
# Manually override the current maxsize
maxsize = self.cache.size = self.cache.fc.maxsize = 3395 # 1245
self.cache.fc = ZEO.cache.FileCache(3395, None, self.cache)
# Trivial test of eviction code. Doesn't test non-current
# eviction.
data = ["z" * i for i in range(100)]
for i in range(50):
n = p64(i)
self.cache.store(n, "", n, None, data[i])
self.assertEquals(len(self.cache), i + 1)
# The cache now uses 1225 bytes. The next insert
# should delete some objects.
n = p64(50)
self.cache.store(n, "", n, None, data[51])
self.assert_(len(self.cache) < 51)
# TODO: Need to make sure eviction of non-current data
# and of version data are handled correctly.
def testSerialization(self):
self.cache.store(n1, "", n2, None, "data for n1")
self.cache.store(n2, "version", n2, None, "version data for n2")
self.cache.store(n3, "", n3, n4, "non-current data for n3")
self.cache.store(n3, "", n4, n5, "more non-current data for n3")
path = tempfile.mktemp()
# Copy data from self.cache into path, reaching into the cache
# guts to make the copy.
dst = open(path, "wb+")
src = self.cache.fc.f
src.seek(0)
dst.write(src.read(self.cache.fc.maxsize))
dst.close()
copy = ZEO.cache.ClientCache(path)
copy.open()
# Verify that internals of both objects are the same.
# Could also test that external API produces the same results.
eq = self.assertEqual
eq(copy.getLastTid(), self.cache.getLastTid())
eq(len(copy), len(self.cache))
eq(copy.version, self.cache.version)
eq(copy.current, self.cache.current)
eq(copy.noncurrent, self.cache.noncurrent)
def test_suite():
return unittest.makeSuite(CacheTests)
ZODB/src/ZEO/tests/__init__.py 0000640 0001750 0000144 00000001202 10057366465 015670 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
ZODB/src/ZEO/tests/forker.py 0000640 0001750 0000144 00000016103 10400504467 015413 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Library for forking storage server and connecting client storage"""
import os
import sys
import time
import errno
import socket
import logging
import StringIO
import tempfile
import logging
logger = logging.getLogger('ZEO.tests.forker')
class ZEOConfig:
"""Class to generate ZEO configuration file. """
def __init__(self, addr):
self.address = addr
self.read_only = None
self.invalidation_queue_size = None
self.monitor_address = None
self.transaction_timeout = None
self.authentication_protocol = None
self.authentication_database = None
self.authentication_realm = None
def dump(self, f):
print >> f, ""
print >> f, "address %s:%s" % self.address
if self.read_only is not None:
print >> f, "read-only", self.read_only and "true" or "false"
if self.invalidation_queue_size is not None:
print >> f, "invalidation-queue-size", self.invalidation_queue_size
if self.monitor_address is not None:
print >> f, "monitor-address %s:%s" % self.monitor_address
if self.transaction_timeout is not None:
print >> f, "transaction-timeout", self.transaction_timeout
if self.authentication_protocol is not None:
print >> f, "authentication-protocol", self.authentication_protocol
if self.authentication_database is not None:
print >> f, "authentication-database", self.authentication_database
if self.authentication_realm is not None:
print >> f, "authentication-realm", self.authentication_realm
print >> f, ""
logger = logging.getLogger()
print >> f
print >> f, ""
print >> f, "level", logger.level
for handler in logger.handlers:
if isinstance(handler, logging.FileHandler):
path = handler.baseFilename
elif isinstance(handler, logging.StreamHandler):
stream = handler.stream
if stream.name == "":
path = "STDOUT"
elif stream.name == "":
path = "STDERR"
else:
# just drop it on the floor; unlikely an issue when testing
continue
else:
# just drop it on the floor; unlikely an issue when testing
continue
# This doesn't convert the level values to names, so the
# generated configuration isn't as nice as it could be,
# but it doesn't really need to be.
print >> f, ""
print >> f, "level", handler.level
print >> f, "path ", path
if handler.formatter:
formatter = handler.formatter
if formatter._fmt:
print >> f, "format", encode_format(formatter._fmt)
if formatter.datefmt:
print >> f, "dateformat", encode_format(formatter.datefmt)
print >> f, ""
print >> f, ""
def __str__(self):
f = StringIO.StringIO()
self.dump(f)
return f.getvalue()
def encode_format(fmt):
# The list of replacements mirrors
# ZConfig.components.logger.handlers._control_char_rewrites
for xform in (("\n", r"\n"), ("\t", r"\t"), ("\b", r"\b"),
("\f", r"\f"), ("\r", r"\r")):
fmt = fmt.replace(*xform)
return fmt
def start_zeo_server(storage_conf, zeo_conf, port, keep=0):
"""Start a ZEO server in a separate process.
Takes two positional arguments a string containing the storage conf
and a ZEOConfig object.
Returns the ZEO port, the test server port, the pid, and the path
to the config file.
"""
# Store the config info in a temp file.
tmpfile = tempfile.mktemp(".conf")
fp = open(tmpfile, 'w')
zeo_conf.dump(fp)
fp.write(storage_conf)
fp.close()
# Find the zeoserver script
import ZEO.tests.zeoserver
script = ZEO.tests.zeoserver.__file__
if script.endswith('.pyc'):
script = script[:-1]
# Create a list of arguments, which we'll tuplify below
qa = _quote_arg
args = [qa(sys.executable), qa(script), '-C', qa(tmpfile)]
if keep:
args.append("-k")
d = os.environ.copy()
d['PYTHONPATH'] = os.pathsep.join(sys.path)
pid = os.spawnve(os.P_NOWAIT, sys.executable, tuple(args), d)
adminaddr = ('localhost', port + 1)
# We need to wait until the server starts, but not forever.
# 30 seconds is a somewhat arbitrary upper bound. A BDBStorage
# takes a long time to open -- more than 10 seconds on occasion.
for i in range(120):
time.sleep(0.25)
try:
logger.debug('connect %s', i)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(adminaddr)
ack = s.recv(1024)
s.close()
logging.debug('acked: %s' % ack)
break
except socket.error, e:
if e[0] not in (errno.ECONNREFUSED, errno.ECONNRESET):
raise
s.close()
else:
logging.debug('boo hoo')
raise
return ('localhost', port), adminaddr, pid, tmpfile
if sys.platform[:3].lower() == "win":
def _quote_arg(s):
return '"%s"' % s
else:
def _quote_arg(s):
return s
def shutdown_zeo_server(adminaddr):
# Do this in a loop to guard against the possibility that the
# client failed to connect to the adminaddr earlier. That really
# only requires two iterations, but do a third for pure
# superstition.
for i in range(3):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(.3)
try:
s.connect(adminaddr)
except socket.timeout:
# On FreeBSD 5.3 the connection just timed out
if i > 0:
break
raise
except socket.error, e:
# Mac OS X uses EINVAL to signal unbound socket timeout failure
if sys.platform == 'darwin':
if e[0] == errno.EINVAL and i > 0:
break
if e[0] == errno.ECONNREFUSED and i > 0:
break
raise
try:
ack = s.recv(1024)
except socket.error, e:
if e[0] == errno.ECONNRESET:
raise
ack = 'no ack received'
logger.debug('shutdown_zeo_server(): acked: %s' % ack)
s.close()
ZODB/src/ZEO/tests/multi.py 0000640 0001750 0000144 00000010531 10214421637 015254 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""A multi-client test of the ZEO storage server"""
# TODO: This code is currently broken.
import ZODB, ZODB.DB, ZODB.FileStorage, ZODB.POSException
import persistent
import persistent.mapping
import transaction
from ZEO.tests import forker
import os
import tempfile
import time
import types
VERBOSE = 1
CLIENTS = 4
RECORDS_PER_CLIENT = 100
CONFLICT_DELAY = 0.1
CONNECT_DELAY = 0.1
CLIENT_CACHE = '' # use temporary cache
class Record(persistent.Persistent):
def __init__(self, client=None, value=None):
self.client = client
self.value = None
self.next = None
def set_next(self, next):
self.next = next
class Stats(persistent.Persistent):
def __init__(self):
self.begin = time.time()
self.end = None
def done(self):
self.end = time.time()
def init_storage():
path = tempfile.mktemp()
if VERBOSE:
print "FileStorage path:", path
fs = ZODB.FileStorage.FileStorage(path)
db = ZODB.DB(fs)
root = db.open().root()
root["multi"] = persistent.mapping.PersistentMapping()
transaction.commit()
return fs
def start_server(addr):
storage = init_storage()
pid, exit = forker.start_zeo_server(storage, addr)
return pid, exit
def start_client(addr, client_func=None):
pid = os.fork()
if pid == 0:
try:
import ZEO.ClientStorage
if VERBOSE:
print "Client process started:", os.getpid()
cli = ZEO.ClientStorage.ClientStorage(addr, client=CLIENT_CACHE)
if client_func is None:
run(cli)
else:
client_func(cli)
cli.close()
finally:
os._exit(0)
else:
return pid
def run(storage):
if hasattr(storage, 'is_connected'):
while not storage.is_connected():
time.sleep(CONNECT_DELAY)
pid = os.getpid()
print "Client process connected:", pid, storage
db = ZODB.DB(storage)
root = db.open().root()
while 1:
try:
s = root[pid] = Stats()
transaction.commit()
except ZODB.POSException.ConflictError:
transaction.abort()
time.sleep(CONFLICT_DELAY)
else:
break
dict = root["multi"]
prev = None
i = 0
while i < RECORDS_PER_CLIENT:
try:
size = len(dict)
r = dict[size] = Record(pid, size)
if prev:
prev.set_next(r)
transaction.commit()
except ZODB.POSException.ConflictError, err:
transaction.abort()
time.sleep(CONFLICT_DELAY)
else:
i = i + 1
if VERBOSE and (i < 5 or i % 10 == 0):
print "Client %s: %s of %s" % (pid, i, RECORDS_PER_CLIENT)
s.done()
transaction.commit()
print "Client completed:", pid
def main(client_func=None):
if VERBOSE:
print "Main process:", os.getpid()
addr = tempfile.mktemp()
t0 = time.time()
server_pid, server = start_server(addr)
t1 = time.time()
pids = []
for i in range(CLIENTS):
pids.append(start_client(addr, client_func))
for pid in pids:
assert type(pid) == types.IntType, "invalid pid type: %s (%s)" % \
(repr(pid), type(pid))
try:
if VERBOSE:
print "waitpid(%s)" % repr(pid)
os.waitpid(pid, 0)
except os.error, err:
print "waitpid(%s) failed: %s" % (repr(pid), err)
t2 = time.time()
server.close()
os.waitpid(server_pid, 0)
# TODO: Should check that the results are consistent!
print "Total time:", t2 - t0
print "Server start time", t1 - t0
print "Client time:", t2 - t1
if __name__ == "__main__":
main()
ZODB/src/ZEO/tests/speed.py 0000640 0001750 0000144 00000013771 10057366465 015247 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
usage="""Test speed of a ZODB storage
Options:
-d file The data file to use as input.
The default is this script.
-n n The number of repititions
-s module A module that defines a 'Storage'
attribute, which is an open storage.
If not specified, a FileStorage will ne
used.
-z Test compressing data
-D Run in debug mode
-L Test loads as well as stores by minimizing
the cache after eachrun
-M Output means only
-C Run with a persistent client cache
-U Run ZEO using a Unix domain socket
-t n Number of concurrent threads to run.
"""
import asyncore
import sys, os, getopt, time
##sys.path.insert(0, os.getcwd())
import persistent
import transaction
import ZODB
from ZODB.POSException import ConflictError
from ZEO.tests import forker
class P(persistent.Persistent):
pass
fs_name = "zeo-speed.fs"
class ZEOExit(asyncore.file_dispatcher):
"""Used to exit ZEO.StorageServer when run is done"""
def writable(self):
return 0
def readable(self):
return 1
def handle_read(self):
buf = self.recv(4)
assert buf == "done"
self.delete_fs()
os._exit(0)
def handle_close(self):
print "Parent process exited unexpectedly"
self.delete_fs()
os._exit(0)
def delete_fs(self):
os.unlink(fs_name)
os.unlink(fs_name + ".lock")
os.unlink(fs_name + ".tmp")
def work(db, results, nrep, compress, data, detailed, minimize, threadno=None):
for j in range(nrep):
for r in 1, 10, 100, 1000:
t = time.time()
conflicts = 0
jar = db.open()
while 1:
try:
transaction.begin()
rt = jar.root()
key = 's%s' % r
if rt.has_key(key):
p = rt[key]
else:
rt[key] = p =P()
for i in range(r):
v = getattr(p, str(i), P())
if compress is not None:
v.d = compress(data)
else:
v.d = data
setattr(p, str(i), v)
transaction.commit()
except ConflictError:
conflicts = conflicts + 1
else:
break
jar.close()
t = time.time() - t
if detailed:
if threadno is None:
print "%s\t%s\t%.4f\t%d" % (j, r, t, conflicts)
else:
print "%s\t%s\t%.4f\t%d\t%d" % (j, r, t, conflicts,
threadno)
results[r].append((t, conflicts))
rt=d=p=v=None # release all references
if minimize:
time.sleep(3)
jar.cacheMinimize()
def main(args):
opts, args = getopt.getopt(args, 'zd:n:Ds:LMt:U')
s = None
compress = None
data=sys.argv[0]
nrep=5
minimize=0
detailed=1
cache = None
domain = 'AF_INET'
threads = 1
for o, v in opts:
if o=='-n': nrep = int(v)
elif o=='-d': data = v
elif o=='-s': s = v
elif o=='-z':
import zlib
compress = zlib.compress
elif o=='-L':
minimize=1
elif o=='-M':
detailed=0
elif o=='-D':
global debug
os.environ['STUPID_LOG_FILE']=''
os.environ['STUPID_LOG_SEVERITY']='-999'
debug = 1
elif o == '-C':
cache = 'speed'
elif o == '-U':
domain = 'AF_UNIX'
elif o == '-t':
threads = int(v)
zeo_pipe = None
if s:
s = __import__(s, globals(), globals(), ('__doc__',))
s = s.Storage
server = None
else:
s, server, pid = forker.start_zeo("FileStorage",
(fs_name, 1), domain=domain)
data=open(data).read()
db=ZODB.DB(s,
# disable cache deactivation
cache_size=4000,
cache_deactivate_after=6000,)
print "Beginning work..."
results={1:[], 10:[], 100:[], 1000:[]}
if threads > 1:
import threading
l = []
for i in range(threads):
t = threading.Thread(target=work,
args=(db, results, nrep, compress, data,
detailed, minimize, i))
l.append(t)
for t in l:
t.start()
for t in l:
t.join()
else:
work(db, results, nrep, compress, data, detailed, minimize)
if server is not None:
server.close()
os.waitpid(pid, 0)
if detailed:
print '-'*24
print "num\tmean\tmin\tmax"
for r in 1, 10, 100, 1000:
times = []
for time, conf in results[r]:
times.append(time)
t = mean(times)
print "%d\t%.4f\t%.4f\t%.4f" % (r, t, min(times), max(times))
def mean(l):
tot = 0
for v in l:
tot = tot + v
return tot / len(l)
##def compress(s):
## c = zlib.compressobj()
## o = c.compress(s)
## return o + c.flush()
if __name__=='__main__':
main(sys.argv[1:])
ZODB/src/ZEO/tests/auth_plaintext.py 0000640 0001750 0000144 00000004033 10057366465 017167 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Implements plaintext password authentication. The password is stored in
an SHA hash in the Database. The client sends over the plaintext
password, and the SHA hashing is done on the server side.
This mechanism offers *no network security at all*; the only security
is provided by not storing plaintext passwords on disk.
"""
import sha
from ZEO.StorageServer import ZEOStorage
from ZEO.auth import register_module
from ZEO.auth.base import Client, Database
def session_key(username, realm, password):
return sha.new("%s:%s:%s" % (username, realm, password)).hexdigest()
class StorageClass(ZEOStorage):
def auth(self, username, password):
try:
dbpw = self.database.get_password(username)
except LookupError:
return 0
password_dig = sha.new(password).hexdigest()
if dbpw == password_dig:
self.connection.setSessionKey(session_key(username,
self.database.realm,
password))
return self.finish_auth(dbpw == password_dig)
class PlaintextClient(Client):
extensions = ["auth"]
def start(self, username, realm, password):
if self.stub.auth(username, password):
return session_key(username, realm, password)
else:
return None
register_module("plaintext", StorageClass, PlaintextClient, Database)
ZODB/src/ZEO/tests/testZEO.py 0000640 0001750 0000144 00000026723 10400647314 015470 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Test suite for ZEO based on ZODB.tests."""
# System imports
import os
import random
import socket
import asyncore
import tempfile
import unittest
import logging
import shutil
# ZODB test support
import ZODB
from ZODB.tests.MinPO import MinPO
from ZODB.tests.StorageTestBase import zodb_unpickle
# ZODB test mixin classes
from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage, \
TransactionalUndoStorage, TransactionalUndoVersionStorage, \
PackableStorage, Synchronization, ConflictResolution, RevisionStorage, \
MTStorage, ReadOnlyStorage
from ZEO.ClientStorage import ClientStorage
from ZEO.tests import forker, Cache, CommitLockTests, ThreadTests
logger = logging.getLogger('ZEO.tests.testZEO')
class DummyDB:
def invalidate(self, *args):
pass
class OneTimeTests(unittest.TestCase):
def checkZEOVersionNumber(self):
import ZEO
# Starting with ZODB 3.4, the ZODB and ZEO version numbers should
# be identical.
self.assertEqual(ZODB.__version__, ZEO.version)
class MiscZEOTests:
"""ZEO tests that don't fit in elsewhere."""
def checkLargeUpdate(self):
obj = MinPO("X" * (10 * 128 * 1024))
self._dostore(data=obj)
def checkZEOInvalidation(self):
addr = self._storage._addr
storage2 = ClientStorage(addr, wait=1, min_disconnect_poll=0.1)
try:
oid = self._storage.new_oid()
ob = MinPO('first')
revid1 = self._dostore(oid, data=ob)
data, serial = storage2.load(oid, '')
self.assertEqual(zodb_unpickle(data), MinPO('first'))
self.assertEqual(serial, revid1)
revid2 = self._dostore(oid, data=MinPO('second'), revid=revid1)
for n in range(3):
# Let the server and client talk for a moment.
# Is there a better way to do this?
asyncore.poll(0.1)
data, serial = storage2.load(oid, '')
self.assertEqual(zodb_unpickle(data), MinPO('second'),
'Invalidation message was not sent!')
self.assertEqual(serial, revid2)
finally:
storage2.close()
def get_port():
"""Return a port that is not in use.
Checks if a port is in use by trying to connect to it. Assumes it
is not in use if connect raises an exception.
Raises RuntimeError after 10 tries.
"""
for i in range(10):
port = random.randrange(20000, 30000)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
try:
s.connect(('localhost', port))
except socket.error:
# Perhaps we should check value of error too.
return port
finally:
s.close()
raise RuntimeError("Can't find port")
class GenericTests(
# Base class for all ZODB tests
StorageTestBase.StorageTestBase,
# ZODB test mixin classes (in the same order as imported)
BasicStorage.BasicStorage,
PackableStorage.PackableStorage,
Synchronization.SynchronizedStorage,
MTStorage.MTStorage,
ReadOnlyStorage.ReadOnlyStorage,
# ZEO test mixin classes (in the same order as imported)
CommitLockTests.CommitLockVoteTests,
ThreadTests.ThreadTests,
# Locally defined (see above)
MiscZEOTests
):
"""Combine tests from various origins in one class."""
def setUp(self):
logger.info("setUp() %s", self.id())
port = get_port()
zconf = forker.ZEOConfig(('', port))
zport, adminaddr, pid, path = forker.start_zeo_server(self.getConfig(),
zconf, port)
self._pids = [pid]
self._servers = [adminaddr]
self._conf_path = path
self.blob_cache_dir = tempfile.mkdtemp() # This is the blob cache for ClientStorage
self._storage = ClientStorage(zport, '1', cache_size=20000000,
min_disconnect_poll=0.5, wait=1,
wait_timeout=60, blob_dir=self.blob_cache_dir)
self._storage.registerDB(DummyDB(), None)
def tearDown(self):
self._storage.close()
os.remove(self._conf_path)
shutil.rmtree(self.blob_cache_dir)
for server in self._servers:
forker.shutdown_zeo_server(server)
if hasattr(os, 'waitpid'):
# Not in Windows Python until 2.3
for pid in self._pids:
os.waitpid(pid, 0)
def open(self, read_only=0):
# Needed to support ReadOnlyStorage tests. Ought to be a
# cleaner way.
addr = self._storage._addr
self._storage.close()
self._storage = ClientStorage(addr, read_only=read_only, wait=1)
def checkWriteMethods(self):
# ReadOnlyStorage defines checkWriteMethods. The decision
# about where to raise the read-only error was changed after
# Zope 2.5 was released. So this test needs to detect Zope
# of the 2.5 vintage and skip the test.
# The __version__ attribute was not present in Zope 2.5.
if hasattr(ZODB, "__version__"):
ReadOnlyStorage.ReadOnlyStorage.checkWriteMethods(self)
def checkSortKey(self):
key = '%s:%s' % (self._storage._storage, self._storage._server_addr)
self.assertEqual(self._storage.sortKey(), key)
class FullGenericTests(
GenericTests,
Cache.StorageWithCache,
Cache.TransUndoStorageWithCache,
CommitLockTests.CommitLockUndoTests,
ConflictResolution.ConflictResolvingStorage,
ConflictResolution.ConflictResolvingTransUndoStorage,
PackableStorage.PackableUndoStorage,
RevisionStorage.RevisionStorage,
TransactionalUndoStorage.TransactionalUndoStorage,
TransactionalUndoVersionStorage.TransactionalUndoVersionStorage,
VersionStorage.VersionStorage,
):
"""Extend GenericTests with tests that MappingStorage can't pass."""
class FileStorageTests(FullGenericTests):
"""Test ZEO backed by a FileStorage."""
level = 2
def getConfig(self):
filename = self.__fs_base = tempfile.mktemp()
return """\
path %s
""" % filename
class MappingStorageTests(GenericTests):
"""ZEO backed by a Mapping storage."""
def getConfig(self):
return """"""
class BlobAdaptedFileStorageTests(GenericTests):
"""ZEO backed by a BlobStorage-adapted FileStorage."""
def setUp(self):
self.blobdir = tempfile.mkdtemp() # This is the blob directory on the ZEO server
self.filestorage = tempfile.mktemp()
super(BlobAdaptedFileStorageTests, self).setUp()
def tearDown(self):
shutil.rmtree(self.blobdir)
os.unlink(self.filestorage)
super(BlobAdaptedFileStorageTests, self).tearDown()
def getConfig(self):
return """
blob-dir %s
path %s
""" % (self.blobdir, self.filestorage)
def checkStoreBlob(self):
from ZODB.utils import oid_repr, tid_repr
from ZODB.Blobs.Blob import Blob
from ZODB.Blobs.BlobStorage import BLOB_SUFFIX
from ZODB.tests.StorageTestBase import zodb_pickle, ZERO, \
handle_serials
import transaction
somedata = 'a' * 10
blob = Blob()
bd_fh = blob.open('w')
bd_fh.write(somedata)
bd_fh.close()
tfname = bd_fh.name
oid = self._storage.new_oid()
data = zodb_pickle(blob)
self.assert_(os.path.exists(tfname))
t = transaction.Transaction()
try:
self._storage.tpc_begin(t)
r1 = self._storage.storeBlob(oid, ZERO, data, tfname, '', t)
r2 = self._storage.tpc_vote(t)
revid = handle_serials(oid, r1, r2)
self._storage.tpc_finish(t)
except:
self._storage.tpc_abort(t)
raise
self.assert_(not os.path.exists(tfname))
filename = os.path.join(self.blobdir, oid_repr(oid),
tid_repr(revid) + BLOB_SUFFIX)
self.assert_(os.path.exists(filename))
self.assertEqual(somedata, open(filename).read())
def checkLoadBlob(self):
from ZODB.Blobs.Blob import Blob
from ZODB.tests.StorageTestBase import zodb_pickle, ZERO, \
handle_serials
import transaction
version = ''
somedata = 'a' * 10
blob = Blob()
bd_fh = blob.open('w')
bd_fh.write(somedata)
bd_fh.close()
tfname = bd_fh.name
oid = self._storage.new_oid()
data = zodb_pickle(blob)
t = transaction.Transaction()
try:
self._storage.tpc_begin(t)
r1 = self._storage.storeBlob(oid, ZERO, data, tfname, '', t)
r2 = self._storage.tpc_vote(t)
serial = handle_serials(oid, r1, r2)
self._storage.tpc_finish(t)
except:
self._storage.tpc_abort(t)
raise
class Dummy:
def __init__(self):
self.acquired = 0
self.released = 0
def acquire(self):
self.acquired += 1
def release(self):
self.released += 1
class statusdict(dict):
def __init__(self):
self.added = []
self.removed = []
def __setitem__(self, k, v):
self.added.append(k)
super(statusdict, self).__setitem__(k, v)
def __delitem__(self, k):
self.removed.append(k)
super(statusdict, self).__delitem__(k)
# ensure that we do locking properly
filename = self._storage.fshelper.getBlobFilename(oid, serial)
thestatuslock = self._storage.blob_status_lock = Dummy()
thebloblock = Dummy()
def getBlobLock():
return thebloblock
# override getBlobLock to test that locking is performed
self._storage.getBlobLock = getBlobLock
thestatusdict = self._storage.blob_status = statusdict()
filename = self._storage.loadBlob(oid, serial, version)
self.assertEqual(thestatuslock.acquired, 2)
self.assertEqual(thestatuslock.released, 2)
self.assertEqual(thebloblock.acquired, 1)
self.assertEqual(thebloblock.released, 1)
self.assertEqual(thestatusdict.added, [(oid, serial)])
self.assertEqual(thestatusdict.removed, [(oid, serial)])
test_classes = [FileStorageTests, MappingStorageTests,
BlobAdaptedFileStorageTests]
def test_suite():
suite = unittest.TestSuite()
for klass in test_classes:
sub = unittest.makeSuite(klass, "check")
suite.addTest(sub)
return suite
if __name__ == "__main__":
unittest.main(defaultTest="test_suite")
ZODB/src/ZEO/tests/ThreadTests.py 0000640 0001750 0000144 00000012021 10057366465 016364 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Compromising positions involving threads."""
import threading
import transaction
from ZODB.tests.StorageTestBase import zodb_pickle, MinPO
import ZEO.ClientStorage
ZERO = '\0'*8
class BasicThread(threading.Thread):
def __init__(self, storage, doNextEvent, threadStartedEvent):
self.storage = storage
self.trans = transaction.Transaction()
self.doNextEvent = doNextEvent
self.threadStartedEvent = threadStartedEvent
self.gotValueError = 0
self.gotDisconnected = 0
threading.Thread.__init__(self)
self.setDaemon(1)
def join(self):
threading.Thread.join(self, 10)
assert not self.isAlive()
class GetsThroughVoteThread(BasicThread):
# This thread gets partially through a transaction before it turns
# execution over to another thread. We're trying to establish that a
# tpc_finish() after a storage has been closed by another thread will get
# a ClientStorageError error.
#
# This class gets does a tpc_begin(), store(), tpc_vote() and is waiting
# to do the tpc_finish() when the other thread closes the storage.
def run(self):
self.storage.tpc_begin(self.trans)
oid = self.storage.new_oid()
self.storage.store(oid, ZERO, zodb_pickle(MinPO("c")), '', self.trans)
self.storage.tpc_vote(self.trans)
self.threadStartedEvent.set()
self.doNextEvent.wait(10)
try:
self.storage.tpc_finish(self.trans)
except ZEO.ClientStorage.ClientStorageError:
self.gotValueError = 1
self.storage.tpc_abort(self.trans)
class GetsThroughBeginThread(BasicThread):
# This class is like the above except that it is intended to be run when
# another thread is already in a tpc_begin(). Thus, this thread will
# block in the tpc_begin until another thread closes the storage. When
# that happens, this one will get disconnected too.
def run(self):
try:
self.storage.tpc_begin(self.trans)
except ZEO.ClientStorage.ClientStorageError:
self.gotValueError = 1
class ThreadTests:
# Thread 1 should start a transaction, but not get all the way through it.
# Main thread should close the connection. Thread 1 should then get
# disconnected.
def checkDisconnectedOnThread2Close(self):
doNextEvent = threading.Event()
threadStartedEvent = threading.Event()
thread1 = GetsThroughVoteThread(self._storage,
doNextEvent, threadStartedEvent)
thread1.start()
threadStartedEvent.wait(10)
self._storage.close()
doNextEvent.set()
thread1.join()
self.assertEqual(thread1.gotValueError, 1)
# Thread 1 should start a transaction, but not get all the way through
# it. While thread 1 is in the middle of the transaction, a second thread
# should start a transaction, and it will block in the tcp_begin() --
# because thread 1 has acquired the lock in its tpc_begin(). Now the main
# thread closes the storage and both sub-threads should get disconnected.
def checkSecondBeginFails(self):
doNextEvent = threading.Event()
threadStartedEvent = threading.Event()
thread1 = GetsThroughVoteThread(self._storage,
doNextEvent, threadStartedEvent)
thread2 = GetsThroughBeginThread(self._storage,
doNextEvent, threadStartedEvent)
thread1.start()
threadStartedEvent.wait(1)
thread2.start()
self._storage.close()
doNextEvent.set()
thread1.join()
thread2.join()
self.assertEqual(thread1.gotValueError, 1)
self.assertEqual(thread2.gotValueError, 1)
# Run a bunch of threads doing small and large stores in parallel
def checkMTStores(self):
threads = []
for i in range(5):
t = threading.Thread(target=self.mtstorehelper)
threads.append(t)
t.start()
for t in threads:
t.join(30)
for i in threads:
self.failUnless(not t.isAlive())
# Helper for checkMTStores
def mtstorehelper(self):
name = threading.currentThread().getName()
objs = []
for i in range(10):
objs.append(MinPO("X" * 200000))
objs.append(MinPO("X"))
for obj in objs:
self._dostore(data=obj)
ZODB/src/ZEO/tests/testTransactionBuffer.py 0000640 0001750 0000144 00000004273 10057366465 020463 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import random
import unittest
from ZEO.TransactionBuffer import TransactionBuffer
def random_string(size):
"""Return a random string of size size."""
l = [chr(random.randrange(256)) for i in range(size)]
return "".join(l)
def new_store_data():
"""Return arbitrary data to use as argument to store() method."""
return random_string(8), '', random_string(random.randrange(1000))
def new_invalidate_data():
"""Return arbitrary data to use as argument to invalidate() method."""
return random_string(8), ''
class TransBufTests(unittest.TestCase):
def checkTypicalUsage(self):
tbuf = TransactionBuffer()
tbuf.store(*new_store_data())
tbuf.invalidate(*new_invalidate_data())
for o in tbuf:
pass
def doUpdates(self, tbuf):
data = []
for i in range(10):
d = new_store_data()
tbuf.store(*d)
data.append(d)
d = new_invalidate_data()
tbuf.invalidate(*d)
data.append(d)
for i, x in enumerate(tbuf):
if x[2] is None:
# the tbuf add a dummy None to invalidates
x = x[:2]
self.assertEqual(x, data[i])
def checkOrderPreserved(self):
tbuf = TransactionBuffer()
self.doUpdates(tbuf)
def checkReusable(self):
tbuf = TransactionBuffer()
self.doUpdates(tbuf)
tbuf.clear()
self.doUpdates(tbuf)
tbuf.clear()
self.doUpdates(tbuf)
def test_suite():
return unittest.makeSuite(TransBufTests, 'check')
ZODB/src/ZEO/tests/testConnection.py 0000640 0001750 0000144 00000007401 10057366465 017137 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Test setup for ZEO connection logic.
The actual tests are in ConnectionTests.py; this file provides the
platform-dependent scaffolding.
"""
# System imports
import unittest
# Import the actual test class
from ZEO.tests import ConnectionTests, InvalidationTests
class FileStorageConfig:
def getConfig(self, path, create, read_only):
return """\
path %s
create %s
read-only %s
""" % (path,
create and 'yes' or 'no',
read_only and 'yes' or 'no')
class BerkeleyStorageConfig:
def getConfig(self, path, create, read_only):
return """\
envdir %s
read-only %s
""" % (path, read_only and "yes" or "no")
class MappingStorageConfig:
def getConfig(self, path, create, read_only):
return """"""
class FileStorageConnectionTests(
FileStorageConfig,
ConnectionTests.ConnectionTests,
InvalidationTests.InvalidationTests
):
"""FileStorage-specific connection tests."""
level = 2
class FileStorageReconnectionTests(
FileStorageConfig,
ConnectionTests.ReconnectionTests,
):
"""FileStorage-specific re-connection tests."""
# Run this at level 1 because MappingStorage can't do reconnection tests
level = 1
class FileStorageInvqTests(
FileStorageConfig,
ConnectionTests.InvqTests
):
"""FileStorage-specific invalidation queue tests."""
level = 1
class FileStorageTimeoutTests(
FileStorageConfig,
ConnectionTests.TimeoutTests
):
level = 2
class BDBConnectionTests(
BerkeleyStorageConfig,
ConnectionTests.ConnectionTests,
InvalidationTests.InvalidationTests
):
"""Berkeley storage connection tests."""
level = 2
class BDBReconnectionTests(
BerkeleyStorageConfig,
ConnectionTests.ReconnectionTests
):
"""Berkeley storage re-connection tests."""
level = 2
class BDBInvqTests(
BerkeleyStorageConfig,
ConnectionTests.InvqTests
):
"""Berkeley storage invalidation queue tests."""
level = 2
class BDBTimeoutTests(
BerkeleyStorageConfig,
ConnectionTests.TimeoutTests
):
level = 2
class MappingStorageConnectionTests(
MappingStorageConfig,
ConnectionTests.ConnectionTests
):
"""Mapping storage connection tests."""
level = 1
# The ReconnectionTests can't work with MappingStorage because it's only an
# in-memory storage and has no persistent state.
class MappingStorageTimeoutTests(
MappingStorageConfig,
ConnectionTests.TimeoutTests
):
level = 1
test_classes = [FileStorageConnectionTests,
FileStorageReconnectionTests,
FileStorageInvqTests,
FileStorageTimeoutTests,
MappingStorageConnectionTests,
MappingStorageTimeoutTests]
def test_suite():
suite = unittest.TestSuite()
for klass in test_classes:
sub = unittest.makeSuite(klass, 'check')
suite.addTest(sub)
return suite
if __name__ == "__main__":
unittest.main(defaultTest='test_suite')
ZODB/src/ZEO/tests/testAuth.py 0000640 0001750 0000144 00000011437 10057366465 015745 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Test suite for AuthZEO."""
import os
import tempfile
import time
import unittest
from ZEO import zeopasswd
from ZEO.Exceptions import ClientDisconnected
from ZEO.tests.ConnectionTests import CommonSetupTearDown
class AuthTest(CommonSetupTearDown):
__super_getServerConfig = CommonSetupTearDown.getServerConfig
__super_setUp = CommonSetupTearDown.setUp
__super_tearDown = CommonSetupTearDown.tearDown
realm = None
def setUp(self):
self.pwfile = tempfile.mktemp()
if self.realm:
self.pwdb = self.dbclass(self.pwfile, self.realm)
else:
self.pwdb = self.dbclass(self.pwfile)
self.pwdb.add_user("foo", "bar")
self.pwdb.save()
self._checkZEOpasswd()
self.__super_setUp()
def _checkZEOpasswd(self):
args = ["-f", self.pwfile, "-p", self.protocol]
if self.protocol == "plaintext":
from ZEO.auth.base import Database
zeopasswd.main(args + ["-d", "foo"], Database)
zeopasswd.main(args + ["foo", "bar"], Database)
else:
zeopasswd.main(args + ["-d", "foo"])
zeopasswd.main(args + ["foo", "bar"])
def tearDown(self):
self.__super_tearDown()
os.remove(self.pwfile)
def getConfig(self, path, create, read_only):
return ""
def getServerConfig(self, addr, ro_svr):
zconf = self.__super_getServerConfig(addr, ro_svr)
zconf.authentication_protocol = self.protocol
zconf.authentication_database = self.pwfile
zconf.authentication_realm = self.realm
return zconf
def wait(self):
for i in range(25):
time.sleep(0.1)
if self._storage.test_connection:
return
self.fail("Timed out waiting for client to authenticate")
def testOK(self):
# Sleep for 0.2 seconds to give the server some time to start up
# seems to be needed before and after creating the storage
self._storage = self.openClientStorage(wait=0, username="foo",
password="bar", realm=self.realm)
self.wait()
self.assert_(self._storage._connection)
self._storage._connection.poll()
self.assert_(self._storage.is_connected())
# Make a call to make sure the mechanism is working
self._storage.versions()
def testNOK(self):
self._storage = self.openClientStorage(wait=0, username="foo",
password="noogie",
realm=self.realm)
self.wait()
# If the test established a connection, then it failed.
self.failIf(self._storage._connection)
def testUnauthenticatedMessage(self):
# Test that an unauthenticated message is rejected by the server
# if it was sent after the connection was authenticated.
# Sleep for 0.2 seconds to give the server some time to start up
# seems to be needed before and after creating the storage
self._storage = self.openClientStorage(wait=0, username="foo",
password="bar", realm=self.realm)
self.wait()
self._storage.versions()
# Manually clear the state of the hmac connection
self._storage._connection._SizedMessageAsyncConnection__hmac_send = None
# Once the client stops using the hmac, it should be disconnected.
self.assertRaises(ClientDisconnected, self._storage.versions)
class PlainTextAuth(AuthTest):
import ZEO.tests.auth_plaintext
protocol = "plaintext"
database = "authdb.sha"
dbclass = ZEO.tests.auth_plaintext.Database
realm = "Plaintext Realm"
class DigestAuth(AuthTest):
import ZEO.auth.auth_digest
protocol = "digest"
database = "authdb.digest"
dbclass = ZEO.auth.auth_digest.DigestDatabase
realm = "Digest Realm"
test_classes = [PlainTextAuth, DigestAuth]
def test_suite():
suite = unittest.TestSuite()
for klass in test_classes:
sub = unittest.makeSuite(klass)
suite.addTest(sub)
return suite
if __name__ == "__main__":
unittest.main(defaultTest='test_suite')
ZODB/src/ZEO/tests/deadlock.py 0000640 0001750 0000144 00000002141 10051215041 015653 0 ustar ctheune users import ZODB
from ZODB.POSException import ConflictError
from ZEO.ClientStorage import ClientStorage, ClientDisconnected
from ZEO.zrpc.error import DisconnectedError
import os
import random
import time
L = range(1, 100)
def main():
z1 = ClientStorage(('localhost', 2001), wait=1)
z2 = ClientStorage(('localhost', 2002), wait=2)
db1 = ZODB.DB(z1)
db2 = ZODB.DB(z2)
c1 = db1.open()
c2 = db2.open()
r1 = c1.root()
r2 = c2.root()
while 1:
try:
try:
update(r1, r2)
except ConflictError, msg:
print msg
transaction.abort()
c1.sync()
c2.sync()
except (ClientDisconnected, DisconnectedError), err:
print "disconnected", err
time.sleep(2)
def update(r1, r2):
k1 = random.choice(L)
k2 = random.choice(L)
updates = [(k1, r1),
(k2, r2)]
random.shuffle(updates)
for key, root in updates:
root[key] = time.time()
transaction.commit()
print os.getpid(), k1, k2
if __name__ == "__main__":
main()
ZODB/src/ZEO/tests/Cache.py 0000640 0001750 0000144 00000007735 10214413267 015141 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Tests of the ZEO cache"""
from ZODB.tests.MinPO import MinPO
from ZODB.tests.StorageTestBase import zodb_unpickle
from transaction import Transaction
class TransUndoStorageWithCache:
def checkUndoInvalidation(self):
oid = self._storage.new_oid()
revid = self._dostore(oid, data=MinPO(23))
revid = self._dostore(oid, revid=revid, data=MinPO(24))
revid = self._dostore(oid, revid=revid, data=MinPO(25))
info = self._storage.undoInfo()
if not info:
# Preserved this comment, but don't understand it:
# "Perhaps we have an old storage implementation that
# does do the negative nonsense."
info = self._storage.undoInfo(0, 20)
tid = info[0]['id']
# We may need to bail at this point if the storage doesn't
# support transactional undo
if not self._storage.supportsTransactionalUndo():
return
# Now start an undo transaction
t = Transaction()
t.note('undo1')
self._storage.tpc_begin(t)
tid, oids = self._storage.undo(tid, t)
# Make sure this doesn't load invalid data into the cache
self._storage.load(oid, '')
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
assert len(oids) == 1
assert oids[0] == oid
data, revid = self._storage.load(oid, '')
obj = zodb_unpickle(data)
assert obj == MinPO(24)
class StorageWithCache:
def checkAbortVersionInvalidation(self):
oid = self._storage.new_oid()
revid = self._dostore(oid, data=MinPO(1))
revid = self._dostore(oid, revid=revid, data=MinPO(2))
revid = self._dostore(oid, revid=revid, data=MinPO(3), version="foo")
revid = self._dostore(oid, revid=revid, data=MinPO(4), version="foo")
t = Transaction()
self._storage.tpc_begin(t)
self._storage.abortVersion("foo", t)
self._storage.load(oid, "foo")
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
data, revid = self._storage.load(oid, "foo")
obj = zodb_unpickle(data)
assert obj == MinPO(2), obj
def checkCommitEmptyVersionInvalidation(self):
oid = self._storage.new_oid()
revid = self._dostore(oid, data=MinPO(1))
revid = self._dostore(oid, revid=revid, data=MinPO(2))
revid = self._dostore(oid, revid=revid, data=MinPO(3), version="foo")
t = Transaction()
self._storage.tpc_begin(t)
self._storage.commitVersion("foo", "", t)
self._storage.load(oid, "")
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
data, revid = self._storage.load(oid, "")
obj = zodb_unpickle(data)
assert obj == MinPO(3), obj
def checkCommitVersionInvalidation(self):
oid = self._storage.new_oid()
revid = self._dostore(oid, data=MinPO(1))
revid = self._dostore(oid, revid=revid, data=MinPO(2))
revid = self._dostore(oid, revid=revid, data=MinPO(3), version="foo")
t = Transaction()
self._storage.tpc_begin(t)
self._storage.commitVersion("foo", "bar", t)
self._storage.load(oid, "")
self._storage.tpc_vote(t)
self._storage.tpc_finish(t)
data, revid = self._storage.load(oid, "bar")
obj = zodb_unpickle(data)
assert obj == MinPO(3), obj
ZODB/src/ZEO/tests/InvalidationTests.py 0000640 0001750 0000144 00000045703 10245661164 017604 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import threading
import time
from random import Random
import transaction
from BTrees.check import check, display
from BTrees.OOBTree import OOBTree
from ZEO.tests.TestThread import TestThread
from ZODB.DB import DB
from ZODB.POSException \
import ReadConflictError, ConflictError, VersionLockError
# The tests here let several threads have a go at one or more database
# instances simultaneously. Each thread appends a disjoint (from the
# other threads) sequence of increasing integers to an OOBTree, one at
# at time (per thread). This provokes lots of conflicts, and BTrees
# work hard at conflict resolution too. An OOBTree is used because
# that flavor has the smallest maximum bucket size, and so splits buckets
# more often than other BTree flavors.
#
# When these tests were first written, they provoked an amazing number
# of obscure timing-related bugs in cache consistency logic, revealed
# by failure of the BTree to pass internal consistency checks at the end,
# and/or by failure of the BTree to contain all the keys the threads
# thought they added (i.e., the keys for which transaction.commit()
# did not raise any exception).
class FailableThread(TestThread):
# mixin class
# subclass must provide
# - self.stop attribute (an event)
# - self._testrun() method
# TestThread.run() invokes testrun().
def testrun(self):
try:
self._testrun()
except:
# Report the failure here to all the other threads, so
# that they stop quickly.
self.stop.set()
raise
class StressTask:
# Append integers startnum, startnum + step, startnum + 2*step, ...
# to 'tree'. If sleep is given, sleep
# that long after each append. At the end, instance var .added_keys
# is a list of the ints the thread believes it added successfully.
def __init__(self, db, threadnum, startnum, step=2, sleep=None):
self.db = db
self.threadnum = threadnum
self.startnum = startnum
self.step = step
self.sleep = sleep
self.added_keys = []
self.tm = transaction.TransactionManager()
self.cn = self.db.open(transaction_manager=self.tm)
self.cn.sync()
def doStep(self):
tree = self.cn.root()["tree"]
key = self.startnum
tree[key] = self.threadnum
def commit(self):
cn = self.cn
key = self.startnum
self.tm.get().note("add key %s" % key)
try:
self.tm.get().commit()
except ConflictError, msg:
self.tm.abort()
else:
if self.sleep:
time.sleep(self.sleep)
self.added_keys.append(key)
self.startnum += self.step
def cleanup(self):
self.tm.get().abort()
self.cn.close()
def _runTasks(rounds, *tasks):
'''run *task* interleaved for *rounds* rounds.'''
def commit(run, actions):
actions.append(':')
for t in run:
t.commit()
del run[:]
r = Random()
r.seed(1064589285) # make it deterministic
run = []
actions = []
try:
for i in range(rounds):
t = r.choice(tasks)
if t in run:
commit(run, actions)
run.append(t)
t.doStep()
actions.append(`t.startnum`)
commit(run,actions)
# stderr.write(' '.join(actions)+'\n')
finally:
for t in tasks:
t.cleanup()
class StressThread(FailableThread):
# Append integers startnum, startnum + step, startnum + 2*step, ...
# to 'tree' until Event stop is set. If sleep is given, sleep
# that long after each append. At the end, instance var .added_keys
# is a list of the ints the thread believes it added successfully.
def __init__(self, db, stop, threadnum, commitdict,
startnum, step=2, sleep=None):
TestThread.__init__(self)
self.db = db
self.stop = stop
self.threadnum = threadnum
self.startnum = startnum
self.step = step
self.sleep = sleep
self.added_keys = []
self.commitdict = commitdict
def _testrun(self):
cn = self.db.open()
while not self.stop.isSet():
try:
tree = cn.root()["tree"]
break
except (ConflictError, KeyError):
transaction.abort()
key = self.startnum
while not self.stop.isSet():
try:
tree[key] = self.threadnum
transaction.get().note("add key %s" % key)
transaction.commit()
self.commitdict[self] = 1
if self.sleep:
time.sleep(self.sleep)
except (ReadConflictError, ConflictError), msg:
transaction.abort()
else:
self.added_keys.append(key)
key += self.step
cn.close()
class LargeUpdatesThread(FailableThread):
# A thread that performs a lot of updates. It attempts to modify
# more than 25 objects so that it can test code that runs vote
# in a separate thread when it modifies more than 25 objects.
def __init__(self, db, stop, threadnum, commitdict, startnum,
step=2, sleep=None):
TestThread.__init__(self)
self.db = db
self.stop = stop
self.threadnum = threadnum
self.startnum = startnum
self.step = step
self.sleep = sleep
self.added_keys = []
self.commitdict = commitdict
def _testrun(self):
cn = self.db.open()
while not self.stop.isSet():
try:
tree = cn.root()["tree"]
break
except (ConflictError, KeyError):
# print "%d getting tree abort" % self.threadnum
transaction.abort()
keys_added = {} # set of keys we commit
tkeys = []
while not self.stop.isSet():
# The test picks 50 keys spread across many buckets.
# self.startnum and self.step ensure that all threads use
# disjoint key sets, to minimize conflict errors.
nkeys = len(tkeys)
if nkeys < 50:
tkeys = range(self.startnum, 3000, self.step)
nkeys = len(tkeys)
step = max(int(nkeys / 50), 1)
keys = [tkeys[i] for i in range(0, nkeys, step)]
for key in keys:
try:
tree[key] = self.threadnum
except (ReadConflictError, ConflictError), msg:
# print "%d setting key %s" % (self.threadnum, msg)
transaction.abort()
break
else:
# print "%d set #%d" % (self.threadnum, len(keys))
transaction.get().note("keys %s" % ", ".join(map(str, keys)))
try:
transaction.commit()
self.commitdict[self] = 1
if self.sleep:
time.sleep(self.sleep)
except ConflictError, msg:
# print "%d commit %s" % (self.threadnum, msg)
transaction.abort()
continue
for k in keys:
tkeys.remove(k)
keys_added[k] = 1
self.added_keys = keys_added.keys()
cn.close()
class VersionStressThread(FailableThread):
def __init__(self, db, stop, threadnum, commitdict, startnum,
step=2, sleep=None):
TestThread.__init__(self)
self.db = db
self.stop = stop
self.threadnum = threadnum
self.startnum = startnum
self.step = step
self.sleep = sleep
self.added_keys = []
self.commitdict = commitdict
def _testrun(self):
commit = 0
key = self.startnum
while not self.stop.isSet():
version = "%s:%s" % (self.threadnum, key)
commit = not commit
if self.oneupdate(version, key, commit):
self.added_keys.append(key)
self.commitdict[self] = 1
key += self.step
def oneupdate(self, version, key, commit=1):
# The mess of sleeps below were added to reduce the number
# of VersionLockErrors, based on empirical observation.
# It looks like the threads don't switch enough without
# the sleeps.
cn = self.db.open(version)
while not self.stop.isSet():
try:
tree = cn.root()["tree"]
break
except (ConflictError, KeyError):
transaction.abort()
while not self.stop.isSet():
try:
tree[key] = self.threadnum
transaction.commit()
if self.sleep:
time.sleep(self.sleep)
break
except (VersionLockError, ReadConflictError, ConflictError), msg:
transaction.abort()
if self.sleep:
time.sleep(self.sleep)
try:
while not self.stop.isSet():
try:
if commit:
self.db.commitVersion(version)
transaction.get().note("commit version %s" % version)
else:
self.db.abortVersion(version)
transaction.get().note("abort version %s" % version)
transaction.commit()
if self.sleep:
time.sleep(self.sleep)
return commit
except ConflictError, msg:
transaction.abort()
finally:
cn.close()
return 0
class InvalidationTests:
level = 2
# Minimum # of seconds the main thread lets the workers run. The
# test stops as soon as this much time has elapsed, and all threads
# have managed to commit a change.
MINTIME = 10
# Maximum # of seconds the main thread lets the workers run. We
# stop after this long has elapsed regardless of whether all threads
# have managed to commit a change.
MAXTIME = 300
StressThread = StressThread
def _check_tree(self, cn, tree):
# Make sure the BTree is sane at the C level.
retries = 3
while retries:
retries -= 1
try:
check(tree)
tree._check()
except ReadConflictError:
if retries:
transaction.abort()
else:
raise
except:
display(tree)
raise
def _check_threads(self, tree, *threads):
# Make sure the thread's view of the world is consistent with
# the actual database state.
expected_keys = []
errormsgs = []
err = errormsgs.append
for t in threads:
if not t.added_keys:
err("thread %d didn't add any keys" % t.threadnum)
expected_keys.extend(t.added_keys)
expected_keys.sort()
actual_keys = list(tree.keys())
if expected_keys != actual_keys:
err("expected keys != actual keys")
for k in expected_keys:
if k not in actual_keys:
err("key %s expected but not in tree" % k)
for k in actual_keys:
if k not in expected_keys:
err("key %s in tree but not expected" % k)
if errormsgs:
display(tree)
self.fail('\n'.join(errormsgs))
def go(self, stop, commitdict, *threads):
# Run the threads
for t in threads:
t.start()
delay = self.MINTIME
start = time.time()
while time.time() - start <= self.MAXTIME:
stop.wait(delay)
if stop.isSet():
# Some thread failed. Stop right now.
break
delay = 2.0
if len(commitdict) >= len(threads):
break
# Some thread still hasn't managed to commit anything.
stop.set()
# Give all the threads some time to stop before trying to clean up.
# cleanup() will cause the test to fail if some thread ended with
# an uncaught exception, and unittest will call the base class
# tearDown then immediately, but if other threads are still
# running that can lead to a cascade of spurious exceptions.
for t in threads:
t.join(10)
for t in threads:
t.cleanup()
def checkConcurrentUpdates2Storages_emulated(self):
self._storage = storage1 = self.openClientStorage()
storage2 = self.openClientStorage()
db1 = DB(storage1)
db2 = DB(storage2)
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
transaction.commit()
# DM: allow time for invalidations to come in and process them
time.sleep(0.1)
# Run two threads that update the BTree
t1 = StressTask(db1, 1, 1,)
t2 = StressTask(db2, 2, 2,)
_runTasks(100, t1, t2)
cn.sync()
self._check_tree(cn, tree)
self._check_threads(tree, t1, t2)
cn.close()
db1.close()
db2.close()
def checkConcurrentUpdates2Storages(self):
self._storage = storage1 = self.openClientStorage()
storage2 = self.openClientStorage()
db1 = DB(storage1)
db2 = DB(storage2)
stop = threading.Event()
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
transaction.commit()
cn.close()
# Run two threads that update the BTree
cd = {}
t1 = self.StressThread(db1, stop, 1, cd, 1)
t2 = self.StressThread(db2, stop, 2, cd, 2)
self.go(stop, cd, t1, t2)
while db1.lastTransaction() != db2.lastTransaction():
db1._storage.sync()
db2._storage.sync()
cn = db1.open()
tree = cn.root()["tree"]
self._check_tree(cn, tree)
self._check_threads(tree, t1, t2)
cn.close()
db1.close()
db2.close()
def checkConcurrentUpdates1Storage(self):
self._storage = storage1 = self.openClientStorage()
db1 = DB(storage1)
stop = threading.Event()
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
transaction.commit()
cn.close()
# Run two threads that update the BTree
cd = {}
t1 = self.StressThread(db1, stop, 1, cd, 1, sleep=0.01)
t2 = self.StressThread(db1, stop, 2, cd, 2, sleep=0.01)
self.go(stop, cd, t1, t2)
cn = db1.open()
tree = cn.root()["tree"]
self._check_tree(cn, tree)
self._check_threads(tree, t1, t2)
cn.close()
db1.close()
def checkConcurrentUpdates2StoragesMT(self):
self._storage = storage1 = self.openClientStorage()
db1 = DB(storage1)
db2 = DB(self.openClientStorage())
stop = threading.Event()
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
transaction.commit()
cn.close()
# Run three threads that update the BTree.
# Two of the threads share a single storage so that it
# is possible for both threads to read the same object
# at the same time.
cd = {}
t1 = self.StressThread(db1, stop, 1, cd, 1, 3)
t2 = self.StressThread(db2, stop, 2, cd, 2, 3, 0.01)
t3 = self.StressThread(db2, stop, 3, cd, 3, 3, 0.01)
self.go(stop, cd, t1, t2, t3)
while db1.lastTransaction() != db2.lastTransaction():
db1._storage.sync()
db2._storage.sync()
cn = db1.open()
tree = cn.root()["tree"]
self._check_tree(cn, tree)
self._check_threads(tree, t1, t2, t3)
cn.close()
db1.close()
db2.close()
# TODO: Temporarily disabled. I know it fails, and there's no point
# getting an endless number of reports about that.
def xxxcheckConcurrentUpdatesInVersions(self):
self._storage = storage1 = self.openClientStorage()
db1 = DB(storage1)
db2 = DB(self.openClientStorage())
stop = threading.Event()
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
transaction.commit()
cn.close()
# Run three threads that update the BTree.
# Two of the threads share a single storage so that it
# is possible for both threads to read the same object
# at the same time.
cd = {}
t1 = VersionStressThread(db1, stop, 1, cd, 1, 3)
t2 = VersionStressThread(db2, stop, 2, cd, 2, 3, 0.01)
t3 = VersionStressThread(db2, stop, 3, cd, 3, 3, 0.01)
self.go(stop, cd, t1, t2, t3)
while db1.lastTransaction() != db2.lastTransaction():
db1._storage.sync()
db2._storage.sync()
cn = db1.open()
tree = cn.root()["tree"]
self._check_tree(cn, tree)
self._check_threads(tree, t1, t2, t3)
cn.close()
db1.close()
db2.close()
def checkConcurrentLargeUpdates(self):
# Use 3 threads like the 2StorageMT test above.
self._storage = storage1 = self.openClientStorage()
db1 = DB(storage1)
db2 = DB(self.openClientStorage())
stop = threading.Event()
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
for i in range(0, 3000, 2):
tree[i] = 0
transaction.commit()
cn.close()
# Run three threads that update the BTree.
# Two of the threads share a single storage so that it
# is possible for both threads to read the same object
# at the same time.
cd = {}
t1 = LargeUpdatesThread(db1, stop, 1, cd, 1, 3, 0.02)
t2 = LargeUpdatesThread(db2, stop, 2, cd, 2, 3, 0.01)
t3 = LargeUpdatesThread(db2, stop, 3, cd, 3, 3, 0.01)
self.go(stop, cd, t1, t2, t3)
while db1.lastTransaction() != db2.lastTransaction():
db1._storage.sync()
db2._storage.sync()
cn = db1.open()
tree = cn.root()["tree"]
self._check_tree(cn, tree)
# Purge the tree of the dummy entries mapping to 0.
losers = [k for k, v in tree.items() if v == 0]
for k in losers:
del tree[k]
transaction.commit()
self._check_threads(tree, t1, t2, t3)
cn.close()
db1.close()
db2.close()
ZODB/src/ZEO/tests/TestThread.py 0000640 0001750 0000144 00000004215 10106276332 016173 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""A Thread base class for use with unittest."""
import threading
import sys
class TestThread(threading.Thread):
"""Base class for defining threads that run from unittest.
The subclass should define a testrun() method instead of a run()
method.
Call cleanup() when the test is done with the thread, instead of join().
If the thread exits with an uncaught exception, it's captured and
re-raised when cleanup() is called. cleanup() should be called by
the main thread! Trying to tell unittest that a test failed from
another thread creates a nightmare of timing-depending cascading
failures and missed errors (tracebacks that show up on the screen,
but don't cause unittest to believe the test failed).
cleanup() also joins the thread. If the thread ended without raising
an uncaught exception, and the join doesn't succeed in the timeout
period, then the test is made to fail with a "Thread still alive"
message.
"""
def __init__(self):
threading.Thread.__init__(self)
# In case this thread hangs, don't stop Python from exiting.
self.setDaemon(1)
self._exc_info = None
def run(self):
try:
self.testrun()
except:
self._exc_info = sys.exc_info()
def cleanup(self, timeout=15):
self.join(timeout)
if self._exc_info:
raise self._exc_info[0], self._exc_info[1], self._exc_info[2]
if self.isAlive():
self._testcase.fail("Thread did not finish: %s" % self)
ZODB/src/ZEO/tests/testZEOOptions.py 0000640 0001750 0000144 00000007300 10243233142 017025 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Test suite for ZEO.runzeo.ZEOOptions."""
import os
import sys
import tempfile
import unittest
import ZODB.config
from ZEO.runzeo import ZEOOptions
from zdaemon.tests.testzdoptions import TestZDOptions
# When a hostname isn't specified in a socket binding address, ZConfig
# supplies the empty string.
DEFAULT_BINDING_HOST = ""
class TestZEOOptions(TestZDOptions):
OptionsClass = ZEOOptions
input_args = ["-f", "Data.fs", "-a", "5555"]
output_opts = [("-f", "Data.fs"), ("-a", "5555")]
output_args = []
configdata = """
address 5555
path Data.fs
"""
def setUp(self):
self.tempfilename = tempfile.mktemp()
f = open(self.tempfilename, "w")
f.write(self.configdata)
f.close()
def tearDown(self):
try:
os.remove(self.tempfilename)
except os.error:
pass
def test_configure(self):
# Hide the base class test_configure
pass
def test_defaults_with_schema(self):
options = self.OptionsClass()
options.realize(["-C", self.tempfilename])
self.assertEqual(options.address, (DEFAULT_BINDING_HOST, 5555))
self.assertEqual(len(options.storages), 1)
opener = options.storages[0]
self.assertEqual(opener.name, "fs")
self.assertEqual(opener.__class__, ZODB.config.FileStorage)
self.assertEqual(options.read_only, 0)
self.assertEqual(options.transaction_timeout, None)
self.assertEqual(options.invalidation_queue_size, 100)
def test_defaults_without_schema(self):
options = self.OptionsClass()
options.realize(["-a", "5555", "-f", "Data.fs"])
self.assertEqual(options.address, (DEFAULT_BINDING_HOST, 5555))
self.assertEqual(len(options.storages), 1)
opener = options.storages[0]
self.assertEqual(opener.name, "1")
self.assertEqual(opener.__class__, ZODB.config.FileStorage)
self.assertEqual(opener.config.path, "Data.fs")
self.assertEqual(options.read_only, 0)
self.assertEqual(options.transaction_timeout, None)
self.assertEqual(options.invalidation_queue_size, 100)
def test_commandline_overrides(self):
options = self.OptionsClass()
options.realize(["-C", self.tempfilename,
"-a", "6666", "-f", "Wisdom.fs"])
self.assertEqual(options.address, (DEFAULT_BINDING_HOST, 6666))
self.assertEqual(len(options.storages), 1)
opener = options.storages[0]
self.assertEqual(opener.__class__, ZODB.config.FileStorage)
self.assertEqual(opener.config.path, "Wisdom.fs")
self.assertEqual(options.read_only, 0)
self.assertEqual(options.transaction_timeout, None)
self.assertEqual(options.invalidation_queue_size, 100)
def test_suite():
suite = unittest.TestSuite()
for cls in [TestZEOOptions]:
suite.addTest(unittest.makeSuite(cls))
return suite
if __name__ == "__main__":
unittest.main(defaultTest='test_suite')
ZODB/src/ZEO/tests/ConnectionTests.py 0000640 0001750 0000144 00000124222 10251136132 017241 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import os
import sys
import time
import random
import asyncore
import tempfile
import threading
import logging
import ZEO.ServerStub
from ZEO.ClientStorage import ClientStorage
from ZEO.Exceptions import ClientDisconnected
from ZEO.zrpc.marshal import Marshaller
from ZEO.tests import forker
from ZODB.DB import DB
from ZODB.POSException import ReadOnlyError, ConflictError
from ZODB.tests.StorageTestBase import StorageTestBase
from ZODB.tests.MinPO import MinPO
from ZODB.tests.StorageTestBase \
import zodb_pickle, zodb_unpickle, handle_all_serials, handle_serials
import transaction
from transaction import Transaction
logger = logging.getLogger('ZEO.tests.ConnectionTests')
ZERO = '\0'*8
class TestServerStub(ZEO.ServerStub.StorageServer):
__super_getInvalidations = ZEO.ServerStub.StorageServer.getInvalidations
def getInvalidations(self, tid):
# squirrel the results away for inspection by test case
self._last_invals = self.__super_getInvalidations(tid)
return self._last_invals
class TestClientStorage(ClientStorage):
test_connection = False
StorageServerStubClass = TestServerStub
def verify_cache(self, stub):
self.end_verify = threading.Event()
self.verify_result = ClientStorage.verify_cache(self, stub)
def endVerify(self):
ClientStorage.endVerify(self)
self.end_verify.set()
def testConnection(self, conn):
try:
return ClientStorage.testConnection(self, conn)
finally:
self.test_connection = True
class DummyDB:
def invalidate(self, *args, **kwargs):
pass
class CommonSetupTearDown(StorageTestBase):
"""Common boilerplate"""
__super_setUp = StorageTestBase.setUp
__super_tearDown = StorageTestBase.tearDown
keep = 0
invq = None
timeout = None
monitor = 0
db_class = DummyDB
def setUp(self):
"""Test setup for connection tests.
This starts only one server; a test may start more servers by
calling self._newAddr() and then self.startServer(index=i)
for i in 1, 2, ...
"""
self.__super_setUp()
logging.info("setUp() %s", self.id())
self.file = tempfile.mktemp()
self.addr = []
self._pids = []
self._servers = []
self.conf_paths = []
self.caches = []
self._newAddr()
self.startServer()
def tearDown(self):
"""Try to cause the tests to halt"""
logging.info("tearDown() %s" % self.id())
for p in self.conf_paths:
os.remove(p)
if getattr(self, '_storage', None) is not None:
self._storage.close()
if hasattr(self._storage, 'cleanup'):
logging.debug("cleanup storage %s" %
self._storage.__name__)
self._storage.cleanup()
for adminaddr in self._servers:
if adminaddr is not None:
forker.shutdown_zeo_server(adminaddr)
if hasattr(os, 'waitpid'):
# Not in Windows Python until 2.3
for pid in self._pids:
os.waitpid(pid, 0)
for c in self.caches:
for i in 0, 1:
for ext in "", ".trace":
base = "%s-%s.zec%s" % (c, "1", ext)
path = os.path.join(tempfile.tempdir, base)
# On Windows before 2.3, we don't have a way to wait for
# the spawned server(s) to close, and they inherited
# file descriptors for our open files. So long as those
# processes are alive, we can't delete the files. Try
# a few times then give up.
need_to_delete = False
if os.path.exists(path):
need_to_delete = True
for dummy in range(5):
try:
os.unlink(path)
except:
time.sleep(0.5)
else:
need_to_delete = False
break
if need_to_delete:
os.unlink(path) # sometimes this is just gonna fail
self.__super_tearDown()
def _newAddr(self):
self.addr.append(self._getAddr())
def _getAddr(self):
# port+1 is also used, so only draw even port numbers
return 'localhost', random.randrange(25000, 30000, 2)
def getConfig(self, path, create, read_only):
raise NotImplementedError
cache_id = 1
def openClientStorage(self, cache=None, cache_size=200000, wait=1,
read_only=0, read_only_fallback=0,
username=None, password=None, realm=None):
if cache is None:
cache = str(self.__class__.cache_id)
self.__class__.cache_id += 1
self.caches.append(cache)
storage = TestClientStorage(self.addr,
client=cache,
var=tempfile.tempdir,
cache_size=cache_size,
wait=wait,
min_disconnect_poll=0.1,
read_only=read_only,
read_only_fallback=read_only_fallback,
username=username,
password=password,
realm=realm)
storage.registerDB(DummyDB(), None)
return storage
def getServerConfig(self, addr, ro_svr):
zconf = forker.ZEOConfig(addr)
if ro_svr:
zconf.read_only = 1
if self.monitor:
zconf.monitor_address = ("", 42000)
if self.invq:
zconf.invalidation_queue_size = self.invq
if self.timeout:
zconf.transaction_timeout = self.timeout
return zconf
def startServer(self, create=1, index=0, read_only=0, ro_svr=0, keep=None):
addr = self.addr[index]
logging.info("startServer(create=%d, index=%d, read_only=%d) @ %s" %
(create, index, read_only, addr))
path = "%s.%d" % (self.file, index)
sconf = self.getConfig(path, create, read_only)
zconf = self.getServerConfig(addr, ro_svr)
if keep is None:
keep = self.keep
zeoport, adminaddr, pid, path = forker.start_zeo_server(
sconf, zconf, addr[1], keep)
self.conf_paths.append(path)
self._pids.append(pid)
self._servers.append(adminaddr)
def shutdownServer(self, index=0):
logging.info("shutdownServer(index=%d) @ %s" %
(index, self._servers[index]))
adminaddr = self._servers[index]
if adminaddr is not None:
forker.shutdown_zeo_server(adminaddr)
self._servers[index] = None
def pollUp(self, timeout=30.0, storage=None):
if storage is None:
storage = self._storage
# Poll until we're connected.
now = time.time()
giveup = now + timeout
while not storage.is_connected():
asyncore.poll(0.1)
now = time.time()
if now > giveup:
self.fail("timed out waiting for storage to connect")
# When the socket map is empty, poll() returns immediately,
# and this is a pure busy-loop then. At least on some Linux
# flavors, that can starve the thread trying to connect,
# leading to grossly increased runtime (typical) or bogus
# "timed out" failures. A little sleep here cures both.
time.sleep(0.1)
def pollDown(self, timeout=30.0):
# Poll until we're disconnected.
now = time.time()
giveup = now + timeout
while self._storage.is_connected():
asyncore.poll(0.1)
now = time.time()
if now > giveup:
self.fail("timed out waiting for storage to disconnect")
# See pollUp() for why we sleep a little here.
time.sleep(0.1)
class ConnectionTests(CommonSetupTearDown):
"""Tests that explicitly manage the server process.
To test the cache or re-connection, these test cases explicit
start and stop a ZEO storage server.
"""
def checkMultipleAddresses(self):
for i in range(4):
self._newAddr()
self._storage = self.openClientStorage('test', 100000)
oid = self._storage.new_oid()
obj = MinPO(12)
self._dostore(oid, data=obj)
self._storage.close()
def checkMultipleServers(self):
# Crude test-- just start two servers and do a commit at each one.
self._newAddr()
self._storage = self.openClientStorage('test', 100000)
self._dostore()
self.shutdownServer(index=0)
self.startServer(index=1)
# If we can still store after shutting down one of the
# servers, we must be reconnecting to the other server.
did_a_store = 0
for i in range(10):
try:
self._dostore()
did_a_store = 1
break
except ClientDisconnected:
time.sleep(0.5)
self.assert_(did_a_store)
self._storage.close()
def checkReadOnlyClient(self):
# Open a read-only client to a read-write server; stores fail
# Start a read-only client for a read-write server
self._storage = self.openClientStorage(read_only=1)
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
self._storage.close()
def checkReadOnlyServer(self):
# Open a read-only client to a read-only *server*; stores fail
# We don't want the read-write server created by setUp()
self.shutdownServer()
self._servers = []
# Start a read-only server
self.startServer(create=0, index=0, ro_svr=1)
# Start a read-only client
self._storage = self.openClientStorage(read_only=1)
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
self._storage.close()
# Get rid of the 'test left new threads behind' warning
time.sleep(0.1)
def checkReadOnlyFallbackWritable(self):
# Open a fallback client to a read-write server; stores succeed
# Start a read-only-fallback client for a read-write server
self._storage = self.openClientStorage(read_only_fallback=1)
# Stores should succeed here
self._dostore()
self._storage.close()
def checkReadOnlyFallbackReadOnlyServer(self):
# Open a fallback client to a read-only *server*; stores fail
# We don't want the read-write server created by setUp()
self.shutdownServer()
self._servers = []
# Start a read-only server
self.startServer(create=0, index=0, ro_svr=1)
# Start a read-only-fallback client
self._storage = self.openClientStorage(read_only_fallback=1)
self.assert_(self._storage.isReadOnly())
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
self._storage.close()
# TODO: Compare checkReconnectXXX() here to checkReconnection()
# further down. Is the code here hopelessly naive, or is
# checkReconnection() overwrought?
def checkReconnectWritable(self):
# A read-write client reconnects to a read-write server
# Start a client
self._storage = self.openClientStorage()
# Stores should succeed here
self._dostore()
# Shut down the server
self.shutdownServer()
self._servers = []
# Poll until the client disconnects
self.pollDown()
# Stores should fail now
self.assertRaises(ClientDisconnected, self._dostore)
# Restart the server
self.startServer(create=0)
# Poll until the client connects
self.pollUp()
# Stores should succeed here
self._dostore()
self._storage.close()
def checkDisconnectionError(self):
# Make sure we get a ClientDisconnected when we try to read an
# object when we're not connected to a storage server and the
# object is not in the cache.
self.shutdownServer()
self._storage = self.openClientStorage('test', 1000, wait=0)
self.assertRaises(ClientDisconnected,
self._storage.load, 'fredwash', '')
self._storage.close()
def checkDisconnectedAbort(self):
self._storage = self.openClientStorage()
self._dostore()
oids = [self._storage.new_oid() for i in range(5)]
txn = Transaction()
self._storage.tpc_begin(txn)
for oid in oids:
data = zodb_pickle(MinPO(oid))
self._storage.store(oid, None, data, '', txn)
self.shutdownServer()
self.assertRaises(ClientDisconnected, self._storage.tpc_vote, txn)
self._storage.tpc_abort(txn)
self.startServer(create=0)
self._storage._wait()
self._dostore()
# This test is supposed to cover the following error, although
# I don't have much confidence that it does. The likely
# explanation for the error is that the _tbuf contained
# objects that weren't in the _seriald, because the client was
# interrupted waiting for tpc_vote() to return. When the next
# transaction committed, it tried to do something with the
# bogus _tbuf entries. The explanation is wrong/incomplete,
# because tpc_begin() should clear the _tbuf.
# 2003-01-15T15:44:19 ERROR(200) ZODB A storage error occurred
# in the last phase of a two-phase commit. This shouldn't happen.
# Traceback (innermost last):
# Module ZODB.Transaction, line 359, in _finish_one
# Module ZODB.Connection, line 691, in tpc_finish
# Module ZEO.ClientStorage, line 679, in tpc_finish
# Module ZEO.ClientStorage, line 709, in _update_cache
# KeyError: ...
def checkBasicPersistence(self):
# Verify cached data persists across client storage instances.
# To verify that the cache is being used, the test closes the
# server and then starts a new client with the server down.
# When the server is down, a load() gets the data from its cache.
self._storage = self.openClientStorage('test', 100000)
oid = self._storage.new_oid()
obj = MinPO(12)
revid1 = self._dostore(oid, data=obj)
self._storage.close()
self.shutdownServer()
self._storage = self.openClientStorage('test', 100000, wait=0)
data, revid2 = self._storage.load(oid, '')
self.assertEqual(zodb_unpickle(data), MinPO(12))
self.assertEqual(revid1, revid2)
self._storage.close()
def checkDisconnectedCacheWorks(self):
# Check that the cache works when the client is disconnected.
self._storage = self.openClientStorage('test')
oid1 = self._storage.new_oid()
obj1 = MinPO("1" * 500)
self._dostore(oid1, data=obj1)
oid2 = self._storage.new_oid()
obj2 = MinPO("2" * 500)
self._dostore(oid2, data=obj2)
expected1 = self._storage.load(oid1, '')
expected2 = self._storage.load(oid2, '')
# Shut it all down, and try loading from the persistent cache file
# without a server present.
self._storage.close()
self.shutdownServer()
self._storage = self.openClientStorage('test', wait=False)
self.assertEqual(expected1, self._storage.load(oid1, ''))
self.assertEqual(expected2, self._storage.load(oid2, ''))
self._storage.close()
def checkDisconnectedCacheFails(self):
# Like checkDisconnectedCacheWorks above, except the cache
# file is so small that only one object can be remembered.
self._storage = self.openClientStorage('test', cache_size=900)
oid1 = self._storage.new_oid()
obj1 = MinPO("1" * 500)
self._dostore(oid1, data=obj1)
oid2 = self._storage.new_oid()
obj2 = MinPO("2" * 500)
# The cache file is so small that adding oid2 will evict oid1.
self._dostore(oid2, data=obj2)
expected2 = self._storage.load(oid2, '')
# Shut it all down, and try loading from the persistent cache file
# without a server present.
self._storage.close()
self.shutdownServer()
self._storage = self.openClientStorage('test', cache_size=900,
wait=False)
# oid2 should still be in cache.
self.assertEqual(expected2, self._storage.load(oid2, ''))
# But oid1 should have been purged, so that trying to load it will
# try to fetch it from the (non-existent) ZEO server.
self.assertRaises(ClientDisconnected, self._storage.load, oid1, '')
self._storage.close()
def checkVerificationInvalidationPersists(self):
# This tests a subtle invalidation bug from ZODB 3.3:
# invalidations processed as part of ZEO cache verification acted
# kinda OK wrt the in-memory cache structures, but had no effect
# on the cache file. So opening the file cache again could
# incorrectly believe that a previously invalidated object was
# still current. This takes some effort to set up.
# First, using a persistent cache ('test'), create an object
# MinPO(13). We used to see this again at the end of this test,
# despite that we modify it, and despite that it gets invalidated
# in 'test', before the end.
self._storage = self.openClientStorage('test')
oid = self._storage.new_oid()
obj = MinPO(13)
self._dostore(oid, data=obj)
self._storage.close()
# Now modify obj via a temp connection. `test` won't learn about
# this until we open a connection using `test` again.
self._storage = self.openClientStorage()
pickle, rev = self._storage.load(oid, '')
newobj = zodb_unpickle(pickle)
self.assertEqual(newobj, obj)
newobj.value = 42 # .value *should* be 42 forever after now, not 13
self._dostore(oid, data=newobj, revid=rev)
self._storage.close()
# Open 'test' again. `oid` in this cache should be (and is)
# invalidated during cache verification. The bug was that it
# got invalidated (kinda) in memory, but not in the cache file.
self._storage = self.openClientStorage('test')
# The invalidation happened already. Now create and store a new
# object before closing this storage: this is so `test` believes
# it's seen transactions beyond the one that invalidated `oid`, so
# that the *next* time we open `test` it doesn't process another
# invalidation for `oid`. It's also important that we not try to
# load `oid` now: because it's been (kinda) invalidated in the
# cache's memory structures, loading it now would fetch the
# current revision from the server, thus hiding the bug.
obj2 = MinPO(666)
oid2 = self._storage.new_oid()
self._dostore(oid2, data=obj2)
self._storage.close()
# Finally, open `test` again and load `oid`. `test` believes
# it's beyond the transaction that modified `oid`, so its view
# of whether it has an up-to-date `oid` comes solely from the disk
# file, unaffected by cache verification.
self._storage = self.openClientStorage('test')
pickle, rev = self._storage.load(oid, '')
newobj_copy = zodb_unpickle(pickle)
# This used to fail, with
# AssertionError: MinPO(13) != MinPO(42)
# That is, `test` retained a stale revision of the object on disk.
self.assertEqual(newobj_copy, newobj)
self._storage.close()
def checkReconnection(self):
# Check that the client reconnects when a server restarts.
self._storage = self.openClientStorage()
oid = self._storage.new_oid()
obj = MinPO(12)
self._dostore(oid, data=obj)
logging.info("checkReconnection(): About to shutdown server")
self.shutdownServer()
logging.info("checkReconnection(): About to restart server")
self.startServer(create=0)
oid = self._storage.new_oid()
obj = MinPO(12)
while 1:
try:
self._dostore(oid, data=obj)
break
except ClientDisconnected:
# Maybe the exception mess is better now
logging.info("checkReconnection(): Error after"
" server restart; retrying.", exc_info=True)
transaction.abort()
# Give the other thread a chance to run.
time.sleep(0.1)
logging.info("checkReconnection(): finished")
self._storage.close()
def checkBadMessage1(self):
# not even close to a real message
self._bad_message("salty")
def checkBadMessage2(self):
# just like a real message, but with an unpicklable argument
global Hack
class Hack:
pass
msg = Marshaller().encode(1, 0, "foo", (Hack(),))
self._bad_message(msg)
del Hack
def _bad_message(self, msg):
# Establish a connection, then send the server an ill-formatted
# request. Verify that the connection is closed and that it is
# possible to establish a new connection.
self._storage = self.openClientStorage()
self._dostore()
# break into the internals to send a bogus message
zrpc_conn = self._storage._server.rpc
zrpc_conn.message_output(msg)
try:
self._dostore()
except ClientDisconnected:
pass
else:
self._storage.close()
self.fail("Server did not disconnect after bogus message")
self._storage.close()
self._storage = self.openClientStorage()
self._dostore()
self._storage.close()
# Test case for multiple storages participating in a single
# transaction. This is not really a connection test, but it needs
# about the same infrastructure (several storage servers).
# TODO: with the current ZEO code, this occasionally fails.
# That's the point of this test. :-)
def NOcheckMultiStorageTransaction(self):
# Configuration parameters (larger values mean more likely deadlocks)
N = 2
# These don't *have* to be all the same, but it's convenient this way
self.nservers = N
self.nthreads = N
self.ntrans = N
self.nobj = N
# Start extra servers
for i in range(1, self.nservers):
self._newAddr()
self.startServer(index=i)
# Spawn threads that each do some transactions on all storages
threads = []
try:
for i in range(self.nthreads):
t = MSTThread(self, "T%d" % i)
threads.append(t)
t.start()
# Wait for all threads to finish
for t in threads:
t.join(60)
self.failIf(t.isAlive(), "%s didn't die" % t.getName())
finally:
for t in threads:
t.closeclients()
def checkCrossDBInvalidations(self):
db1 = DB(self.openClientStorage())
c1 = db1.open()
r1 = c1.root()
r1["a"] = MinPO("a")
transaction.commit()
self.assertEqual(r1._p_state, 0) # up-to-date
db2 = DB(self.openClientStorage())
r2 = db2.open().root()
self.assertEqual(r2["a"].value, "a")
r2["b"] = MinPO("b")
transaction.commit()
# Make sure the invalidation is received in the other client.
# We've had problems with this timing out on "slow" and/or "very
# busy" machines, so we increase the sleep time on each trip, and
# are willing to wait quite a long time.
for i in range(20):
c1.sync()
if r1._p_state == -1:
break
time.sleep(i / 10.0)
self.assertEqual(r1._p_state, -1) # ghost
r1.keys() # unghostify
self.assertEqual(r1._p_serial, r2._p_serial)
self.assertEqual(r1["b"].value, "b")
db2.close()
db1.close()
class InvqTests(CommonSetupTearDown):
invq = 3
def checkQuickVerificationWith2Clients(self):
perstorage = self.openClientStorage(cache="test")
self.assertEqual(perstorage.verify_result, "full verification")
self._storage = self.openClientStorage()
oid = self._storage.new_oid()
oid2 = self._storage.new_oid()
# When we create a new storage, it should always do a full
# verification
self.assertEqual(self._storage.verify_result, "full verification")
# do two storages of the object to make sure an invalidation
# message is generated
revid = self._dostore(oid)
revid = self._dostore(oid, revid)
# Create a second object and revision to guarantee it doesn't
# show up in the list of invalidations sent when perstore restarts.
revid2 = self._dostore(oid2)
revid2 = self._dostore(oid2, revid2)
# sync() is needed to prevent invalidation for oid from arriving
# in the middle of the load() call.
perstorage.sync()
perstorage.load(oid, '')
perstorage.close()
revid = self._dostore(oid, revid)
perstorage = self.openClientStorage(cache="test")
self.assertEqual(perstorage.verify_result, "quick verification")
self.assertEqual(perstorage._server._last_invals,
(revid, [(oid, '')]))
self.assertEqual(perstorage.load(oid, ''),
self._storage.load(oid, ''))
perstorage.close()
def checkVerificationWith2ClientsInvqOverflow(self):
perstorage = self.openClientStorage(cache="test")
self.assertEqual(perstorage.verify_result, "full verification")
self._storage = self.openClientStorage()
oid = self._storage.new_oid()
# When we create a new storage, it should always do a full
# verification
self.assertEqual(self._storage.verify_result, "full verification")
# do two storages of the object to make sure an invalidation
# message is generated
revid = self._dostore(oid)
revid = self._dostore(oid, revid)
perstorage.load(oid, '')
perstorage.close()
# the test code sets invq bound to 2
for i in range(5):
revid = self._dostore(oid, revid)
perstorage = self.openClientStorage(cache="test")
self.assertEqual(perstorage.verify_result, "full verification")
t = time.time() + 30
while not perstorage.end_verify.isSet():
perstorage.sync()
if time.time() > t:
self.fail("timed out waiting for endVerify")
self.assertEqual(self._storage.load(oid, '')[1], revid)
self.assertEqual(perstorage.load(oid, ''),
self._storage.load(oid, ''))
perstorage.close()
class ReconnectionTests(CommonSetupTearDown):
# The setUp() starts a server automatically. In order for its
# state to persist, we set the class variable keep to 1. In
# order for its state to be cleaned up, the last startServer()
# call in the test must pass keep=0.
keep = 1
invq = 2
def checkReadOnlyStorage(self):
# Open a read-only client to a read-only *storage*; stores fail
# We don't want the read-write server created by setUp()
self.shutdownServer()
self._servers = []
# Start a read-only server
self.startServer(create=0, index=0, read_only=1, keep=0)
# Start a read-only client
self._storage = self.openClientStorage(read_only=1)
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
def checkReadOnlyFallbackReadOnlyStorage(self):
# Open a fallback client to a read-only *storage*; stores fail
# We don't want the read-write server created by setUp()
self.shutdownServer()
self._servers = []
# Start a read-only server
self.startServer(create=0, index=0, read_only=1, keep=0)
# Start a read-only-fallback client
self._storage = self.openClientStorage(read_only_fallback=1)
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
def checkReconnectReadOnly(self):
# A read-only client reconnects from a read-write to a
# read-only server
# Start a client
self._storage = self.openClientStorage(read_only=1)
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
# Shut down the server
self.shutdownServer()
self._servers = []
# Poll until the client disconnects
self.pollDown()
# Stores should still fail
self.assertRaises(ReadOnlyError, self._dostore)
# Restart the server
self.startServer(create=0, read_only=1, keep=0)
# Poll until the client connects
self.pollUp()
# Stores should still fail
self.assertRaises(ReadOnlyError, self._dostore)
def checkReconnectFallback(self):
# A fallback client reconnects from a read-write to a
# read-only server
# Start a client in fallback mode
self._storage = self.openClientStorage(read_only_fallback=1)
# Stores should succeed here
self._dostore()
# Shut down the server
self.shutdownServer()
self._servers = []
# Poll until the client disconnects
self.pollDown()
# Stores should fail now
self.assertRaises(ClientDisconnected, self._dostore)
# Restart the server
self.startServer(create=0, read_only=1, keep=0)
# Poll until the client connects
self.pollUp()
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
def checkReconnectUpgrade(self):
# A fallback client reconnects from a read-only to a
# read-write server
# We don't want the read-write server created by setUp()
self.shutdownServer()
self._servers = []
# Start a read-only server
self.startServer(create=0, read_only=1)
# Start a client in fallback mode
self._storage = self.openClientStorage(read_only_fallback=1)
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
# Shut down the server
self.shutdownServer()
self._servers = []
# Poll until the client disconnects
self.pollDown()
# Stores should fail now
self.assertRaises(ClientDisconnected, self._dostore)
# Restart the server, this time read-write
self.startServer(create=0, keep=0)
# Poll until the client sconnects
self.pollUp()
# Stores should now succeed
self._dostore()
def checkReconnectSwitch(self):
# A fallback client initially connects to a read-only server,
# then discovers a read-write server and switches to that
# We don't want the read-write server created by setUp()
self.shutdownServer()
self._servers = []
# Allocate a second address (for the second server)
self._newAddr()
# Start a read-only server
self.startServer(create=0, index=0, read_only=1, keep=0)
# Start a client in fallback mode
self._storage = self.openClientStorage(read_only_fallback=1)
# Stores should fail here
self.assertRaises(ReadOnlyError, self._dostore)
# Start a read-write server
self.startServer(index=1, read_only=0, keep=0)
# After a while, stores should work
for i in range(300): # Try for 30 seconds
try:
self._dostore()
break
except (ClientDisconnected, ReadOnlyError):
# If the client isn't connected at all, sync() returns
# quickly and the test fails because it doesn't wait
# long enough for the client.
time.sleep(0.1)
else:
self.fail("Couldn't store after starting a read-write server")
def checkNoVerificationOnServerRestart(self):
self._storage = self.openClientStorage()
# When we create a new storage, it should always do a full
# verification
self.assertEqual(self._storage.verify_result, "full verification")
self._dostore()
self.shutdownServer()
self.pollDown()
self._storage.verify_result = None
self.startServer(create=0, keep=0)
self.pollUp()
# There were no transactions committed, so no verification
# should be needed.
self.assertEqual(self._storage.verify_result, "no verification")
def checkNoVerificationOnServerRestartWith2Clients(self):
perstorage = self.openClientStorage(cache="test")
self.assertEqual(perstorage.verify_result, "full verification")
self._storage = self.openClientStorage()
oid = self._storage.new_oid()
# When we create a new storage, it should always do a full
# verification
self.assertEqual(self._storage.verify_result, "full verification")
# do two storages of the object to make sure an invalidation
# message is generated
revid = self._dostore(oid)
self._dostore(oid, revid)
perstorage.load(oid, '')
self.shutdownServer()
self.pollDown()
self._storage.verify_result = None
perstorage.verify_result = None
logging.info('2ALLBEEF')
self.startServer(create=0, keep=0)
self.pollUp()
self.pollUp(storage=perstorage)
# There were no transactions committed, so no verification
# should be needed.
self.assertEqual(self._storage.verify_result, "no verification")
self.assertEqual(perstorage.verify_result, "no verification")
perstorage.close()
self._storage.close()
class TimeoutTests(CommonSetupTearDown):
timeout = 1
def checkTimeout(self):
storage = self.openClientStorage()
txn = Transaction()
storage.tpc_begin(txn)
storage.tpc_vote(txn)
time.sleep(2)
self.assertRaises(ClientDisconnected, storage.tpc_finish, txn)
storage.close()
def checkTimeoutOnAbort(self):
storage = self.openClientStorage()
txn = Transaction()
storage.tpc_begin(txn)
storage.tpc_vote(txn)
storage.tpc_abort(txn)
storage.close()
def checkTimeoutOnAbortNoLock(self):
storage = self.openClientStorage()
txn = Transaction()
storage.tpc_begin(txn)
storage.tpc_abort(txn)
storage.close()
def checkTimeoutAfterVote(self):
raises = self.assertRaises
unless = self.failUnless
self._storage = storage = self.openClientStorage()
# Assert that the zeo cache is empty
unless(not list(storage._cache.contents()))
# Create the object
oid = storage.new_oid()
obj = MinPO(7)
# Now do a store, sleeping before the finish so as to cause a timeout
t = Transaction()
storage.tpc_begin(t)
revid1 = storage.store(oid, ZERO, zodb_pickle(obj), '', t)
storage.tpc_vote(t)
# Now sleep long enough for the storage to time out
time.sleep(3)
storage.sync()
unless(not storage.is_connected())
storage._wait()
unless(storage.is_connected())
# We expect finish to fail
raises(ClientDisconnected, storage.tpc_finish, t)
# The cache should still be empty
unless(not list(storage._cache.contents()))
# Load should fail since the object should not be in either the cache
# or the server.
raises(KeyError, storage.load, oid, '')
def checkTimeoutProvokingConflicts(self):
eq = self.assertEqual
raises = self.assertRaises
require = self.assert_
self._storage = storage = self.openClientStorage()
# Assert that the zeo cache is empty.
require(not list(storage._cache.contents()))
# Create the object
oid = storage.new_oid()
obj = MinPO(7)
# We need to successfully commit an object now so we have something to
# conflict about.
t = Transaction()
storage.tpc_begin(t)
revid1a = storage.store(oid, ZERO, zodb_pickle(obj), '', t)
revid1b = storage.tpc_vote(t)
revid1 = handle_serials(oid, revid1a, revid1b)
storage.tpc_finish(t)
# Now do a store, sleeping before the finish so as to cause a timeout.
obj.value = 8
t = Transaction()
storage.tpc_begin(t)
revid2a = storage.store(oid, revid1, zodb_pickle(obj), '', t)
revid2b = storage.tpc_vote(t)
revid2 = handle_serials(oid, revid2a, revid2b)
# Now sleep long enough for the storage to time out.
# This used to sleep for 3 seconds, and sometimes (but very rarely)
# failed then. Now we try for a minute. It typically succeeds
# on the second time thru the loop, and, since self.timeout is 1,
# it's typically faster now (2/1.8 ~= 1.11 seconds sleeping instead
# of 3).
deadline = time.time() + 60 # wait up to a minute
while time.time() < deadline:
if storage.is_connected():
time.sleep(self.timeout / 1.8)
storage.sync()
else:
break
storage.sync()
require(not storage.is_connected())
storage._wait()
require(storage.is_connected())
# We expect finish to fail.
raises(ClientDisconnected, storage.tpc_finish, t)
# Now we think we've committed the second transaction, but we really
# haven't. A third one should produce a POSKeyError on the server,
# which manifests as a ConflictError on the client.
obj.value = 9
t = Transaction()
storage.tpc_begin(t)
storage.store(oid, revid2, zodb_pickle(obj), '', t)
raises(ConflictError, storage.tpc_vote, t)
# Even aborting won't help.
storage.tpc_abort(t)
storage.tpc_finish(t)
# Try again.
obj.value = 10
t = Transaction()
storage.tpc_begin(t)
storage.store(oid, revid2, zodb_pickle(obj), '', t)
# Even aborting won't help.
raises(ConflictError, storage.tpc_vote, t)
# Abort this one and try a transaction that should succeed.
storage.tpc_abort(t)
storage.tpc_finish(t)
# Now do a store.
obj.value = 11
t = Transaction()
storage.tpc_begin(t)
revid2a = storage.store(oid, revid1, zodb_pickle(obj), '', t)
revid2b = storage.tpc_vote(t)
revid2 = handle_serials(oid, revid2a, revid2b)
storage.tpc_finish(t)
# Now load the object and verify that it has a value of 11.
data, revid = storage.load(oid, '')
eq(zodb_unpickle(data), MinPO(11))
eq(revid, revid2)
class MSTThread(threading.Thread):
__super_init = threading.Thread.__init__
def __init__(self, testcase, name):
self.__super_init(name=name)
self.testcase = testcase
self.clients = []
def run(self):
tname = self.getName()
testcase = self.testcase
# Create client connections to each server
clients = self.clients
for i in range(len(testcase.addr)):
c = testcase.openClientStorage(addr=testcase.addr[i])
c.__name = "C%d" % i
clients.append(c)
for i in range(testcase.ntrans):
# Because we want a transaction spanning all storages,
# we can't use _dostore(). This is several _dostore() calls
# expanded in-line (mostly).
# Create oid->serial mappings
for c in clients:
c.__oids = []
c.__serials = {}
# Begin a transaction
t = Transaction()
for c in clients:
#print "%s.%s.%s begin\n" % (tname, c.__name, i),
c.tpc_begin(t)
for j in range(testcase.nobj):
for c in clients:
# Create and store a new object on each server
oid = c.new_oid()
c.__oids.append(oid)
data = MinPO("%s.%s.t%d.o%d" % (tname, c.__name, i, j))
#print data.value
data = zodb_pickle(data)
s = c.store(oid, ZERO, data, '', t)
c.__serials.update(handle_all_serials(oid, s))
# Vote on all servers and handle serials
for c in clients:
#print "%s.%s.%s vote\n" % (tname, c.__name, i),
s = c.tpc_vote(t)
c.__serials.update(handle_all_serials(None, s))
# Finish on all servers
for c in clients:
#print "%s.%s.%s finish\n" % (tname, c.__name, i),
c.tpc_finish(t)
for c in clients:
# Check that we got serials for all oids
for oid in c.__oids:
testcase.failUnless(c.__serials.has_key(oid))
# Check that we got serials for no other oids
for oid in c.__serials.keys():
testcase.failUnless(oid in c.__oids)
def closeclients(self):
# Close clients opened by run()
for c in self.clients:
try:
c.close()
except:
pass
ZODB/src/ZEO/tests/stress.py 0000640 0001750 0000144 00000007304 10214421637 015451 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""A ZEO client-server stress test to look for leaks.
The stress test should run in an infinite loop and should involve
multiple connections.
"""
# TODO: This code is currently broken.
import transaction
import ZODB
from ZODB.MappingStorage import MappingStorage
from ZODB.tests import MinPO
from ZEO.ClientStorage import ClientStorage
from ZEO.tests import forker
import os
import random
import types
NUM_TRANSACTIONS_PER_CONN = 10
NUM_CONNECTIONS = 10
NUM_ROOTS = 20
MAX_DEPTH = 20
MIN_OBJSIZE = 128
MAX_OBJSIZE = 2048
def an_object():
"""Return an object suitable for a PersistentMapping key"""
size = random.randrange(MIN_OBJSIZE, MAX_OBJSIZE)
if os.path.exists("/dev/urandom"):
f = open("/dev/urandom")
buf = f.read(size)
f.close()
return buf
else:
f = open(MinPO.__file__)
l = list(f.read(size))
f.close()
random.shuffle(l)
return "".join(l)
def setup(cn):
"""Initialize the database with some objects"""
root = cn.root()
for i in range(NUM_ROOTS):
prev = an_object()
for j in range(random.randrange(1, MAX_DEPTH)):
o = MinPO.MinPO(prev)
prev = o
root[an_object()] = o
transaction.commit()
cn.close()
def work(cn):
"""Do some work with a transaction"""
cn.sync()
root = cn.root()
obj = random.choice(root.values())
# walk down to the bottom
while not isinstance(obj.value, types.StringType):
obj = obj.value
obj.value = an_object()
transaction.commit()
def main():
# Yuck! Need to cleanup forker so that the API is consistent
# across Unix and Windows, at least if that's possible.
if os.name == "nt":
zaddr, tport, pid = forker.start_zeo_server('MappingStorage', ())
def exitserver():
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(tport)
s.close()
else:
zaddr = '', random.randrange(20000, 30000)
pid, exitobj = forker.start_zeo_server(MappingStorage(), zaddr)
def exitserver():
exitobj.close()
while 1:
pid = start_child(zaddr)
print "started", pid
os.waitpid(pid, 0)
exitserver()
def start_child(zaddr):
pid = os.fork()
if pid != 0:
return pid
try:
_start_child(zaddr)
finally:
os._exit(0)
def _start_child(zaddr):
storage = ClientStorage(zaddr, debug=1, min_disconnect_poll=0.5, wait=1)
db = ZODB.DB(storage, pool_size=NUM_CONNECTIONS)
setup(db.open())
conns = []
conn_count = 0
for i in range(NUM_CONNECTIONS):
c = db.open()
c.__count = 0
conns.append(c)
conn_count += 1
while conn_count < 25:
c = random.choice(conns)
if c.__count > NUM_TRANSACTIONS_PER_CONN:
conns.remove(c)
c.close()
conn_count += 1
c = db.open()
c.__count = 0
conns.append(c)
else:
c.__count += 1
work(c)
if __name__ == "__main__":
main()
ZODB/src/ZEO/tests/CommitLockTests.py 0000640 0001750 0000144 00000016523 10214413267 017215 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Tests of the distributed commit lock."""
import threading
import time
from persistent.TimeStamp import TimeStamp
import transaction
from ZODB.tests.StorageTestBase import zodb_pickle, MinPO
import ZEO.ClientStorage
from ZEO.Exceptions import ClientDisconnected
from ZEO.tests.TestThread import TestThread
ZERO = '\0'*8
class DummyDB:
def invalidate(self, *args, **kwargs):
pass
class WorkerThread(TestThread):
# run the entire test in a thread so that the blocking call for
# tpc_vote() doesn't hang the test suite.
def __init__(self, storage, trans, method="tpc_finish"):
self.storage = storage
self.trans = trans
self.method = method
self.ready = threading.Event()
TestThread.__init__(self)
def testrun(self):
try:
self.storage.tpc_begin(self.trans)
oid = self.storage.new_oid()
p = zodb_pickle(MinPO("c"))
self.storage.store(oid, ZERO, p, '', self.trans)
oid = self.storage.new_oid()
p = zodb_pickle(MinPO("c"))
self.storage.store(oid, ZERO, p, '', self.trans)
self.myvote()
if self.method == "tpc_finish":
self.storage.tpc_finish(self.trans)
else:
self.storage.tpc_abort(self.trans)
except ClientDisconnected:
pass
def myvote(self):
# The vote() call is synchronous, which makes it difficult to
# coordinate the action of multiple threads that all call
# vote(). This method sends the vote call, then sets the
# event saying vote was called, then waits for the vote
# response. It digs deep into the implementation of the client.
# This method is a replacement for:
# self.ready.set()
# self.storage.tpc_vote(self.trans)
rpc = self.storage._server.rpc
msgid = rpc._deferred_call('vote', id(self.trans))
self.ready.set()
rpc._deferred_wait(msgid)
self.storage._check_serials()
class CommitLockTests:
NUM_CLIENTS = 5
# The commit lock tests verify that the storage successfully
# blocks and restarts transactions when there is contention for a
# single storage. There are a lot of cases to cover.
# The general flow of these tests is to start a transaction by
# getting far enough into 2PC to acquire the commit lock. Then
# begin one or more other connections that also want to commit.
# This causes the commit lock code to be exercised. Once the
# other connections are started, the first transaction completes.
def _cleanup(self):
for store, trans in self._storages:
store.tpc_abort(trans)
store.close()
self._storages = []
def _start_txn(self):
txn = transaction.Transaction()
self._storage.tpc_begin(txn)
oid = self._storage.new_oid()
self._storage.store(oid, ZERO, zodb_pickle(MinPO(1)), '', txn)
return oid, txn
def _begin_threads(self):
# Start a second transaction on a different connection without
# blocking the test thread. Returns only after each thread has
# set it's ready event.
self._storages = []
self._threads = []
for i in range(self.NUM_CLIENTS):
storage = self._duplicate_client()
txn = transaction.Transaction()
tid = self._get_timestamp()
t = WorkerThread(storage, txn)
self._threads.append(t)
t.start()
t.ready.wait()
# Close on the connections abnormally to test server response
if i == 0:
storage.close()
else:
self._storages.append((storage, txn))
def _finish_threads(self):
for t in self._threads:
t.cleanup()
def _duplicate_client(self):
"Open another ClientStorage to the same server."
# It's hard to find the actual address.
# The rpc mgr addr attribute is a list. Each element in the
# list is a socket domain (AF_INET, AF_UNIX, etc.) and an
# address.
addr = self._storage._addr
new = ZEO.ClientStorage.ClientStorage(addr, wait=1)
new.registerDB(DummyDB(), None)
return new
def _get_timestamp(self):
t = time.time()
t = TimeStamp(*time.gmtime(t)[:5]+(t%60,))
return `t`
class CommitLockVoteTests(CommitLockTests):
def checkCommitLockVoteFinish(self):
oid, txn = self._start_txn()
self._storage.tpc_vote(txn)
self._begin_threads()
self._storage.tpc_finish(txn)
self._storage.load(oid, '')
self._finish_threads()
self._dostore()
self._cleanup()
def checkCommitLockVoteAbort(self):
oid, txn = self._start_txn()
self._storage.tpc_vote(txn)
self._begin_threads()
self._storage.tpc_abort(txn)
self._finish_threads()
self._dostore()
self._cleanup()
def checkCommitLockVoteClose(self):
oid, txn = self._start_txn()
self._storage.tpc_vote(txn)
self._begin_threads()
self._storage.close()
self._finish_threads()
self._cleanup()
class CommitLockUndoTests(CommitLockTests):
def _get_trans_id(self):
self._dostore()
L = self._storage.undoInfo()
return L[0]['id']
def _begin_undo(self, trans_id, txn):
rpc = self._storage._server.rpc
return rpc._deferred_call('undo', trans_id, id(txn))
def _finish_undo(self, msgid):
return self._storage._server.rpc._deferred_wait(msgid)
def checkCommitLockUndoFinish(self):
trans_id = self._get_trans_id()
oid, txn = self._start_txn()
msgid = self._begin_undo(trans_id, txn)
self._begin_threads()
self._finish_undo(msgid)
self._storage.tpc_vote(txn)
self._storage.tpc_finish(txn)
self._storage.load(oid, '')
self._finish_threads()
self._dostore()
self._cleanup()
def checkCommitLockUndoAbort(self):
trans_id = self._get_trans_id()
oid, txn = self._start_txn()
msgid = self._begin_undo(trans_id, txn)
self._begin_threads()
self._finish_undo(msgid)
self._storage.tpc_vote(txn)
self._storage.tpc_abort(txn)
self._finish_threads()
self._dostore()
self._cleanup()
def checkCommitLockUndoClose(self):
trans_id = self._get_trans_id()
oid, txn = self._start_txn()
msgid = self._begin_undo(trans_id, txn)
self._begin_threads()
self._finish_undo(msgid)
self._storage.tpc_vote(txn)
self._storage.close()
self._finish_threads()
self._cleanup()
ZODB/src/ZEO/tests/zeoserver.py 0000640 0001750 0000144 00000016246 10240676003 016155 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Helper file used to launch a ZEO server cross platform"""
import os
import sys
import time
import errno
import getopt
import socket
import signal
import asyncore
import threading
import logging
import ThreadedAsync.LoopCallback
from ZEO.StorageServer import StorageServer
from ZEO.runzeo import ZEOOptions
def cleanup(storage):
# FileStorage and the Berkeley storages have this method, which deletes
# all files and directories used by the storage. This prevents @-files
# from clogging up /tmp
try:
storage.cleanup()
except AttributeError:
pass
logger = logging.getLogger('ZEO.tests.zeoserver')
def log(label, msg, *args):
message = "(%s) %s" % (label, msg)
logger.debug(message, *args)
class ZEOTestServer(asyncore.dispatcher):
"""A server for killing the whole process at the end of a test.
The first time we connect to this server, we write an ack character down
the socket. The other end should block on a recv() of the socket so it
can guarantee the server has started up before continuing on.
The second connect to the port immediately exits the process, via
os._exit(), without writing data on the socket. It does close and clean
up the storage first. The other end will get the empty string from its
recv() which will be enough to tell it that the server has exited.
I think this should prevent us from ever getting a legitimate addr-in-use
error.
"""
__super_init = asyncore.dispatcher.__init__
def __init__(self, addr, server, keep):
self.__super_init()
self._server = server
self._sockets = [self]
self._keep = keep
# Count down to zero, the number of connects
self._count = 1
self._label ='%d @ %s' % (os.getpid(), addr)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
# Some ZEO tests attempt a quick start of the server using the same
# port so we have to set the reuse flag.
self.set_reuse_addr()
try:
self.bind(addr)
except:
# We really want to see these exceptions
import traceback
traceback.print_exc()
raise
self.listen(5)
self.log('bound and listening')
def log(self, msg, *args):
log(self._label, msg, *args)
def handle_accept(self):
sock, addr = self.accept()
self.log('in handle_accept()')
# When we're done with everything, close the storage. Do not write
# the ack character until the storage is finished closing.
if self._count <= 0:
self.log('closing the storage')
self._server.close_server()
if not self._keep:
for storage in self._server.storages.values():
cleanup(storage)
self.log('exiting')
# Close all the other sockets so that we don't have to wait
# for os._exit() to get to it before starting the next
# server process.
for s in self._sockets:
s.close()
# Now explicitly close the socket returned from accept(),
# since it didn't go through the wrapper.
sock.close()
os._exit(0)
self.log('continuing')
sock.send('X')
self._count -= 1
def register_socket(self, sock):
# Register a socket to be closed when server shutsdown.
self._sockets.append(sock)
class Suicide(threading.Thread):
def __init__(self, addr):
threading.Thread.__init__(self)
self._adminaddr = addr
def run(self):
# If this process doesn't exit in 330 seconds, commit suicide.
# The client threads in the ConcurrentUpdate tests will run for
# as long as 300 seconds. Set this timeout to 330 to minimize
# chance that the server gives up before the clients.
time.sleep(330)
log(str(os.getpid()), "suicide thread invoking shutdown")
# If the server hasn't shut down yet, the client may not be
# able to connect to it. If so, try to kill the process to
# force it to shutdown.
if hasattr(os, "kill"):
os.kill(pid, signal.SIGTERM)
time.sleep(5)
os.kill(pid, signal.SIGKILL)
else:
from ZEO.tests.forker import shutdown_zeo_server
# Nott: If the -k option was given to zeoserver, then the
# process will go away but the temp files won't get
# cleaned up.
shutdown_zeo_server(self._adminaddr)
def main():
global pid
pid = os.getpid()
label = str(pid)
log(label, "starting")
# We don't do much sanity checking of the arguments, since if we get it
# wrong, it's a bug in the test suite.
keep = 0
configfile = None
# Parse the arguments and let getopt.error percolate
opts, args = getopt.getopt(sys.argv[1:], 'kC:')
for opt, arg in opts:
if opt == '-k':
keep = 1
elif opt == '-C':
configfile = arg
zo = ZEOOptions()
zo.realize(["-C", configfile])
zeo_port = int(zo.address[1])
if zo.auth_protocol == "plaintext":
import ZEO.tests.auth_plaintext
# Open the config file and let ZConfig parse the data there. Then remove
# the config file, otherwise we'll leave turds.
# The rest of the args are hostname, portnum
test_port = zeo_port + 1
test_addr = ('localhost', test_port)
addr = ('localhost', zeo_port)
log(label, 'creating the storage server')
storage = zo.storages[0].open()
mon_addr = None
if zo.monitor_address:
mon_addr = zo.monitor_address
server = StorageServer(
zo.address,
{"1": storage},
read_only=zo.read_only,
invalidation_queue_size=zo.invalidation_queue_size,
transaction_timeout=zo.transaction_timeout,
monitor_address=mon_addr,
auth_protocol=zo.auth_protocol,
auth_database=zo.auth_database,
auth_realm=zo.auth_realm)
try:
log(label, 'creating the test server, keep: %s', keep)
t = ZEOTestServer(test_addr, server, keep)
except socket.error, e:
if e[0] <> errno.EADDRINUSE: raise
log(label, 'addr in use, closing and exiting')
storage.close()
cleanup(storage)
sys.exit(2)
t.register_socket(server.dispatcher)
# Create daemon suicide thread
d = Suicide(test_addr)
d.setDaemon(1)
d.start()
# Loop for socket events
log(label, 'entering ThreadedAsync loop')
ThreadedAsync.LoopCallback.loop()
if __name__ == '__main__':
main()
ZODB/src/ZEO/tests/testMonitor.py 0000640 0001750 0000144 00000005404 10214421637 016454 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Test that the monitor produce sensible results.
$Id: testMonitor.py 29448 2005-03-11 23:02:55Z tim_one $
"""
import socket
import unittest
from ZEO.tests.ConnectionTests import CommonSetupTearDown
from ZEO.monitor import StorageStats
class MonitorTests(CommonSetupTearDown):
monitor = 1
def get_monitor_output(self):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(('localhost', 42000))
L = []
while 1:
buf = s.recv(8192)
if buf:
L.append(buf)
else:
break
s.close()
return "".join(L)
def parse(self, s):
# Return a list of StorageStats, one for each storage.
lines = s.split("\n")
self.assert_(lines[0].startswith("ZEO monitor server"))
# lines[1] is a date
# Break up rest of lines into sections starting with Storage:
# and ending with a blank line.
sections = []
cur = None
for line in lines[2:]:
if line.startswith("Storage:"):
cur = [line]
elif line:
cur.append(line)
else:
if cur is not None:
sections.append(cur)
cur = None
assert cur is None # bug in the test code if this fails
d = {}
for sect in sections:
hdr = sect[0]
key, value = hdr.split(":")
storage = int(value)
s = d[storage] = StorageStats()
s.parse("\n".join(sect[1:]))
return d
def getConfig(self, path, create, read_only):
return """"""
def testMonitor(self):
# Just open a client to know that the server is up and running
# TODO: should put this in setUp.
self.storage = self.openClientStorage()
s = self.get_monitor_output()
self.storage.close()
self.assert_(s.find("monitor") != -1)
d = self.parse(s)
stats = d[1]
self.assertEqual(stats.clients, 1)
self.assertEqual(stats.commits, 0)
def test_suite():
return unittest.makeSuite(MonitorTests)
ZODB/src/ZEO/StorageServer.py 0000640 0001750 0000144 00000116171 10375443702 015567 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002, 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""The StorageServer class and the exception that it may raise.
This server acts as a front-end for one or more real storages, like
file storage or Berkeley storage.
TODO: Need some basic access control-- a declaration of the methods
exported for invocation by the server.
"""
import asyncore
import cPickle
import os
import sys
import threading
import time
import logging
import transaction
from ZEO import ClientStub
from ZEO.CommitLog import CommitLog
from ZEO.monitor import StorageStats, StatsServer
from ZEO.zrpc.server import Dispatcher
from ZEO.zrpc.connection import ManagedServerConnection, Delay, MTDelay
from ZEO.zrpc.trigger import trigger
from ZEO.Exceptions import AuthError
from ZODB.ConflictResolution import ResolvedSerial
from ZODB.POSException import StorageError, StorageTransactionError
from ZODB.POSException import TransactionError, ReadOnlyError, ConflictError
from ZODB.serialize import referencesf
from ZODB.utils import u64, oid_repr, mktemp
from ZODB.loglevels import BLATHER
logger = logging.getLogger('ZEO.StorageServer')
# TODO: This used to say "ZSS", which is now implied in the logger name.
# Can this be either set to str(os.getpid()) (if that makes sense) or removed?
_label = "" # default label used for logging.
def set_label():
"""Internal helper to reset the logging label (e.g. after fork())."""
global _label
_label = "%s" % os.getpid()
def log(message, level=logging.INFO, label=None, exc_info=False):
"""Internal helper to log a message."""
label = label or _label
if label:
message = "(%s) %s" % (label, message)
logger.log(level, message, exc_info=exc_info)
class StorageServerError(StorageError):
"""Error reported when an unpicklable exception is raised."""
class ZEOStorage:
"""Proxy to underlying storage for a single remote client."""
# Classes we instantiate. A subclass might override.
ClientStorageStubClass = ClientStub.ClientStorage
# A list of extension methods. A subclass with extra methods
# should override.
extensions = []
def __init__(self, server, read_only=0, auth_realm=None):
self.server = server
# timeout and stats will be initialized in register()
self.timeout = None
self.stats = None
self.connection = None
self.client = None
self.storage = None
self.storage_id = "uninitialized"
self.transaction = None
self.read_only = read_only
self.locked = 0
self.verifying = 0
self.store_failed = 0
self.log_label = _label
self.authenticated = 0
self.auth_realm = auth_realm
self.blob_transfer = {}
self.blob_log = []
self.blob_loads = {}
# The authentication protocol may define extra methods.
self._extensions = {}
for func in self.extensions:
self._extensions[func.func_name] = None
def finish_auth(self, authenticated):
if not self.auth_realm:
return 1
self.authenticated = authenticated
return authenticated
def set_database(self, database):
self.database = database
def notifyConnected(self, conn):
self.connection = conn # For restart_other() below
self.client = self.ClientStorageStubClass(conn)
addr = conn.addr
if isinstance(addr, type("")):
label = addr
else:
host, port = addr
label = str(host) + ":" + str(port)
self.log_label = _label + "/" + label
def notifyDisconnected(self):
# When this storage closes, we must ensure that it aborts
# any pending transaction.
if self.transaction is not None:
self.log("disconnected during transaction %s" % self.transaction)
self._abort()
else:
self.log("disconnected")
if self.stats is not None:
self.stats.clients -= 1
def __repr__(self):
tid = self.transaction and repr(self.transaction.id)
if self.storage:
stid = (self.storage._transaction and
repr(self.storage._transaction.id))
else:
stid = None
name = self.__class__.__name__
return "<%s %X trans=%s s_trans=%s>" % (name, id(self), tid, stid)
def log(self, msg, level=logging.INFO, exc_info=False):
log(msg, level=level, label=self.log_label, exc_info=exc_info)
def setup_delegation(self):
"""Delegate several methods to the storage"""
self.versionEmpty = self.storage.versionEmpty
self.versions = self.storage.versions
self.getSerial = self.storage.getSerial
self.history = self.storage.history
self.load = self.storage.load
self.loadSerial = self.storage.loadSerial
self.modifiedInVersion = self.storage.modifiedInVersion
try:
fn = self.storage.getExtensionMethods
except AttributeError:
# We must be running with a ZODB which
# predates adding getExtensionMethods to
# BaseStorage. Eventually this try/except
# can be removed
pass
else:
d = fn()
self._extensions.update(d)
for name in d.keys():
assert not hasattr(self, name)
setattr(self, name, getattr(self.storage, name))
self.lastTransaction = self.storage.lastTransaction
def _check_tid(self, tid, exc=None):
if self.read_only:
raise ReadOnlyError()
if self.transaction is None:
caller = sys._getframe().f_back.f_code.co_name
self.log("no current transaction: %s()" % caller,
level=logging.WARNING)
if exc is not None:
raise exc(None, tid)
else:
return 0
if self.transaction.id != tid:
caller = sys._getframe().f_back.f_code.co_name
self.log("%s(%s) invalid; current transaction = %s" %
(caller, repr(tid), repr(self.transaction.id)),
logging.WARNING)
if exc is not None:
raise exc(self.transaction.id, tid)
else:
return 0
return 1
def getAuthProtocol(self):
"""Return string specifying name of authentication module to use.
The module name should be auth_%s where %s is auth_protocol."""
protocol = self.server.auth_protocol
if not protocol or protocol == 'none':
return None
return protocol
def register(self, storage_id, read_only):
"""Select the storage that this client will use
This method must be the first one called by the client.
For authenticated storages this method will be called by the client
immediately after authentication is finished.
"""
if self.auth_realm and not self.authenticated:
raise AuthError("Client was never authenticated with server!")
if self.storage is not None:
self.log("duplicate register() call")
raise ValueError("duplicate register() call")
storage = self.server.storages.get(storage_id)
if storage is None:
self.log("unknown storage_id: %s" % storage_id)
raise ValueError("unknown storage: %s" % storage_id)
if not read_only and (self.read_only or storage.isReadOnly()):
raise ReadOnlyError()
self.read_only = self.read_only or read_only
self.storage_id = storage_id
self.storage = storage
self.setup_delegation()
self.timeout, self.stats = self.server.register_connection(storage_id,
self)
def get_info(self):
return {'length': len(self.storage),
'size': self.storage.getSize(),
'name': self.storage.getName(),
'supportsUndo': self.storage.supportsUndo(),
'supportsVersions': self.storage.supportsVersions(),
'extensionMethods': self.getExtensionMethods(),
}
def get_size_info(self):
return {'length': len(self.storage),
'size': self.storage.getSize(),
}
def getExtensionMethods(self):
return self._extensions
def loadEx(self, oid, version):
self.stats.loads += 1
return self.storage.loadEx(oid, version)
def loadBefore(self, oid, tid):
self.stats.loads += 1
return self.storage.loadBefore(oid, tid)
def zeoLoad(self, oid):
self.stats.loads += 1
v = self.storage.modifiedInVersion(oid)
if v:
pv, sv = self.storage.load(oid, v)
else:
pv = sv = None
try:
p, s = self.storage.load(oid, '')
except KeyError:
if sv:
# Created in version, no non-version data
p = s = None
else:
raise
return p, s, v, pv, sv
def getInvalidations(self, tid):
invtid, invlist = self.server.get_invalidations(tid)
if invtid is None:
return None
self.log("Return %d invalidations up to tid %s"
% (len(invlist), u64(invtid)))
return invtid, invlist
def verify(self, oid, version, tid):
try:
t = self.storage.getTid(oid)
except KeyError:
self.client.invalidateVerify((oid, ""))
else:
if tid != t:
# This will invalidate non-version data when the
# client only has invalid version data. Since this is
# an uncommon case, we avoid the cost of checking
# whether the serial number matches the current
# non-version data.
self.client.invalidateVerify((oid, version))
def zeoVerify(self, oid, s, sv):
if not self.verifying:
self.verifying = 1
self.stats.verifying_clients += 1
try:
os = self.storage.getTid(oid)
except KeyError:
self.client.invalidateVerify((oid, ''))
# It's not clear what we should do now. The KeyError
# could be caused by an object uncreation, in which case
# invalidation is right. It could be an application bug
# that left a dangling reference, in which case it's bad.
else:
# If the client has version data, the logic is a bit more
# complicated. If the current serial number matches the
# client serial number, then the non-version data must
# also be valid. If the current serialno is for a
# version, then the non-version data can't change.
# If the version serialno isn't valid, then the
# non-version serialno may or may not be valid. Rather
# than trying to figure it whether it is valid, we just
# invalidate it. Sending an invalidation for the
# non-version data implies invalidating the version data
# too, since an update to non-version data can only occur
# after the version is aborted or committed.
if sv:
if sv != os:
self.client.invalidateVerify((oid, ''))
else:
if s != os:
self.client.invalidateVerify((oid, ''))
def endZeoVerify(self):
if self.verifying:
self.stats.verifying_clients -= 1
self.verifying = 0
self.client.endVerify()
def pack(self, time, wait=1):
# Yes, you can pack a read-only server or storage!
if wait:
return run_in_thread(self._pack_impl, time)
else:
# If the client isn't waiting for a reply, start a thread
# and forget about it.
t = threading.Thread(target=self._pack_impl, args=(time,))
t.start()
return None
def _pack_impl(self, time):
self.log("pack(time=%s) started..." % repr(time))
self.storage.pack(time, referencesf)
self.log("pack(time=%s) complete" % repr(time))
# Broadcast new size statistics
self.server.invalidate(0, self.storage_id, None,
(), self.get_size_info())
def new_oids(self, n=100):
"""Return a sequence of n new oids, where n defaults to 100"""
if self.read_only:
raise ReadOnlyError()
if n <= 0:
n = 1
return [self.storage.new_oid() for i in range(n)]
# undoLog and undoInfo are potentially slow methods
def undoInfo(self, first, last, spec):
return run_in_thread(self.storage.undoInfo, first, last, spec)
def undoLog(self, first, last):
return run_in_thread(self.storage.undoLog, first, last)
def tpc_begin(self, id, user, description, ext, tid=None, status=" "):
if self.read_only:
raise ReadOnlyError()
if self.transaction is not None:
if self.transaction.id == id:
self.log("duplicate tpc_begin(%s)" % repr(id))
return
else:
raise StorageTransactionError("Multiple simultaneous tpc_begin"
" requests from one client.")
self.transaction = t = transaction.Transaction()
t.id = id
t.user = user
t.description = description
t._extension = ext
self.serials = []
self.invalidated = []
self.txnlog = CommitLog()
self.tid = tid
self.status = status
self.store_failed = 0
self.stats.active_txns += 1
def tpc_finish(self, id):
if not self._check_tid(id):
return
assert self.locked
self.stats.active_txns -= 1
self.stats.commits += 1
self.storage.tpc_finish(self.transaction)
tid = self.storage.lastTransaction()
if self.invalidated:
self.server.invalidate(self, self.storage_id, tid,
self.invalidated, self.get_size_info())
self._clear_transaction()
# Return the tid, for cache invalidation optimization
return tid
def tpc_abort(self, id):
if not self._check_tid(id):
return
self.stats.active_txns -= 1
self.stats.aborts += 1
if self.locked:
self.storage.tpc_abort(self.transaction)
self._clear_transaction()
def _clear_transaction(self):
# Common code at end of tpc_finish() and tpc_abort()
self.transaction = None
self.txnlog.close()
if self.locked:
self.locked = 0
self.timeout.end(self)
self.stats.lock_time = None
self.log("Transaction released storage lock", BLATHER)
# _handle_waiting() can start another transaction (by
# restarting a waiting one) so must be done last
self._handle_waiting()
def _abort(self):
# called when a connection is closed unexpectedly
if not self.locked:
# Delete (d, zeo_storage) from the _waiting list, if found.
waiting = self.storage._waiting
for i in range(len(waiting)):
d, z = waiting[i]
if z is self:
del waiting[i]
self.log("Closed connection removed from waiting list."
" Clients waiting: %d." % len(waiting))
break
if self.transaction:
self.stats.active_txns -= 1
self.stats.aborts += 1
self.tpc_abort(self.transaction.id)
# The public methods of the ZEO client API do not do the real work.
# They defer work until after the storage lock has been acquired.
# Most of the real implementations are in methods beginning with
# an _.
def storea(self, oid, serial, data, version, id):
self._check_tid(id, exc=StorageTransactionError)
self.stats.stores += 1
self.txnlog.store(oid, serial, data, version)
def storeBlobEnd(self, oid, serial, data, version, id):
key = (oid, id)
if key not in self.blob_transfer:
raise Exception, "Can't finish a non-started Blob"
tempname, tempfile = self.blob_transfer.pop(key)
tempfile.close()
self.blob_log.append((oid, serial, data, tempname, version))
def storeBlob(self, oid, serial, chunk, version, id):
# XXX check that underlying storage supports blobs
key = (oid, id)
if key not in self.blob_transfer:
tempname = mktemp()
tempfile = open(tempname, "wb")
self.blob_transfer[key] = (tempname, tempfile) # XXX Force close and remove them when Storage closes
else:
tempname, tempfile = self.blob_transfer[key]
tempfile.write(chunk)
def loadBlob(self, oid, serial, version, offset):
key = (oid, serial)
if not key in self.blob_loads:
self.blob_loads[key] = \
open(self.storage.loadBlob(oid, serial, version))
blobdata = self.blob_loads[key]
blobdata.seek(offset)
chunk = blobdata.read(4096)
if not chunk:
del self.blob_loads[key]
return chunk
# The following four methods return values, so they must acquire
# the storage lock and begin the transaction before returning.
def vote(self, id):
self._check_tid(id, exc=StorageTransactionError)
if self.locked:
return self._vote()
else:
return self._wait(lambda: self._vote())
def abortVersion(self, src, id):
self._check_tid(id, exc=StorageTransactionError)
if self.locked:
return self._abortVersion(src)
else:
return self._wait(lambda: self._abortVersion(src))
def commitVersion(self, src, dest, id):
self._check_tid(id, exc=StorageTransactionError)
if self.locked:
return self._commitVersion(src, dest)
else:
return self._wait(lambda: self._commitVersion(src, dest))
def undo(self, trans_id, id):
self._check_tid(id, exc=StorageTransactionError)
if self.locked:
return self._undo(trans_id)
else:
return self._wait(lambda: self._undo(trans_id))
def _tpc_begin(self, txn, tid, status):
self.locked = 1
self.timeout.begin(self)
self.stats.lock_time = time.time()
self.storage.tpc_begin(txn, tid, status)
def _store(self, oid, serial, data, version):
err = None
try:
newserial = self.storage.store(oid, serial, data, version,
self.transaction)
except (SystemExit, KeyboardInterrupt):
raise
except Exception, err:
self.store_failed = 1
if isinstance(err, ConflictError):
self.stats.conflicts += 1
self.log("conflict error oid=%s msg=%s" %
(oid_repr(oid), str(err)), BLATHER)
if not isinstance(err, TransactionError):
# Unexpected errors are logged and passed to the client
self.log("store error: %s, %s" % sys.exc_info()[:2],
logging.ERROR, exc_info=True)
# Try to pickle the exception. If it can't be pickled,
# the RPC response would fail, so use something else.
pickler = cPickle.Pickler()
pickler.fast = 1
try:
pickler.dump(err, 1)
except:
msg = "Couldn't pickle storage exception: %s" % repr(err)
self.log(msg, logging.ERROR)
err = StorageServerError(msg)
# The exception is reported back as newserial for this oid
newserial = err
else:
if serial != "\0\0\0\0\0\0\0\0":
self.invalidated.append((oid, version))
if newserial == ResolvedSerial:
self.stats.conflicts_resolved += 1
self.log("conflict resolved oid=%s" % oid_repr(oid), BLATHER)
self.serials.append((oid, newserial))
return err is None
def _vote(self):
self.client.serialnos(self.serials)
# If a store call failed, then return to the client immediately.
# The serialnos() call will deliver an exception that will be
# handled by the client in its tpc_vote() method.
if self.store_failed:
return
return self.storage.tpc_vote(self.transaction)
def _abortVersion(self, src):
tid, oids = self.storage.abortVersion(src, self.transaction)
inv = [(oid, src) for oid in oids]
self.invalidated.extend(inv)
return tid, oids
def _commitVersion(self, src, dest):
tid, oids = self.storage.commitVersion(src, dest, self.transaction)
inv = [(oid, dest) for oid in oids]
self.invalidated.extend(inv)
if dest:
inv = [(oid, src) for oid in oids]
self.invalidated.extend(inv)
return tid, oids
def _undo(self, trans_id):
tid, oids = self.storage.undo(trans_id, self.transaction)
inv = [(oid, None) for oid in oids]
self.invalidated.extend(inv)
return tid, oids
# When a delayed transaction is restarted, the dance is
# complicated. The restart occurs when one ZEOStorage instance
# finishes as a transaction and finds another instance is in the
# _waiting list.
# It might be better to have a mechanism to explicitly send
# the finishing transaction's reply before restarting the waiting
# transaction. If the restart takes a long time, the previous
# client will be blocked until it finishes.
def _wait(self, thunk):
# Wait for the storage lock to be acquired.
self._thunk = thunk
if self.storage._transaction:
d = Delay()
self.storage._waiting.append((d, self))
self.log("Transaction blocked waiting for storage. "
"Clients waiting: %d." % len(self.storage._waiting))
return d
else:
self.log("Transaction acquired storage lock.", BLATHER)
return self._restart()
def _restart(self, delay=None):
# Restart when the storage lock is available.
if self.txnlog.stores == 1:
template = "Preparing to commit transaction: %d object, %d bytes"
else:
template = "Preparing to commit transaction: %d objects, %d bytes"
self.log(template % (self.txnlog.stores, self.txnlog.size()),
level=BLATHER)
self._tpc_begin(self.transaction, self.tid, self.status)
loads, loader = self.txnlog.get_loader()
for i in range(loads):
# load oid, serial, data, version
if not self._store(*loader.load()):
break
# Blob support
while self.blob_log:
oid, oldserial, data, blobfilename, version = self.blob_log.pop()
self.storage.storeBlob(oid, oldserial, data, blobfilename,
version, self.transaction,)
resp = self._thunk()
if delay is not None:
delay.reply(resp)
else:
return resp
def _handle_waiting(self):
# Restart any client waiting for the storage lock.
while self.storage._waiting:
delay, zeo_storage = self.storage._waiting.pop(0)
if self._restart_other(zeo_storage, delay):
if self.storage._waiting:
n = len(self.storage._waiting)
self.log("Blocked transaction restarted. "
"Clients waiting: %d" % n)
else:
self.log("Blocked transaction restarted.")
return
def _restart_other(self, zeo_storage, delay):
# Return True if the server restarted.
# call the restart() method on the appropriate server.
try:
zeo_storage._restart(delay)
except:
self.log("Unexpected error handling waiting transaction",
level=logging.WARNING, exc_info=True)
zeo_storage.connection.close()
return 0
else:
return 1
class StorageServer:
"""The server side implementation of ZEO.
The StorageServer is the 'manager' for incoming connections. Each
connection is associated with its own ZEOStorage instance (defined
below). The StorageServer may handle multiple storages; each
ZEOStorage instance only handles a single storage.
"""
# Classes we instantiate. A subclass might override.
DispatcherClass = Dispatcher
ZEOStorageClass = ZEOStorage
ManagedServerConnectionClass = ManagedServerConnection
def __init__(self, addr, storages, read_only=0,
invalidation_queue_size=100,
transaction_timeout=None,
monitor_address=None,
auth_protocol=None,
auth_database=None,
auth_realm=None):
"""StorageServer constructor.
This is typically invoked from the start.py script.
Arguments (the first two are required and positional):
addr -- the address at which the server should listen. This
can be a tuple (host, port) to signify a TCP/IP connection
or a pathname string to signify a Unix domain socket
connection. A hostname may be a DNS name or a dotted IP
address.
storages -- a dictionary giving the storage(s) to handle. The
keys are the storage names, the values are the storage
instances, typically FileStorage or Berkeley storage
instances. By convention, storage names are typically
strings representing small integers starting at '1'.
read_only -- an optional flag saying whether the server should
operate in read-only mode. Defaults to false. Note that
even if the server is operating in writable mode,
individual storages may still be read-only. But if the
server is in read-only mode, no write operations are
allowed, even if the storages are writable. Note that
pack() is considered a read-only operation.
invalidation_queue_size -- The storage server keeps a queue
of the objects modified by the last N transactions, where
N == invalidation_queue_size. This queue is used to
speed client cache verification when a client disconnects
for a short period of time.
transaction_timeout -- The maximum amount of time to wait for
a transaction to commit after acquiring the storage lock.
If the transaction takes too long, the client connection
will be closed and the transaction aborted.
monitor_address -- The address at which the monitor server
should listen. If specified, a monitor server is started.
The monitor server provides server statistics in a simple
text format.
auth_protocol -- The name of the authentication protocol to use.
Examples are "digest" and "srp".
auth_database -- The name of the password database filename.
It should be in a format compatible with the authentication
protocol used; for instance, "sha" and "srp" require different
formats.
Note that to implement an authentication protocol, a server
and client authentication mechanism must be implemented in a
auth_* module, which should be stored inside the "auth"
subdirectory. This module may also define a DatabaseClass
variable that should indicate what database should be used
by the authenticator.
"""
self.addr = addr
self.storages = storages
set_label()
msg = ", ".join(
["%s:%s:%s" % (name, storage.isReadOnly() and "RO" or "RW",
storage.getName())
for name, storage in storages.items()])
log("%s created %s with storages: %s" %
(self.__class__.__name__, read_only and "RO" or "RW", msg))
for s in storages.values():
s._waiting = []
self.read_only = read_only
self.auth_protocol = auth_protocol
self.auth_database = auth_database
self.auth_realm = auth_realm
self.database = None
if auth_protocol:
self._setup_auth(auth_protocol)
# A list of at most invalidation_queue_size invalidations.
# The list is kept in sorted order with the most recent
# invalidation at the front. The list never has more than
# self.invq_bound elements.
self.invq = []
self.invq_bound = invalidation_queue_size
self.connections = {}
self.dispatcher = self.DispatcherClass(addr,
factory=self.new_connection)
self.stats = {}
self.timeouts = {}
for name in self.storages.keys():
self.stats[name] = StorageStats()
if transaction_timeout is None:
# An object with no-op methods
timeout = StubTimeoutThread()
else:
timeout = TimeoutThread(transaction_timeout)
timeout.start()
self.timeouts[name] = timeout
if monitor_address:
self.monitor = StatsServer(monitor_address, self.stats)
else:
self.monitor = None
def _setup_auth(self, protocol):
# Can't be done in global scope, because of cyclic references
from ZEO.auth import get_module
name = self.__class__.__name__
module = get_module(protocol)
if not module:
log("%s: no such an auth protocol: %s" % (name, protocol))
return
storage_class, client, db_class = module
if not storage_class or not issubclass(storage_class, ZEOStorage):
log(("%s: %s isn't a valid protocol, must have a StorageClass" %
(name, protocol)))
self.auth_protocol = None
return
self.ZEOStorageClass = storage_class
log("%s: using auth protocol: %s" % (name, protocol))
# We create a Database instance here for use with the authenticator
# modules. Having one instance allows it to be shared between multiple
# storages, avoiding the need to bloat each with a new authenticator
# Database that would contain the same info, and also avoiding any
# possibly synchronization issues between them.
self.database = db_class(self.auth_database)
if self.database.realm != self.auth_realm:
raise ValueError("password database realm %r "
"does not match storage realm %r"
% (self.database.realm, self.auth_realm))
def new_connection(self, sock, addr):
"""Internal: factory to create a new connection.
This is called by the Dispatcher class in ZEO.zrpc.server
whenever accept() returns a socket for a new incoming
connection.
"""
if self.auth_protocol and self.database:
zstorage = self.ZEOStorageClass(self, self.read_only,
auth_realm=self.auth_realm)
zstorage.set_database(self.database)
else:
zstorage = self.ZEOStorageClass(self, self.read_only)
c = self.ManagedServerConnectionClass(sock, addr, zstorage, self)
log("new connection %s: %s" % (addr, repr(c)))
return c
def register_connection(self, storage_id, conn):
"""Internal: register a connection with a particular storage.
This is called by ZEOStorage.register().
The dictionary self.connections maps each storage name to a
list of current connections for that storage; this information
is needed to handle invalidation. This function updates this
dictionary.
Returns the timeout and stats objects for the appropriate storage.
"""
l = self.connections.get(storage_id)
if l is None:
l = self.connections[storage_id] = []
l.append(conn)
stats = self.stats[storage_id]
stats.clients += 1
return self.timeouts[storage_id], stats
def invalidate(self, conn, storage_id, tid, invalidated=(), info=None):
"""Internal: broadcast info and invalidations to clients.
This is called from several ZEOStorage methods.
This can do three different things:
- If the invalidated argument is non-empty, it broadcasts
invalidateTransaction() messages to all clients of the given
storage except the current client (the conn argument).
- If the invalidated argument is empty and the info argument
is a non-empty dictionary, it broadcasts info() messages to
all clients of the given storage, including the current
client.
- If both the invalidated argument and the info argument are
non-empty, it broadcasts invalidateTransaction() messages to all
clients except the current, and sends an info() message to
the current client.
"""
if invalidated:
if len(self.invq) >= self.invq_bound:
self.invq.pop()
self.invq.insert(0, (tid, invalidated))
for p in self.connections.get(storage_id, ()):
if invalidated and p is not conn:
p.client.invalidateTransaction(tid, invalidated)
elif info is not None:
p.client.info(info)
def get_invalidations(self, tid):
"""Return a tid and list of all objects invalidation since tid.
The tid is the most recent transaction id seen by the client.
Returns None if it is unable to provide a complete list
of invalidations for tid. In this case, client should
do full cache verification.
"""
if not self.invq:
log("invq empty")
return None, []
earliest_tid = self.invq[-1][0]
if earliest_tid > tid:
log("tid to old for invq %s < %s" % (u64(tid), u64(earliest_tid)))
return None, []
oids = {}
for _tid, L in self.invq:
if _tid <= tid:
break
for key in L:
oids[key] = 1
latest_tid = self.invq[0][0]
return latest_tid, oids.keys()
def close_server(self):
"""Close the dispatcher so that there are no new connections.
This is only called from the test suite, AFAICT.
"""
self.dispatcher.close()
if self.monitor is not None:
self.monitor.close()
for storage in self.storages.values():
storage.close()
# Force the asyncore mainloop to exit by hackery, i.e. close
# every socket in the map. loop() will return when the map is
# empty.
for s in asyncore.socket_map.values():
try:
s.close()
except:
pass
def close_conn(self, conn):
"""Internal: remove the given connection from self.connections.
This is the inverse of register_connection().
"""
for cl in self.connections.values():
if conn.obj in cl:
cl.remove(conn.obj)
class StubTimeoutThread:
def begin(self, client):
pass
def end(self, client):
pass
class TimeoutThread(threading.Thread):
"""Monitors transaction progress and generates timeouts."""
# There is one TimeoutThread per storage, because there's one
# transaction lock per storage.
def __init__(self, timeout):
threading.Thread.__init__(self)
self.setDaemon(1)
self._timeout = timeout
self._client = None
self._deadline = None
self._cond = threading.Condition() # Protects _client and _deadline
self._trigger = trigger()
def begin(self, client):
# Called from the restart code the "main" thread, whenever the
# storage lock is being acquired. (Serialized by asyncore.)
self._cond.acquire()
try:
assert self._client is None
self._client = client
self._deadline = time.time() + self._timeout
self._cond.notify()
finally:
self._cond.release()
def end(self, client):
# Called from the "main" thread whenever the storage lock is
# being released. (Serialized by asyncore.)
self._cond.acquire()
try:
assert self._client is not None
assert self._client is client
self._client = None
self._deadline = None
finally:
self._cond.release()
def run(self):
# Code running in the thread.
while 1:
self._cond.acquire()
try:
while self._deadline is None:
self._cond.wait()
howlong = self._deadline - time.time()
if howlong <= 0:
# Prevent reporting timeout more than once
self._deadline = None
client = self._client # For the howlong <= 0 branch below
finally:
self._cond.release()
if howlong <= 0:
client.log("Transaction timeout after %s seconds" %
self._timeout)
self._trigger.pull_trigger(lambda: client.connection.close())
else:
time.sleep(howlong)
def run_in_thread(method, *args):
t = SlowMethodThread(method, args)
t.start()
return t.delay
class SlowMethodThread(threading.Thread):
"""Thread to run potentially slow storage methods.
Clients can use the delay attribute to access the MTDelay object
used to send a zrpc response at the right time.
"""
# Some storage methods can take a long time to complete. If we
# run these methods via a standard asyncore read handler, they
# will block all other server activity until they complete. To
# avoid blocking, we spawn a separate thread, return an MTDelay()
# object, and have the thread reply() when it finishes.
def __init__(self, method, args):
threading.Thread.__init__(self)
self._method = method
self._args = args
self.delay = MTDelay()
def run(self):
try:
result = self._method(*self._args)
except (SystemExit, KeyboardInterrupt):
raise
except Exception:
self.delay.error(sys.exc_info())
else:
self.delay.reply(result)
ZODB/src/ZEO/CommitLog.py 0000640 0001750 0000144 00000003205 10057366465 014666 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Log a transaction's commit info during two-phase commit.
A storage server allows multiple clients to commit transactions, but
must serialize them as the actually execute at the server. The
concurrent commits are achieved by logging actions up until the
tpc_vote(). At that point, the entire transaction is committed on the
real storage.
"""
import cPickle
import tempfile
class CommitLog:
def __init__(self):
self.file = tempfile.TemporaryFile(suffix=".log")
self.pickler = cPickle.Pickler(self.file, 1)
self.pickler.fast = 1
self.stores = 0
self.read = 0
def size(self):
return self.file.tell()
def store(self, oid, serial, data, version):
self.pickler.dump((oid, serial, data, version))
self.stores += 1
def get_loader(self):
self.read = 1
self.file.seek(0)
return self.stores, cPickle.Unpickler(self.file)
def close(self):
if self.file:
self.file.close()
self.file = None
ZODB/src/ZEO/zrpc/ 0000751 0001750 0000144 00000000000 10424632373 013370 5 ustar ctheune users ZODB/src/ZEO/zrpc/connection.py 0000640 0001750 0000144 00000100356 10375443702 016107 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import asyncore
import errno
import select
import sys
import threading
import types
import logging
import ThreadedAsync
from ZEO.zrpc import smac
from ZEO.zrpc.error import ZRPCError, DisconnectedError
from ZEO.zrpc.marshal import Marshaller
from ZEO.zrpc.trigger import trigger
from ZEO.zrpc.log import short_repr, log
from ZODB.loglevels import BLATHER, TRACE
REPLY = ".reply" # message name used for replies
ASYNC = 1
class Delay:
"""Used to delay response to client for synchronous calls.
When a synchronous call is made and the original handler returns
without handling the call, it returns a Delay object that prevents
the mainloop from sending a response.
"""
def set_sender(self, msgid, send_reply, return_error):
self.msgid = msgid
self.send_reply = send_reply
self.return_error = return_error
def reply(self, obj):
self.send_reply(self.msgid, obj)
def error(self, exc_info):
log("Error raised in delayed method", logging.ERROR, exc_info=True)
self.return_error(self.msgid, 0, *exc_info[:2])
class MTDelay(Delay):
def __init__(self):
self.ready = threading.Event()
def set_sender(self, msgid, send_reply, return_error):
Delay.set_sender(self, msgid, send_reply, return_error)
self.ready.set()
def reply(self, obj):
self.ready.wait()
Delay.reply(self, obj)
def error(self, exc_info):
self.ready.wait()
Delay.error(self, exc_info)
# PROTOCOL NEGOTIATION
#
# The code implementing protocol version 2.0.0 (which is deployed
# in the field and cannot be changed) *only* talks to peers that
# send a handshake indicating protocol version 2.0.0. In that
# version, both the client and the server immediately send out
# their protocol handshake when a connection is established,
# without waiting for their peer, and disconnect when a different
# handshake is receive.
#
# The new protocol uses this to enable new clients to talk to
# 2.0.0 servers. In the new protocol:
#
# The server sends its protocol handshake to the client at once.
#
# The client waits until it receives the server's protocol handshake
# before sending its own handshake. The client sends the lower of its
# own protocol version and the server protocol version, allowing it to
# talk to servers using later protocol versions (2.0.2 and higher) as
# well: the effective protocol used will be the lower of the client
# and server protocol. However, this changed in ZODB 3.3.1 (and
# should have changed in ZODB 3.3) because an older server doesn't
# support MVCC methods required by 3.3 clients.
#
# [Ugly details: In order to treat the first received message (protocol
# handshake) differently than all later messages, both client and server
# start by patching their message_input() method to refer to their
# recv_handshake() method instead. In addition, the client has to arrange
# to queue (delay) outgoing messages until it receives the server's
# handshake, so that the first message the client sends to the server is
# the client's handshake. This multiply-special treatment of the first
# message is delicate, and several asyncore and thread subtleties were
# handled unsafely before ZODB 3.2.6.
# ]
#
# The ZEO modules ClientStorage and ServerStub have backwards
# compatibility code for dealing with the previous version of the
# protocol. The client accepts the old version of some messages,
# and will not send new messages when talking to an old server.
#
# As long as the client hasn't sent its handshake, it can't send
# anything else; output messages are queued during this time.
# (Output can happen because the connection testing machinery can
# start sending requests before the handshake is received.)
#
# UPGRADING FROM ZEO 2.0.0 TO NEWER VERSIONS:
#
# Because a new client can talk to an old server, but not vice
# versa, all clients should be upgraded before upgrading any
# servers. Protocol upgrades beyond 2.0.1 will not have this
# restriction, because clients using protocol 2.0.1 or later can
# talk to both older and newer servers.
#
# No compatibility with protocol version 1 is provided.
# Connection is abstract (it must be derived from). ManagedServerConnection
# and ManagedClientConnection are the concrete subclasses. They need to
# supply a handshake() method appropriate for their role in protocol
# negotiation.
class Connection(smac.SizedMessageAsyncConnection, object):
"""Dispatcher for RPC on object on both sides of socket.
The connection supports synchronous calls, which expect a return,
and asynchronous calls, which do not.
It uses the Marshaller class to handle encoding and decoding of
method calls and arguments. Marshaller uses pickle to encode
arbitrary Python objects. The code here doesn't ever see the wire
format.
A Connection is designed for use in a multithreaded application,
where a synchronous call must block until a response is ready.
A socket connection between a client and a server allows either
side to invoke methods on the other side. The processes on each
end of the socket use a Connection object to manage communication.
The Connection deals with decoded RPC messages. They are
represented as four-tuples containing: msgid, flags, method name,
and a tuple of method arguments.
The msgid starts at zero and is incremented by one each time a
method call message is sent. Each side of the connection has a
separate msgid state.
When one side of the connection (the client) calls a method, it
sends a message with a new msgid. The other side (the server),
replies with a message that has the same msgid, the string
".reply" (the global variable REPLY) as the method name, and the
actual return value in the args position. Note that each side of
the Connection can initiate a call, in which case it will be the
client for that particular call.
The protocol also supports asynchronous calls. The client does
not wait for a return value for an asynchronous call. The only
defined flag is ASYNC. If a method call message has the ASYNC
flag set, the server will raise an exception.
If a method call raises an Exception, the exception is propagated
back to the client via the REPLY message. The client side will
raise any exception it receives instead of returning the value to
the caller.
"""
__super_init = smac.SizedMessageAsyncConnection.__init__
__super_close = smac.SizedMessageAsyncConnection.close
__super_setSessionKey = smac.SizedMessageAsyncConnection.setSessionKey
# Protocol history:
#
# Z200 -- Original ZEO 2.0 protocol
#
# Z201 -- Added invalidateTransaction() to client.
# Renamed several client methods.
# Added several sever methods:
# lastTransaction()
# getAuthProtocol() and scheme-specific authentication methods
# getExtensionMethods().
# getInvalidations().
#
# Z303 -- named after the ZODB release 3.3
# Added methods for MVCC:
# loadBefore()
# loadEx()
# A Z303 client cannot talk to a Z201 server, because the latter
# doesn't support MVCC. A Z201 client can talk to a Z303 server,
# but because (at least) the type of the root object changed
# from ZODB.PersistentMapping to persistent.mapping, the older
# client can't actually make progress if a Z303 client created,
# or ever modified, the root.
# Protocol variables:
# Our preferred protocol.
current_protocol = "Z303"
# If we're a client, an exhaustive list of the server protocols we
# can accept.
servers_we_can_talk_to = [current_protocol]
# If we're a server, an exhaustive list of the client protocols we
# can accept.
clients_we_can_talk_to = ["Z200", "Z201", current_protocol]
# This is pretty excruciating. Details:
#
# 3.3 server 3.2 client
# server sends Z303 to client
# client computes min(Z303, Z201) == Z201 as the protocol to use
# client sends Z201 to server
# OK, because Z201 is in the server's clients_we_can_talk_to
#
# 3.2 server 3.3 client
# server sends Z201 to client
# client computes min(Z303, Z201) == Z201 as the protocol to use
# Z201 isn't in the client's servers_we_can_talk_to, so client
# raises exception
#
# 3.3 server 3.3 client
# server sends Z303 to client
# client computes min(Z303, Z303) == Z303 as the protocol to use
# Z303 is in the client's servers_we_can_talk_to, so client
# sends Z303 to server
# OK, because Z303 is in the server's clients_we_can_talk_to
# Client constructor passes 'C' for tag, server constructor 'S'. This
# is used in log messages, and to determine whether we can speak with
# our peer.
def __init__(self, sock, addr, obj, tag):
self.obj = None
self.marshal = Marshaller()
self.closed = False
self.peer_protocol_version = None # set in recv_handshake()
assert tag in "CS"
self.tag = tag
self.logger = logging.getLogger('ZEO.zrpc.Connection(%c)' % tag)
if isinstance(addr, types.TupleType):
self.log_label = "(%s:%d) " % addr
else:
self.log_label = "(%s) " % addr
# Supply our own socket map, so that we don't get registered with
# the asyncore socket map just yet. The initial protocol messages
# are treated very specially, and we dare not get invoked by asyncore
# before that special-case setup is complete. Some of that setup
# occurs near the end of this constructor, and the rest is done by
# a concrete subclass's handshake() method. Unfortunately, because
# we ultimately derive from asyncore.dispatcher, it's not possible
# to invoke the superclass constructor without asyncore stuffing
# us into _some_ socket map.
ourmap = {}
self.__super_init(sock, addr, map=ourmap)
# A Connection either uses asyncore directly or relies on an
# asyncore mainloop running in a separate thread. If
# thr_async is true, then the mainloop is running in a
# separate thread. If thr_async is true, then the asyncore
# trigger (self.trigger) is used to notify that thread of
# activity on the current thread.
self.thr_async = False
self.trigger = None
self._prepare_async()
# The singleton dict is used in synchronous mode when a method
# needs to call into asyncore to try to force some I/O to occur.
# The singleton dict is a socket map containing only this object.
self._singleton = {self._fileno: self}
# msgid_lock guards access to msgid
self.msgid = 0
self.msgid_lock = threading.Lock()
# replies_cond is used to block when a synchronous call is
# waiting for a response
self.replies_cond = threading.Condition()
self.replies = {}
# waiting_for_reply is used internally to indicate whether
# a call is in progress. setting a session key is deferred
# until after the call returns.
self.waiting_for_reply = False
self.delay_sesskey = None
self.register_object(obj)
# The first message we see is a protocol handshake. message_input()
# is temporarily replaced by recv_handshake() to treat that message
# specially. revc_handshake() does "del self.message_input", which
# uncovers the normal message_input() method thereafter.
self.message_input = self.recv_handshake
# Server and client need to do different things for protocol
# negotiation, and handshake() is implemented differently in each.
self.handshake()
# Now it's safe to register with asyncore's socket map; it was not
# safe before message_input was replaced, or before handshake() was
# invoked.
# Obscure: in Python 2.4, the base asyncore.dispatcher class grew
# a ._map attribute, which is used instead of asyncore's global
# socket map when ._map isn't None. Because we passed `ourmap` to
# the base class constructor above, in 2.4 asyncore believes we want
# to use `ourmap` instead of the global socket map -- but we don't.
# So we have to replace our ._map with the global socket map, and
# update the global socket map with `ourmap`. Replacing our ._map
# isn't necessary before Python 2.4, but doesn't hurt then (it just
# gives us an unused attribute in 2.3); updating the global socket
# map is necessary regardless of Python version.
self._map = asyncore.socket_map
asyncore.socket_map.update(ourmap)
def __repr__(self):
return "<%s %s>" % (self.__class__.__name__, self.addr)
__str__ = __repr__ # Defeat asyncore's dreaded __getattr__
def log(self, message, level=BLATHER, exc_info=False):
self.logger.log(level, self.log_label + message, exc_info=exc_info)
def close(self):
if self.closed:
return
self._singleton.clear()
self.closed = True
self.close_trigger()
self.__super_close()
def close_trigger(self):
# Overridden by ManagedClientConnection.
if self.trigger is not None:
self.trigger.close()
def register_object(self, obj):
"""Register obj as the true object to invoke methods on."""
self.obj = obj
# Subclass must implement. handshake() is called by the constructor,
# near its end, but before self is added to asyncore's socket map.
# When a connection is created the first message sent is a 4-byte
# protocol version. This allows the protocol to evolve over time, and
# lets servers handle clients using multiple versions of the protocol.
# In general, the server's handshake() just needs to send the server's
# preferred protocol; the client's also needs to queue (delay) outgoing
# messages until it sees the handshake from the server.
def handshake(self):
raise NotImplementedError
# Replaces message_input() for the first message received. Records the
# protocol sent by the peer in `peer_protocol_version`, restores the
# normal message_input() method, and raises an exception if the peer's
# protocol is unacceptable. That's all the server needs to do. The
# client needs to do additional work in response to the server's
# handshake, and extends this method.
def recv_handshake(self, proto):
# Extended by ManagedClientConnection.
del self.message_input # uncover normal-case message_input()
self.peer_protocol_version = proto
if self.tag == 'C':
good_protos = self.servers_we_can_talk_to
else:
assert self.tag == 'S'
good_protos = self.clients_we_can_talk_to
if proto in good_protos:
self.log("received handshake %r" % proto, level=logging.INFO)
else:
self.log("bad handshake %s" % short_repr(proto),
level=logging.ERROR)
raise ZRPCError("bad handshake %r" % proto)
def message_input(self, message):
"""Decode an incoming message and dispatch it"""
# If something goes wrong during decoding, the marshaller
# will raise an exception. The exception will ultimately
# result in asycnore calling handle_error(), which will
# close the connection.
msgid, flags, name, args = self.marshal.decode(message)
if __debug__:
self.log("recv msg: %s, %s, %s, %s" % (msgid, flags, name,
short_repr(args)),
level=TRACE)
if name == REPLY:
self.handle_reply(msgid, flags, args)
else:
self.handle_request(msgid, flags, name, args)
def handle_reply(self, msgid, flags, args):
if __debug__:
self.log("recv reply: %s, %s, %s"
% (msgid, flags, short_repr(args)), level=TRACE)
self.replies_cond.acquire()
try:
self.replies[msgid] = flags, args
self.replies_cond.notifyAll()
finally:
self.replies_cond.release()
def handle_request(self, msgid, flags, name, args):
if not self.check_method(name):
msg = "Invalid method name: %s on %s" % (name, repr(self.obj))
raise ZRPCError(msg)
if __debug__:
self.log("calling %s%s" % (name, short_repr(args)),
level=logging.DEBUG)
meth = getattr(self.obj, name)
try:
self.waiting_for_reply = True
try:
ret = meth(*args)
finally:
self.waiting_for_reply = False
except (SystemExit, KeyboardInterrupt):
raise
except Exception, msg:
self.log("%s() raised exception: %s" % (name, msg), logging.INFO,
exc_info=True)
error = sys.exc_info()[:2]
return self.return_error(msgid, flags, *error)
if flags & ASYNC:
if ret is not None:
raise ZRPCError("async method %s returned value %s" %
(name, short_repr(ret)))
else:
if __debug__:
self.log("%s returns %s" % (name, short_repr(ret)),
logging.DEBUG)
if isinstance(ret, Delay):
ret.set_sender(msgid, self.send_reply, self.return_error)
else:
self.send_reply(msgid, ret)
if self.delay_sesskey:
self.__super_setSessionKey(self.delay_sesskey)
self.delay_sesskey = None
def handle_error(self):
if sys.exc_info()[0] == SystemExit:
raise sys.exc_info()
self.log("Error caught in asyncore",
level=logging.ERROR, exc_info=True)
self.close()
def check_method(self, name):
# TODO: This is hardly "secure".
if name.startswith('_'):
return None
return hasattr(self.obj, name)
def send_reply(self, msgid, ret):
# encode() can pass on a wide variety of exceptions from cPickle.
# While a bare `except` is generally poor practice, in this case
# it's acceptable -- we really do want to catch every exception
# cPickle may raise.
try:
msg = self.marshal.encode(msgid, 0, REPLY, ret)
except: # see above
try:
r = short_repr(ret)
except:
r = ""
err = ZRPCError("Couldn't pickle return %.100s" % r)
msg = self.marshal.encode(msgid, 0, REPLY, (ZRPCError, err))
self.message_output(msg)
self.poll()
def return_error(self, msgid, flags, err_type, err_value):
if flags & ASYNC:
self.log("Asynchronous call raised exception: %s" % self,
level=logging.ERROR, exc_info=True)
return
if type(err_value) is not types.InstanceType:
err_value = err_type, err_value
# encode() can pass on a wide variety of exceptions from cPickle.
# While a bare `except` is generally poor practice, in this case
# it's acceptable -- we really do want to catch every exception
# cPickle may raise.
try:
msg = self.marshal.encode(msgid, 0, REPLY, (err_type, err_value))
except: # see above
try:
r = short_repr(err_value)
except:
r = ""
err = ZRPCError("Couldn't pickle error %.100s" % r)
msg = self.marshal.encode(msgid, 0, REPLY, (ZRPCError, err))
self.message_output(msg)
self.poll()
def setSessionKey(self, key):
if self.waiting_for_reply:
self.delay_sesskey = key
else:
self.__super_setSessionKey(key)
# The next two public methods (call and callAsync) are used by
# clients to invoke methods on remote objects
def send_call(self, method, args, flags):
# send a message and return its msgid
self.msgid_lock.acquire()
try:
msgid = self.msgid
self.msgid = self.msgid + 1
finally:
self.msgid_lock.release()
if __debug__:
self.log("send msg: %d, %d, %s, ..." % (msgid, flags, method),
level=TRACE)
buf = self.marshal.encode(msgid, flags, method, args)
self.message_output(buf)
return msgid
def call(self, method, *args):
if self.closed:
raise DisconnectedError()
msgid = self.send_call(method, args, 0)
r_flags, r_args = self.wait(msgid)
if (isinstance(r_args, types.TupleType) and len(r_args) > 1
and type(r_args[0]) == types.ClassType
and issubclass(r_args[0], Exception)):
inst = r_args[1]
raise inst # error raised by server
else:
return r_args
# For testing purposes, it is useful to begin a synchronous call
# but not block waiting for its response. Since these methods are
# used for testing they can assume they are not in async mode and
# call asyncore.poll() directly to get the message out without
# also waiting for the reply.
def _deferred_call(self, method, *args):
if self.closed:
raise DisconnectedError()
msgid = self.send_call(method, args, 0)
asyncore.poll(0.01, self._singleton)
return msgid
def _deferred_wait(self, msgid):
r_flags, r_args = self.wait(msgid)
if (isinstance(r_args, types.TupleType)
and type(r_args[0]) == types.ClassType
and issubclass(r_args[0], Exception)):
inst = r_args[1]
raise inst # error raised by server
else:
return r_args
def callAsync(self, method, *args):
if self.closed:
raise DisconnectedError()
self.send_call(method, args, ASYNC)
self.poll()
def callAsyncNoPoll(self, method, *args):
# Like CallAsync but doesn't poll. This exists so that we can
# send invalidations atomically to all clients without
# allowing any client to sneak in a load request.
if self.closed:
raise DisconnectedError()
self.send_call(method, args, ASYNC)
# handle IO, possibly in async mode
def _prepare_async(self):
self.thr_async = False
ThreadedAsync.register_loop_callback(self.set_async)
# TODO: If we are not in async mode, this will cause dead
# Connections to be leaked.
def set_async(self, map):
self.trigger = trigger()
self.thr_async = True
def is_async(self):
# Overridden by ManagedConnection
if self.thr_async:
return 1
else:
return 0
def _pull_trigger(self, tryagain=10):
try:
self.trigger.pull_trigger()
except OSError:
self.trigger.close()
self.trigger = trigger()
if tryagain > 0:
self._pull_trigger(tryagain=tryagain-1)
def wait(self, msgid):
"""Invoke asyncore mainloop and wait for reply."""
if __debug__:
self.log("wait(%d), async=%d" % (msgid, self.is_async()),
level=TRACE)
if self.is_async():
self._pull_trigger()
# Delay used when we call asyncore.poll() directly.
# Start with a 1 msec delay, double until 1 sec.
delay = 0.001
self.replies_cond.acquire()
try:
while 1:
if self.closed:
raise DisconnectedError()
reply = self.replies.get(msgid)
if reply is not None:
del self.replies[msgid]
if __debug__:
self.log("wait(%d): reply=%s" %
(msgid, short_repr(reply)), level=TRACE)
return reply
if self.is_async():
self.replies_cond.wait(10.0)
else:
self.replies_cond.release()
try:
try:
if __debug__:
self.log("wait(%d): asyncore.poll(%s)" %
(msgid, delay), level=TRACE)
asyncore.poll(delay, self._singleton)
if delay < 1.0:
delay += delay
except select.error, err:
self.log("Closing. asyncore.poll() raised %s."
% err, level=BLATHER)
self.close()
finally:
self.replies_cond.acquire()
finally:
self.replies_cond.release()
def flush(self):
"""Invoke poll() until the output buffer is empty."""
if __debug__:
self.log("flush")
while self.writable():
self.poll()
def poll(self):
"""Invoke asyncore mainloop to get pending message out."""
if __debug__:
self.log("poll(), async=%d" % self.is_async(), level=TRACE)
if self.is_async():
self._pull_trigger()
else:
asyncore.poll(0.0, self._singleton)
def pending(self, timeout=0):
"""Invoke mainloop until any pending messages are handled."""
if __debug__:
self.log("pending(), async=%d" % self.is_async(), level=TRACE)
if self.is_async():
return
# Inline the asyncore poll() function to know whether any input
# was actually read. Repeat until no input is ready.
# Pending does reads and writes. In the case of server
# startup, we may need to write out zeoVerify() messages.
# Always check for read status, but don't check for write status
# only there is output to do. Only continue in this loop as
# long as there is data to read.
r = r_in = [self._fileno]
x_in = []
while r and not self.closed:
if self.writable():
w_in = [self._fileno]
else:
w_in = []
try:
r, w, x = select.select(r_in, w_in, x_in, timeout)
except select.error, err:
if err[0] == errno.EINTR:
timeout = 0
continue
else:
raise
else:
# Make sure any subsequent select does not block. The
# loop is only intended to make sure all incoming data is
# returned.
# Insecurity: What if the server sends a lot of
# invalidations, such that pending never finishes? Seems
# unlikely, but possible.
timeout = 0
if r:
try:
self.handle_read_event()
except asyncore.ExitNow:
raise
except:
self.handle_error()
if w:
try:
self.handle_write_event()
except asyncore.ExitNow:
raise
except:
self.handle_error()
class ManagedServerConnection(Connection):
"""Server-side Connection subclass."""
__super_init = Connection.__init__
__super_close = Connection.close
def __init__(self, sock, addr, obj, mgr):
self.mgr = mgr
self.__super_init(sock, addr, obj, 'S')
self.obj.notifyConnected(self)
def handshake(self):
# Send the server's preferred protocol to the client.
self.message_output(self.current_protocol)
def close(self):
self.obj.notifyDisconnected()
self.mgr.close_conn(self)
self.__super_close()
class ManagedClientConnection(Connection):
"""Client-side Connection subclass."""
__super_init = Connection.__init__
__super_close = Connection.close
base_message_output = Connection.message_output
def __init__(self, sock, addr, obj, mgr):
self.mgr = mgr
# We can't use the base smac's message_output directly because the
# client needs to queue outgoing messages until it's seen the
# initial protocol handshake from the server. So we have our own
# message_ouput() method, and support for initial queueing. This is
# a delicate design, requiring an output mutex to be wholly
# thread-safe.
# Caution: we must set this up before calling the base class
# constructor, because the latter registers us with asyncore;
# we need to guarantee that we'll queue outgoing messages before
# asyncore learns about us.
self.output_lock = threading.Lock()
self.queue_output = True
self.queued_messages = []
self.__super_init(sock, addr, obj, tag='C')
self.check_mgr_async()
# Our message_ouput() queues messages until recv_handshake() gets the
# protocol handshake from the server.
def message_output(self, message):
self.output_lock.acquire()
try:
if self.queue_output:
self.queued_messages.append(message)
else:
assert not self.queued_messages
self.base_message_output(message)
finally:
self.output_lock.release()
def handshake(self):
# The client waits to see the server's handshake. Outgoing messages
# are queued for the duration. The client will send its own
# handshake after the server's handshake is seen, in recv_handshake()
# below. It will then send any messages queued while waiting.
assert self.queue_output # the constructor already set this
def recv_handshake(self, proto):
# The protocol to use is the older of our and the server's preferred
# protocols.
proto = min(proto, self.current_protocol)
# Restore the normal message_input method, and raise an exception
# if the protocol version is too old.
Connection.recv_handshake(self, proto)
# Tell the server the protocol in use, then send any messages that
# were queued while waiting to hear the server's protocol, and stop
# queueing messages.
self.output_lock.acquire()
try:
self.base_message_output(proto)
for message in self.queued_messages:
self.base_message_output(message)
self.queued_messages = []
self.queue_output = False
finally:
self.output_lock.release()
# Defer the ThreadedAsync work to the manager.
def close_trigger(self):
# the manager should actually close the trigger
# TODO: what is that comment trying to say? What 'manager'?
del self.trigger
def set_async(self, map):
pass
def _prepare_async(self):
# Don't do the register_loop_callback that the superclass does
pass
def check_mgr_async(self):
if not self.thr_async and self.mgr.thr_async:
assert self.mgr.trigger is not None, \
"manager (%s) has no trigger" % self.mgr
self.thr_async = True
self.trigger = self.mgr.trigger
return 1
return 0
def is_async(self):
# TODO: could the check_mgr_async() be avoided on each test?
if self.thr_async:
return 1
return self.check_mgr_async()
def close(self):
self.mgr.close_conn(self)
self.__super_close()
ZODB/src/ZEO/zrpc/client.py 0000640 0001750 0000144 00000046532 10305413212 015215 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import errno
import select
import socket
import sys
import threading
import time
import types
import logging
import ThreadedAsync
from ZODB.POSException import ReadOnlyError
from ZODB.loglevels import BLATHER
from ZEO.zrpc.log import log
from ZEO.zrpc.trigger import trigger
from ZEO.zrpc.connection import ManagedClientConnection
class ConnectionManager(object):
"""Keeps a connection up over time"""
def __init__(self, addrs, client, tmin=1, tmax=180):
self.addrlist = self._parse_addrs(addrs)
self.client = client
self.tmin = tmin
self.tmax = tmax
self.cond = threading.Condition(threading.Lock())
self.connection = None # Protected by self.cond
self.closed = 0
# If thread is not None, then there is a helper thread
# attempting to connect.
self.thread = None # Protected by self.cond
self.trigger = None
self.thr_async = 0
ThreadedAsync.register_loop_callback(self.set_async)
def __repr__(self):
return "<%s for %s>" % (self.__class__.__name__, self.addrlist)
def _parse_addrs(self, addrs):
# Return a list of (addr_type, addr) pairs.
# For backwards compatibility (and simplicity?) the
# constructor accepts a single address in the addrs argument --
# a string for a Unix domain socket or a 2-tuple with a
# hostname and port. It can also accept a list of such addresses.
addr_type = self._guess_type(addrs)
if addr_type is not None:
return [(addr_type, addrs)]
else:
addrlist = []
for addr in addrs:
addr_type = self._guess_type(addr)
if addr_type is None:
raise ValueError("unknown address in list: %s" % repr(addr))
addrlist.append((addr_type, addr))
return addrlist
def _guess_type(self, addr):
if isinstance(addr, types.StringType):
return socket.AF_UNIX
if (len(addr) == 2
and isinstance(addr[0], types.StringType)
and isinstance(addr[1], types.IntType)):
return socket.AF_INET
# not anything I know about
return None
def close(self):
"""Prevent ConnectionManager from opening new connections"""
self.closed = 1
ThreadedAsync.remove_loop_callback(self.set_async)
self.cond.acquire()
try:
t = self.thread
self.thread = None
conn = self.connection
finally:
self.cond.release()
if t is not None:
log("CM.close(): stopping and joining thread")
t.stop()
t.join(30)
if t.isAlive():
log("CM.close(): self.thread.join() timed out",
level=logging.WARNING)
if conn is not None:
# This will call close_conn() below which clears self.connection
conn.close()
if self.trigger is not None:
self.trigger.close()
self.trigger = None
ThreadedAsync.remove_loop_callback(self.set_async)
def set_async(self, map):
# This is the callback registered with ThreadedAsync. The
# callback might be called multiple times, so it shouldn't
# create a trigger every time and should never do anything
# after it's closed.
# It may be that the only case where it is called multiple
# times is in the test suite, where ThreadedAsync's loop can
# be started in a child process after a fork. Regardless,
# it's good to be defensive.
# We need each connection started with async==0 to have a
# callback.
log("CM.set_async(%s)" % repr(map), level=logging.DEBUG)
if not self.closed and self.trigger is None:
log("CM.set_async(): first call")
self.trigger = trigger()
self.thr_async = 1 # needs to be set on the Connection
def attempt_connect(self):
"""Attempt a connection to the server without blocking too long.
There isn't a crisp definition for too long. When a
ClientStorage is created, it attempts to connect to the
server. If the server isn't immediately available, it can
operate from the cache. This method will start the background
connection thread and wait a little while to see if it
finishes quickly.
"""
# Will a single attempt take too long?
# Answer: it depends -- normally, you'll connect or get a
# connection refused error very quickly. Packet-eating
# firewalls and other mishaps may cause the connect to take a
# long time to time out though. It's also possible that you
# connect quickly to a slow server, and the attempt includes
# at least one roundtrip to the server (the register() call).
# But that's as fast as you can expect it to be.
self.connect()
self.cond.acquire()
try:
t = self.thread
conn = self.connection
finally:
self.cond.release()
if t is not None and conn is None:
event = t.one_attempt
event.wait()
self.cond.acquire()
try:
conn = self.connection
finally:
self.cond.release()
return conn is not None
def connect(self, sync=0):
self.cond.acquire()
try:
if self.connection is not None:
return
t = self.thread
if t is None:
log("CM.connect(): starting ConnectThread")
self.thread = t = ConnectThread(self, self.client,
self.addrlist,
self.tmin, self.tmax)
t.setDaemon(1)
t.start()
if sync:
while self.connection is None:
self.cond.wait(30)
if self.connection is None:
log("CM.connect(sync=1): still waiting...")
finally:
self.cond.release()
if sync:
assert self.connection is not None
def connect_done(self, conn, preferred):
# Called by ConnectWrapper.notify_client() after notifying the client
log("CM.connect_done(preferred=%s)" % preferred)
self.cond.acquire()
try:
self.connection = conn
if preferred:
self.thread = None
self.cond.notifyAll() # Wake up connect(sync=1)
finally:
self.cond.release()
def close_conn(self, conn):
# Called by the connection when it is closed
self.cond.acquire()
try:
if conn is not self.connection:
# Closing a non-current connection
log("CM.close_conn() non-current", level=BLATHER)
return
log("CM.close_conn()")
self.connection = None
finally:
self.cond.release()
self.client.notifyDisconnected()
if not self.closed:
self.connect()
def is_connected(self):
self.cond.acquire()
try:
return self.connection is not None
finally:
self.cond.release()
# When trying to do a connect on a non-blocking socket, some outcomes
# are expected. Set _CONNECT_IN_PROGRESS to the errno value(s) expected
# when an initial connect can't complete immediately. Set _CONNECT_OK
# to the errno value(s) expected if the connect succeeds *or* if it's
# already connected (our code can attempt redundant connects).
if hasattr(errno, "WSAEWOULDBLOCK"): # Windows
# Caution: The official Winsock docs claim that WSAEALREADY should be
# treated as yet another "in progress" indicator, but we've never
# seen this.
_CONNECT_IN_PROGRESS = (errno.WSAEWOULDBLOCK,)
# Win98: WSAEISCONN; Win2K: WSAEINVAL
_CONNECT_OK = (0, errno.WSAEISCONN, errno.WSAEINVAL)
else: # Unix
_CONNECT_IN_PROGRESS = (errno.EINPROGRESS,)
_CONNECT_OK = (0, errno.EISCONN)
class ConnectThread(threading.Thread):
"""Thread that tries to connect to server given one or more addresses.
The thread is passed a ConnectionManager and the manager's client
as arguments. It calls testConnection() on the client when a
socket connects; that should return 1 or 0 indicating whether this
is a preferred or a fallback connection. It may also raise an
exception, in which case the connection is abandoned.
The thread will continue to run, attempting connections, until a
preferred connection is seen and successfully handed over to the
manager and client.
As soon as testConnection() finds a preferred connection, or after
all sockets have been tried and at least one fallback connection
has been seen, notifyConnected(connection) is called on the client
and connect_done() on the manager. If this was a preferred
connection, the thread then exits; otherwise, it keeps trying
until it gets a preferred connection, and then reconnects the
client using that connection.
"""
__super_init = threading.Thread.__init__
# We don't expect clients to call any methods of this Thread other
# than close() and those defined by the Thread API.
def __init__(self, mgr, client, addrlist, tmin, tmax):
self.__super_init(name="Connect(%s)" % addrlist)
self.mgr = mgr
self.client = client
self.addrlist = addrlist
self.tmin = tmin
self.tmax = tmax
self.stopped = 0
self.one_attempt = threading.Event()
# A ConnectThread keeps track of whether it has finished a
# call to try_connecting(). This allows the ConnectionManager
# to make an attempt to connect right away, but not block for
# too long if the server isn't immediately available.
def stop(self):
self.stopped = 1
def run(self):
delay = self.tmin
success = 0
# Don't wait too long the first time.
# TODO: make timeout configurable?
attempt_timeout = 5
while not self.stopped:
success = self.try_connecting(attempt_timeout)
if not self.one_attempt.isSet():
self.one_attempt.set()
attempt_timeout = 75
if success > 0:
break
time.sleep(delay)
if self.mgr.is_connected():
log("CT: still trying to replace fallback connection",
level=logging.INFO)
delay = min(delay*2, self.tmax)
log("CT: exiting thread: %s" % self.getName())
def try_connecting(self, timeout):
"""Try connecting to all self.addrlist addresses.
Return 1 if a preferred connection was found; 0 if no
connection was found; and -1 if a fallback connection was
found.
If no connection is found within timeout seconds, return 0.
"""
log("CT: attempting to connect on %d sockets" % len(self.addrlist))
deadline = time.time() + timeout
wrappers = self._create_wrappers()
for wrap in wrappers.keys():
if wrap.state == "notified":
return 1
try:
if time.time() > deadline:
return 0
r = self._connect_wrappers(wrappers, deadline)
if r is not None:
return r
if time.time() > deadline:
return 0
r = self._fallback_wrappers(wrappers, deadline)
if r is not None:
return r
# Alas, no luck.
assert not wrappers
finally:
for wrap in wrappers.keys():
wrap.close()
del wrappers
return 0
def _create_wrappers(self):
# Create socket wrappers
wrappers = {} # keys are active wrappers
for domain, addr in self.addrlist:
wrap = ConnectWrapper(domain, addr, self.mgr, self.client)
wrap.connect_procedure()
if wrap.state == "notified":
for w in wrappers.keys():
w.close()
return {wrap: wrap}
if wrap.state != "closed":
wrappers[wrap] = wrap
return wrappers
def _connect_wrappers(self, wrappers, deadline):
# Next wait until they all actually connect (or fail)
# The deadline is necessary, because we'd wait forever if a
# sockets never connects or fails.
while wrappers:
if self.stopped:
for wrap in wrappers.keys():
wrap.close()
return 0
# Select connecting wrappers
connecting = [wrap
for wrap in wrappers.keys()
if wrap.state == "connecting"]
if not connecting:
break
if time.time() > deadline:
break
try:
r, w, x = select.select([], connecting, connecting, 1.0)
log("CT: select() %d, %d, %d" % tuple(map(len, (r,w,x))))
except select.error, msg:
log("CT: select failed; msg=%s" % str(msg),
level=logging.WARNING)
continue
# Exceptable wrappers are in trouble; close these suckers
for wrap in x:
log("CT: closing troubled socket %s" % str(wrap.addr))
del wrappers[wrap]
wrap.close()
# Writable sockets are connected
for wrap in w:
wrap.connect_procedure()
if wrap.state == "notified":
del wrappers[wrap] # Don't close this one
for wrap in wrappers.keys():
wrap.close()
return 1
if wrap.state == "closed":
del wrappers[wrap]
def _fallback_wrappers(self, wrappers, deadline):
# If we've got wrappers left at this point, they're fallback
# connections. Try notifying them until one succeeds.
for wrap in wrappers.keys():
assert wrap.state == "tested" and wrap.preferred == 0
if self.mgr.is_connected():
wrap.close()
else:
wrap.notify_client()
if wrap.state == "notified":
del wrappers[wrap] # Don't close this one
for wrap in wrappers.keys():
wrap.close()
return -1
assert wrap.state == "closed"
del wrappers[wrap]
# TODO: should check deadline
class ConnectWrapper:
"""An object that handles the connection procedure for one socket.
This is a little state machine with states:
closed
opened
connecting
connected
tested
notified
"""
def __init__(self, domain, addr, mgr, client):
"""Store arguments and create non-blocking socket."""
self.domain = domain
self.addr = addr
self.mgr = mgr
self.client = client
# These attributes are part of the interface
self.state = "closed"
self.sock = None
self.conn = None
self.preferred = 0
log("CW: attempt to connect to %s" % repr(addr))
try:
self.sock = socket.socket(domain, socket.SOCK_STREAM)
except socket.error, err:
log("CW: can't create socket, domain=%s: %s" % (domain, err),
level=logging.ERROR)
self.close()
return
self.sock.setblocking(0)
self.state = "opened"
def connect_procedure(self):
"""Call sock.connect_ex(addr) and interpret result."""
if self.state in ("opened", "connecting"):
try:
err = self.sock.connect_ex(self.addr)
except socket.error, msg:
log("CW: connect_ex(%r) failed: %s" % (self.addr, msg),
level=logging.ERROR)
self.close()
return
log("CW: connect_ex(%s) returned %s" %
(self.addr, errno.errorcode.get(err) or str(err)))
if err in _CONNECT_IN_PROGRESS:
self.state = "connecting"
return
if err not in _CONNECT_OK:
log("CW: error connecting to %s: %s" %
(self.addr, errno.errorcode.get(err) or str(err)),
level=logging.WARNING)
self.close()
return
self.state = "connected"
if self.state == "connected":
self.test_connection()
def test_connection(self):
"""Establish and test a connection at the zrpc level.
Call the client's testConnection(), giving the client a chance
to do app-level check of the connection.
"""
self.conn = ManagedClientConnection(self.sock, self.addr,
self.client, self.mgr)
self.sock = None # The socket is now owned by the connection
try:
self.preferred = self.client.testConnection(self.conn)
self.state = "tested"
except ReadOnlyError:
log("CW: ReadOnlyError in testConnection (%s)" % repr(self.addr))
self.close()
return
except:
log("CW: error in testConnection (%s)" % repr(self.addr),
level=logging.ERROR, exc_info=True)
self.close()
return
if self.preferred:
self.notify_client()
def notify_client(self):
"""Call the client's notifyConnected().
If this succeeds, call the manager's connect_done().
If the client is already connected, we assume it's a fallback
connection, and the new connection must be a preferred
connection. The client will close the old connection.
"""
try:
self.client.notifyConnected(self.conn)
except:
log("CW: error in notifyConnected (%s)" % repr(self.addr),
level=logging.ERROR, exc_info=True)
self.close()
return
self.state = "notified"
self.mgr.connect_done(self.conn, self.preferred)
def close(self):
"""Close the socket and reset everything."""
self.state = "closed"
self.mgr = self.client = None
self.preferred = 0
if self.conn is not None:
# Closing the ZRPC connection will eventually close the
# socket, somewhere in asyncore. Guido asks: Why do we care?
self.conn.close()
self.conn = None
if self.sock is not None:
self.sock.close()
self.sock = None
def fileno(self):
return self.sock.fileno()
ZODB/src/ZEO/zrpc/error.py 0000640 0001750 0000144 00000002073 10057366465 015105 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
from ZODB import POSException
from ZEO.Exceptions import ClientDisconnected
class ZRPCError(POSException.StorageError):
pass
class DisconnectedError(ZRPCError, ClientDisconnected):
"""The database storage is disconnected from the storage server.
The error occurred because a problem in the low-level RPC connection,
or because the connection was closed.
"""
# This subclass is raised when zrpc catches the error.
ZODB/src/ZEO/zrpc/__init__.py 0000640 0001750 0000144 00000002117 10057366465 015512 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# zrpc is a package with the following modules
# client -- manages connection creation to remote server
# connection -- object dispatcher
# log -- logging helper
# error -- exceptions raised by zrpc
# marshal -- internal, handles basic protocol issues
# server -- manages incoming connections from remote clients
# smac -- sized message async connections
# trigger -- medusa's trigger
# zrpc is not an advertised subpackage of ZEO; its interfaces are internal
ZODB/src/ZEO/zrpc/smac.py 0000640 0001750 0000144 00000027056 10214421637 014673 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Sized Message Async Connections.
This class extends the basic asyncore layer with a record-marking
layer. The message_output() method accepts an arbitrary sized string
as its argument. It sends over the wire the length of the string
encoded using struct.pack('>I') and the string itself. The receiver
passes the original string to message_input().
This layer also supports an optional message authentication code
(MAC). If a session key is present, it uses HMAC-SHA-1 to generate a
20-byte MAC. If a MAC is present, the high-order bit of the length
is set to 1 and the MAC immediately follows the length.
"""
import asyncore
import errno
try:
import hmac
except ImportError:
import _hmac as hmac
import sha
import socket
import struct
import threading
import logging
from types import StringType
from ZODB.loglevels import TRACE
from ZEO.zrpc.log import log, short_repr
from ZEO.zrpc.error import DisconnectedError
# Use the dictionary to make sure we get the minimum number of errno
# entries. We expect that EWOULDBLOCK == EAGAIN on most systems --
# or that only one is actually used.
tmp_dict = {errno.EWOULDBLOCK: 0,
errno.EAGAIN: 0,
errno.EINTR: 0,
}
expected_socket_read_errors = tuple(tmp_dict.keys())
tmp_dict = {errno.EAGAIN: 0,
errno.EWOULDBLOCK: 0,
errno.ENOBUFS: 0,
errno.EINTR: 0,
}
expected_socket_write_errors = tuple(tmp_dict.keys())
del tmp_dict
# We chose 60000 as the socket limit by looking at the largest strings
# that we could pass to send() without blocking.
SEND_SIZE = 60000
MAC_BIT = 0x80000000L
class SizedMessageAsyncConnection(asyncore.dispatcher):
__super_init = asyncore.dispatcher.__init__
__super_close = asyncore.dispatcher.close
__closed = True # Marker indicating that we're closed
socket = None # to outwit Sam's getattr
def __init__(self, sock, addr, map=None, debug=None):
self.addr = addr
if debug is not None:
self._debug = debug
elif not hasattr(self, '_debug'):
self._debug = __debug__
# __input_lock protects __inp, __input_len, __state, __msg_size
self.__input_lock = threading.Lock()
self.__inp = None # None, a single String, or a list
self.__input_len = 0
# Instance variables __state, __msg_size and __has_mac work together:
# when __state == 0:
# __msg_size == 4, and the next thing read is a message size;
# __has_mac is set according to the MAC_BIT in the header
# when __state == 1:
# __msg_size is variable, and the next thing read is a message.
# __has_mac indicates if we're in MAC mode or not (and
# therefore, if we need to check the mac header)
# The next thing read is always of length __msg_size.
# The state alternates between 0 and 1.
self.__state = 0
self.__has_mac = 0
self.__msg_size = 4
self.__output_lock = threading.Lock() # Protects __output
self.__output = []
self.__closed = False
# Each side of the connection sends and receives messages. A
# MAC is generated for each message and depends on each
# previous MAC; the state of the MAC generator depends on the
# history of operations it has performed. So the MACs must be
# generated in the same order they are verified.
# Each side is guaranteed to receive messages in the order
# they are sent, but there is no ordering constraint between
# message sends and receives. If the two sides are A and B
# and message An indicates the nth message sent by A, then
# A1 A2 B1 B2 and A1 B1 B2 A2 are both legitimate total
# orderings of the messages.
# As a result, there must be seperate MAC generators for each
# side of the connection. If not, the generator state would
# be different after A1 A2 B1 B2 than it would be after
# A1 B1 B2 A2; if the generator state was different, the MAC
# could not be verified.
self.__hmac_send = None
self.__hmac_recv = None
self.__super_init(sock, map)
def setSessionKey(self, sesskey):
log("set session key %r" % sesskey)
self.__hmac_send = hmac.HMAC(sesskey, digestmod=sha)
self.__hmac_recv = hmac.HMAC(sesskey, digestmod=sha)
def get_addr(self):
return self.addr
# TODO: avoid expensive getattr calls? Can't remember exactly what
# this comment was supposed to mean, but it has something to do
# with the way asyncore uses getattr and uses if sock:
def __nonzero__(self):
return 1
def handle_read(self):
self.__input_lock.acquire()
try:
# Use a single __inp buffer and integer indexes to make this fast.
try:
d = self.recv(8192)
except socket.error, err:
if err[0] in expected_socket_read_errors:
return
raise
if not d:
return
input_len = self.__input_len + len(d)
msg_size = self.__msg_size
state = self.__state
has_mac = self.__has_mac
inp = self.__inp
if msg_size > input_len:
if inp is None:
self.__inp = d
elif type(self.__inp) is StringType:
self.__inp = [self.__inp, d]
else:
self.__inp.append(d)
self.__input_len = input_len
return # keep waiting for more input
# load all previous input and d into single string inp
if isinstance(inp, StringType):
inp = inp + d
elif inp is None:
inp = d
else:
inp.append(d)
inp = "".join(inp)
offset = 0
while (offset + msg_size) <= input_len:
msg = inp[offset:offset + msg_size]
offset = offset + msg_size
if not state:
msg_size = struct.unpack(">I", msg)[0]
has_mac = msg_size & MAC_BIT
if has_mac:
msg_size ^= MAC_BIT
msg_size += 20
elif self.__hmac_send:
raise ValueError("Received message without MAC")
state = 1
else:
msg_size = 4
state = 0
# Obscure: We call message_input() with __input_lock
# held!!! And message_input() may end up calling
# message_output(), which has its own lock. But
# message_output() cannot call message_input(), so
# the locking order is always consistent, which
# prevents deadlock. Also, message_input() may
# take a long time, because it can cause an
# incoming call to be handled. During all this
# time, the __input_lock is held. That's a good
# thing, because it serializes incoming calls.
if has_mac:
mac = msg[:20]
msg = msg[20:]
if self.__hmac_recv:
self.__hmac_recv.update(msg)
_mac = self.__hmac_recv.digest()
if mac != _mac:
raise ValueError("MAC failed: %r != %r"
% (_mac, mac))
else:
log("Received MAC but no session key set")
elif self.__hmac_send:
raise ValueError("Received message without MAC")
self.message_input(msg)
self.__state = state
self.__has_mac = has_mac
self.__msg_size = msg_size
self.__inp = inp[offset:]
self.__input_len = input_len - offset
finally:
self.__input_lock.release()
def readable(self):
return True
def writable(self):
if len(self.__output) == 0:
return False
else:
return True
def handle_write(self):
self.__output_lock.acquire()
try:
output = self.__output
while output:
# Accumulate output into a single string so that we avoid
# multiple send() calls, but avoid accumulating too much
# data. If we send a very small string and have more data
# to send, we will likely incur delays caused by the
# unfortunate interaction between the Nagle algorithm and
# delayed acks. If we send a very large string, only a
# portion of it will actually be delivered at a time.
l = 0
for i in range(len(output)):
l += len(output[i])
if l > SEND_SIZE:
break
i += 1
# It is very unlikely that i will be 1.
v = "".join(output[:i])
del output[:i]
try:
n = self.send(v)
except socket.error, err:
if err[0] in expected_socket_write_errors:
break # we couldn't write anything
raise
if n < len(v):
output.insert(0, v[n:])
break # we can't write any more
finally:
self.__output_lock.release()
def handle_close(self):
self.close()
def message_output(self, message):
if __debug__:
if self._debug:
log("message_output %d bytes: %s hmac=%d" %
(len(message), short_repr(message),
self.__hmac_send and 1 or 0),
level=TRACE)
if self.__closed:
raise DisconnectedError(
"This action is temporarily unavailable.
")
self.__output_lock.acquire()
try:
# do two separate appends to avoid copying the message string
if self.__hmac_send:
self.__output.append(struct.pack(">I", len(message) | MAC_BIT))
self.__hmac_send.update(message)
self.__output.append(self.__hmac_send.digest())
else:
self.__output.append(struct.pack(">I", len(message)))
if len(message) <= SEND_SIZE:
self.__output.append(message)
else:
for i in range(0, len(message), SEND_SIZE):
self.__output.append(message[i:i+SEND_SIZE])
finally:
self.__output_lock.release()
def close(self):
if not self.__closed:
self.__closed = True
self.__super_close()
ZODB/src/ZEO/zrpc/log.py 0000640 0001750 0000144 00000004700 10057366465 014534 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import os
import threading
import logging
from ZODB.loglevels import BLATHER
LOG_THREAD_ID = 0 # Set this to 1 during heavy debugging
logger = logging.getLogger('ZEO.zrpc')
_label = "%s" % os.getpid()
def new_label():
global _label
_label = str(os.getpid())
def log(message, level=BLATHER, label=None, exc_info=False):
label = label or _label
if LOG_THREAD_ID:
label = label + ':' + threading.currentThread().getName()
logger.log(level, '(%s) %s' % (label, message), exc_info=exc_info)
REPR_LIMIT = 60
def short_repr(obj):
"Return an object repr limited to REPR_LIMIT bytes."
# Some of the objects being repr'd are large strings. A lot of memory
# would be wasted to repr them and then truncate, so they are treated
# specially in this function.
# Also handle short repr of a tuple containing a long string.
# This strategy works well for arguments to StorageServer methods.
# The oid is usually first and will get included in its entirety.
# The pickle is near the beginning, too, and you can often fit the
# module name in the pickle.
if isinstance(obj, str):
if len(obj) > REPR_LIMIT:
r = repr(obj[:REPR_LIMIT])
else:
r = repr(obj)
if len(r) > REPR_LIMIT:
r = r[:REPR_LIMIT-4] + '...' + r[-1]
return r
elif isinstance(obj, (list, tuple)):
elts = []
size = 0
for elt in obj:
r = short_repr(elt)
elts.append(r)
size += len(r)
if size > REPR_LIMIT:
break
if isinstance(obj, tuple):
r = "(%s)" % (", ".join(elts))
else:
r = "[%s]" % (", ".join(elts))
else:
r = repr(obj)
if len(r) > REPR_LIMIT:
return r[:REPR_LIMIT] + '...'
else:
return r
ZODB/src/ZEO/zrpc/trigger.py 0000640 0001750 0000144 00000020305 10273501124 015374 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001-2005 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import asyncore
import os
import socket
import thread
import errno
from ZODB.utils import positive_id
# Original comments follow; they're hard to follow in the context of
# ZEO's use of triggers. TODO: rewrite from a ZEO perspective.
# Wake up a call to select() running in the main thread.
#
# This is useful in a context where you are using Medusa's I/O
# subsystem to deliver data, but the data is generated by another
# thread. Normally, if Medusa is in the middle of a call to
# select(), new output data generated by another thread will have
# to sit until the call to select() either times out or returns.
# If the trigger is 'pulled' by another thread, it should immediately
# generate a READ event on the trigger object, which will force the
# select() invocation to return.
#
# A common use for this facility: letting Medusa manage I/O for a
# large number of connections; but routing each request through a
# thread chosen from a fixed-size thread pool. When a thread is
# acquired, a transaction is performed, but output data is
# accumulated into buffers that will be emptied more efficiently
# by Medusa. [picture a server that can process database queries
# rapidly, but doesn't want to tie up threads waiting to send data
# to low-bandwidth connections]
#
# The other major feature provided by this class is the ability to
# move work back into the main thread: if you call pull_trigger()
# with a thunk argument, when select() wakes up and receives the
# event it will call your thunk from within that thread. The main
# purpose of this is to remove the need to wrap thread locks around
# Medusa's data structures, which normally do not need them. [To see
# why this is true, imagine this scenario: A thread tries to push some
# new data onto a channel's outgoing data queue at the same time that
# the main thread is trying to remove some]
class _triggerbase(object):
"""OS-independent base class for OS-dependent trigger class."""
kind = None # subclass must set to "pipe" or "loopback"; used by repr
def __init__(self):
self._closed = False
# `lock` protects the `thunks` list from being traversed and
# appended to simultaneously.
self.lock = thread.allocate_lock()
# List of no-argument callbacks to invoke when the trigger is
# pulled. These run in the thread running the asyncore mainloop,
# regardless of which thread pulls the trigger.
self.thunks = []
def readable(self):
return 1
def writable(self):
return 0
def handle_connect(self):
pass
def handle_close(self):
self.close()
# Override the asyncore close() method, because it doesn't know about
# (so can't close) all the gimmicks we have open. Subclass must
# supply a _close() method to do platform-specific closing work. _close()
# will be called iff we're not already closed.
def close(self):
if not self._closed:
self._closed = True
self.del_channel()
self._close() # subclass does OS-specific stuff
def _close(self): # see close() above; subclass must supply
raise NotImplementedError
def pull_trigger(self, thunk=None):
if thunk:
self.lock.acquire()
try:
self.thunks.append(thunk)
finally:
self.lock.release()
self._physical_pull()
# Subclass must supply _physical_pull, which does whatever the OS
# needs to do to provoke the "write" end of the trigger.
def _physical_pull(self):
raise NotImplementedError
def handle_read(self):
try:
self.recv(8192)
except socket.error:
return
self.lock.acquire()
try:
for thunk in self.thunks:
try:
thunk()
except:
nil, t, v, tbinfo = asyncore.compact_traceback()
print ('exception in trigger thunk:'
' (%s:%s %s)' % (t, v, tbinfo))
self.thunks = []
finally:
self.lock.release()
def __repr__(self):
return '' % (self.kind, positive_id(self))
if os.name == 'posix':
class trigger(_triggerbase, asyncore.file_dispatcher):
kind = "pipe"
def __init__(self):
_triggerbase.__init__(self)
r, self.trigger = self._fds = os.pipe()
asyncore.file_dispatcher.__init__(self, r)
def _close(self):
for fd in self._fds:
os.close(fd)
self._fds = []
def _physical_pull(self):
os.write(self.trigger, 'x')
else:
# Windows version; uses just sockets, because a pipe isn't select'able
# on Windows.
class trigger(_triggerbase, asyncore.dispatcher):
kind = "loopback"
def __init__(self):
_triggerbase.__init__(self)
# Get a pair of connected sockets. The trigger is the 'w'
# end of the pair, which is connected to 'r'. 'r' is put
# in the asyncore socket map. "pulling the trigger" then
# means writing something on w, which will wake up r.
w = socket.socket()
# Disable buffering -- pulling the trigger sends 1 byte,
# and we want that sent immediately, to wake up asyncore's
# select() ASAP.
w.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
count = 0
while 1:
count += 1
# Bind to a local port; for efficiency, let the OS pick
# a free port for us.
# Unfortunately, stress tests showed that we may not
# be able to connect to that port ("Address already in
# use") despite that the OS picked it. This appears
# to be a race bug in the Windows socket implementation.
# So we loop until a connect() succeeds (almost always
# on the first try). See the long thread at
# http://mail.zope.org/pipermail/zope/2005-July/160433.html
# for hideous details.
a = socket.socket()
a.bind(("127.0.0.1", 0))
connect_address = a.getsockname() # assigned (host, port) pair
a.listen(1)
try:
w.connect(connect_address)
break # success
except socket.error, detail:
if detail[0] != errno.WSAEADDRINUSE:
# "Address already in use" is the only error
# I've seen on two WinXP Pro SP2 boxes, under
# Pythons 2.3.5 and 2.4.1.
raise
# (10048, 'Address already in use')
# assert count <= 2 # never triggered in Tim's tests
if count >= 10: # I've never seen it go above 2
a.close()
w.close()
raise BindError("Cannot bind trigger!")
# Close `a` and try again. Note: I originally put a short
# sleep() here, but it didn't appear to help or hurt.
a.close()
r, addr = a.accept() # r becomes asyncore's (self.)socket
a.close()
self.trigger = w
asyncore.dispatcher.__init__(self, r)
def _close(self):
# self.socket is r, and self.trigger is w, from __init__
self.socket.close()
self.trigger.close()
def _physical_pull(self):
self.trigger.send('x')
ZODB/src/ZEO/zrpc/server.py 0000640 0001750 0000144 00000003565 10057366465 015271 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import asyncore
import socket
import types
from ZEO.zrpc.connection import Connection
from ZEO.zrpc.log import log
import logging
import ThreadedAsync.LoopCallback
# Export the main asyncore loop
loop = ThreadedAsync.LoopCallback.loop
class Dispatcher(asyncore.dispatcher):
"""A server that accepts incoming RPC connections"""
__super_init = asyncore.dispatcher.__init__
def __init__(self, addr, factory=Connection):
self.__super_init()
self.addr = addr
self.factory = factory
self._open_socket()
def _open_socket(self):
if type(self.addr) == types.TupleType:
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
else:
self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.set_reuse_addr()
log("listening on %s" % str(self.addr), logging.INFO)
self.bind(self.addr)
self.listen(5)
def writable(self):
return 0
def readable(self):
return 1
def handle_accept(self):
try:
sock, addr = self.accept()
except socket.error, msg:
log("accepted failed: %s" % msg)
return
c = self.factory(sock, addr)
log("connect from %s: %s" % (repr(addr), c))
ZODB/src/ZEO/zrpc/_hmac.py 0000640 0001750 0000144 00000006404 10305413212 015000 0 ustar ctheune users # This file is a slightly modified copy of Python 2.3's Lib/hmac.py.
# This file is under the Python Software Foundation (PSF) license.
"""HMAC (Keyed-Hashing for Message Authentication) Python module.
Implements the HMAC algorithm as described by RFC 2104.
"""
def _strxor(s1, s2):
"""Utility method. XOR the two strings s1 and s2 (must have same length).
"""
return "".join(map(lambda x, y: chr(ord(x) ^ ord(y)), s1, s2))
# The size of the digests returned by HMAC depends on the underlying
# hashing module used.
digest_size = None
class HMAC:
"""RFC2104 HMAC class.
This supports the API for Cryptographic Hash Functions (PEP 247).
"""
def __init__(self, key, msg = None, digestmod = None):
"""Create a new HMAC object.
key: key for the keyed hash object.
msg: Initial input for the hash, if provided.
digestmod: A module supporting PEP 247. Defaults to the md5 module.
"""
if digestmod is None:
import md5
digestmod = md5
self.digestmod = digestmod
self.outer = digestmod.new()
self.inner = digestmod.new()
# Python 2.1 and 2.2 differ about the correct spelling
try:
self.digest_size = digestmod.digestsize
except AttributeError:
self.digest_size = digestmod.digest_size
blocksize = 64
ipad = "\x36" * blocksize
opad = "\x5C" * blocksize
if len(key) > blocksize:
key = digestmod.new(key).digest()
key = key + chr(0) * (blocksize - len(key))
self.outer.update(_strxor(key, opad))
self.inner.update(_strxor(key, ipad))
if msg is not None:
self.update(msg)
## def clear(self):
## raise NotImplementedError("clear() method not available in HMAC.")
def update(self, msg):
"""Update this hashing object with the string msg.
"""
self.inner.update(msg)
def copy(self):
"""Return a separate copy of this hashing object.
An update to this copy won't affect the original object.
"""
other = HMAC("")
other.digestmod = self.digestmod
other.inner = self.inner.copy()
other.outer = self.outer.copy()
return other
def digest(self):
"""Return the hash value of this hashing object.
This returns a string containing 8-bit data. The object is
not altered in any way by this function; you can continue
updating the object after calling this function.
"""
h = self.outer.copy()
h.update(self.inner.digest())
return h.digest()
def hexdigest(self):
"""Like digest(), but returns a string of hexadecimal digits instead.
"""
return "".join([hex(ord(x))[2:].zfill(2)
for x in tuple(self.digest())])
def new(key, msg = None, digestmod = None):
"""Create a new hashing object and return it.
key: The starting key for the hash.
msg: if available, will immediately be hashed into the object's starting
state.
You can now feed arbitrary strings into the object using its update()
method, and can ask for the hash value at any time by calling its digest()
method.
"""
return HMAC(key, msg, digestmod)
ZODB/src/ZEO/zrpc/marshal.py 0000640 0001750 0000144 00000005617 10214421637 015376 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import cPickle
from cStringIO import StringIO
import types
import logging
from ZEO.zrpc.error import ZRPCError
from ZEO.zrpc.log import log, short_repr
class Marshaller:
"""Marshal requests and replies to second across network"""
def encode(self, msgid, flags, name, args):
"""Returns an encoded message"""
# (We used to have a global pickler, but that's not thread-safe. :-( )
# Note that args may contain very large binary pickles already; for
# this reason, it's important to use proto 1 (or higher) pickles here
# too. For a long time, this used proto 0 pickles, and that can
# bloat our pickle to 4x the size (due to high-bit and control bytes
# being represented by \xij escapes in proto 0).
# Undocumented: cPickle.Pickler accepts a lone protocol argument;
# pickle.py does not.
pickler = cPickle.Pickler(1)
pickler.fast = 1
# Undocumented: pickler.dump(), for a cPickle.Pickler, takes
# an optional boolean argument. When true, it returns the pickle;
# when false or unspecified, it returns the pickler object itself.
# pickle.py does none of this.
return pickler.dump((msgid, flags, name, args), 1)
def decode(self, msg):
"""Decodes msg and returns its parts"""
unpickler = cPickle.Unpickler(StringIO(msg))
unpickler.find_global = find_global
try:
return unpickler.load() # msgid, flags, name, args
except:
log("can't decode message: %s" % short_repr(msg),
level=logging.ERROR)
raise
_globals = globals()
_silly = ('__doc__',)
def find_global(module, name):
"""Helper for message unpickler"""
try:
m = __import__(module, _globals, _globals, _silly)
except ImportError, msg:
raise ZRPCError("import error %s: %s" % (module, msg))
try:
r = getattr(m, name)
except AttributeError:
raise ZRPCError("module %s has no global %s" % (module, name))
safe = getattr(r, '__no_side_effects__', 0)
if safe:
return r
# TODO: is there a better way to do this?
if type(r) == types.ClassType and issubclass(r, Exception):
return r
raise ZRPCError("Unsafe global: %s.%s" % (module, name))
ZODB/src/ZEO/DEPENDENCIES.cfg 0000640 0001750 0000144 00000000101 10050174114 014676 0 ustar ctheune users BTrees
ThreadedAsync
ZConfig
ZODB
persistent
transaction
zdaemon
ZODB/src/ZEO/__init__.py 0000640 0001750 0000144 00000001630 10375443702 014524 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""ZEO -- Zope Enterprise Objects.
See the file README.txt in this directory for an overview.
ZEO is now part of ZODB; ZODB's home on the web is
http://www.zope.org/Wikis/ZODB
"""
# The next line must use double quotes, so release.py recognizes it.
version = "3.7.0a0"
ZODB/src/ZEO/zeoctl.py 0000640 0001750 0000144 00000002003 10307651713 014256 0 ustar ctheune users #!/usr/bin/env python2.3
##############################################################################
#
# Copyright (c) 2005 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Wrapper script for zdctl.py that causes it to use the ZEO schema."""
import os
import ZEO
import zdaemon.zdctl
# Main program
def main(args=None):
options = zdaemon.zdctl.ZDCtlOptions()
options.schemadir = os.path.dirname(ZEO.__file__)
options.schemafile = "zeoctl.xml"
zdaemon.zdctl.main(args, options)
if __name__ == "__main__":
main()
ZODB/src/ZEO/cache.py 0000640 0001750 0000144 00000124271 10275731260 014035 0 ustar ctheune users ##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Disk-based client cache for ZEO.
ClientCache exposes an API used by the ZEO client storage. FileCache stores
objects on disk using a 2-tuple of oid and tid as key.
ClientCache's API is similar to a storage API, with methods like load(),
store(), and invalidate(). It manages in-memory data structures that allow
it to map this richer API onto the simple key-based API of the lower-level
FileCache.
"""
import bisect
import logging
import os
import struct
import tempfile
import time
from ZODB.utils import z64, u64
logger = logging.getLogger("ZEO.cache")
##
# A disk-based cache for ZEO clients.
#