# -*- test-case-name: mv3d.test.util.test_persist -*-
# Copyright (C) 2010-2012 Mortal Coil Games
# See LICENSE for details.
"""
Persist is an ORM or object store that uses pluggable back ends. Classes that
inherit from Persistable can define Attributes which will be stored. The
object store is fully queryable and keeps a cache of loaded items. See the
unit tests in mv3d.test.util.test_persist for example usage.

@author:  mike
"""
import uuid
import sys
import operator
from fnmatch import fnmatch
from twisted.internet import reactor
from mv3d.util.profiler import timed
from twisted.python.util import mergeFunctionMetadata
from twisted.internet.defer import Deferred
from twisted.python.log import deferr
from mv3d.util import getmembers
try:
    import cPickle as pickle
except ImportError:
    import pickle


def getClass(path):
    """
    Assumes path is a module and class and will import the module and return
    the class

    * Sadly copied from mv3d.util.classgen due to circular import
    """
    moduleName = ".".join(path.split(".")[:-1])
    className = path.split(".")[-1]
    return getattr(__import__(moduleName, fromlist=[className]), className)



def autoStore(func):
    """
    Automatically store this object after the completion of the wrapped function
    if the wrapped function returns a deferred, wait for it to complete before
    saving.
    If the object didn't come from or hasn't been saved in a store, it won't
    be saved now.
    """
    def wrapper(self, *args, **kw):
        d = func(self, *args, **kw)
        def done(r):
            if hasattr(self, "save") and hasattr(self, "store"):
                if AutoAttrib.saveQueue is not None:
                    try:
                        AutoAttrib.saveQueue.remove((self, True))
                    except ValueError:
                        pass
                    if not (self, False) in AutoAttrib.saveQueue:
                        AutoAttrib.saveQueue.append((self, False))
                        if AutoAttrib.saveCall is None:
                            AutoAttrib.saveCall = reactor.callLater(0, #@UndefinedVariable
                                AutoAttrib._runQueue)
                else:
                    self.save()
            return r
        if isinstance(d, Deferred):
            return d.addCallback(done)
        return done(d)
    mergeFunctionMetadata(func, wrapper)
    return wrapper



class UpgradeError(Exception):
    """
    Raised when there's a problem upgrading.
    """
    def __init__(self, oldVersion, newVersion, message):
        Exception.__init__(self, "Upgrade %d to %d: %s" % (oldVersion,
            newVersion, message))


def convertAutoSave(state):
    """
    Removes "_auto_" from the start of any keys in the state dict.
    """
    fix = []
    for key in state.keys():
        if key.startswith("_auto_"):
            fix.append(key)
    for key in fix:
        state[key[6:]] = state[key]
        del state[key]


class AttributeComparison(object):
    """
    Defines a comparison between two attributes
    """
    def __init__(self, left, operator, right):
        self.left = left
        self.operator = operator
        self.right = right


    def __and__(self, other):
        return AttributeComparison(self, " and ", other)


    def __or__(self, other):
        return AttributeComparison(self, " or ", other)


    def build(self):
        """
        Recursively build the set of tuples to pass to the store for this
        comparison.
        """
        if isinstance(self.left, (AttributeComparison, Attribute)):
            left = self.left.build()
        else:
            left = self.left
        if isinstance(self.right, (AttributeComparison, Attribute)):
            right = self.right.build()
        else:
            right = self.right
        return (left, self.operator, right)


    def __str__(self):
        """
        Nicely stringify this
        """
        if isinstance(self.left, (Attribute, AttributeComparison)):
            left = str(self.left)
        else:
            left = repr(self.left)
        if isinstance(self.right, (Attribute, AttributeComparison)):
            right = str(self.right)
        else:
            right = repr(self.right)
        return "%s %s %s" % (left, self.operator, right)



class Attribute(object):
    """
    Defines a SQL attribute that can be persisted.

    Constructor parameters:
        isID - defines whether this attribute is an ID attribute (False)
        autoSave - if True, whenever set, attribute will cause the object to
            save. (False)
        partialSave - if True, when used in conjunction with autoSave,
            only dirty attributes (and this one) will be autoSaved. (False)
        autoDirty - if True, whenever set, attribute will be marked dirty for
            the next partial save. (False)
        createIndex - when set, the attribute will be indexed in the database
            for faster querying on it. (False normally, but True if isID)
        transmit - the attribute should be sent over the network as part of
            a Cacheable, Copyable, or HighlyAvailable object. (False)
        transient - don't actually save the attribute in the database. (False)
        editDataType - defines what type of gui editor this field should use in
            a guide PropertyGrid widget. Values are from the PropertyDataTypes
            enum. If not specified, the field type will be guessed from the
            attribute type.
    """
    def __init__(self, isID=False, autoSave=False, partialSave=False,
            autoDirty=False, createIndex=False, transmit=False,
            transient=False, editDataType=None):
        # TODO: transmit defaults to true or false?
        self.isID = isID
        self.autoSave = autoSave
        self.partialSave = partialSave
        self.autoDirty = autoDirty
        self.createIndex = createIndex
        if isID:
            self.createIndex = True
        self.transmit = transmit
        self.transient = transient
        self.editDataType = editDataType
        self.schema = {}
        self.schema["type"] = self.__class__.__name__
        self.schema["index"] = self.createIndex


    def __eq__(self, other):
        if isinstance(other, Attribute):
            other = other.schema["name"]
        else:
            other = self.toStore(None, other, None)
        return AttributeComparison(self, "=", other)


    def __lt__(self, other):
        if isinstance(other, Attribute):
            other = other.schema["name"]
        else:
            other = self.toStore(None, other, None)
        return AttributeComparison(self, "<", other)


    def __gt__(self, other):
        if isinstance(other, Attribute):
            other = other.schema["name"]
        else:
            other = self.toStore(None, other, None)
        return AttributeComparison(self, ">", other)


    def __le__(self, other):
        if isinstance(other, Attribute):
            other = other.schema["name"]
        else:
            other = self.toStore(None, other, None)
        return AttributeComparison(self, "<=", other)


    def __ge__(self, other):
        if isinstance(other, Attribute):
            other = other.schema["name"]
        else:
            other = self.toStore(None, other, None)
        return AttributeComparison(self, ">=", other)


    def __ne__(self, other):
        if isinstance(other, Attribute):
            other = other.schema["name"]
        else:
            other = self.toStore(None, other, None)
        return AttributeComparison(self, "!=", other)


    def __xor__(self, other):
        if isinstance(other, Attribute):
            other = other.schema["name"]
        else:
            other = self.toStore(None, other, None)
        return AttributeComparison(self, " glob ", other)


    def __str__(self):
        """
        Nicely stringify
        """
        return "%s.%s" % (self.schema["owner"], self.schema["name"])


    asc = property(lambda self: (self.schema["name"], "asc"))
    desc = property(lambda self: (self.schema["name"], "desc"))


    def installingOn(self, store, transactionID=None):
        """
        Called when the owner of the attribute is being installed on the store. 
        """


    def toStore(self, store, pythonObject, transactionID=None):
        """
        Called when a python value needs to be converted to a format that can
        be stored.
        """


    def fromStore(self, storeObject):
        """
        Called when a value retrieved from the store needs to be converted back
        into a python value.
        """


    def setStore(self, store, data):
        """
        Override to set the store on any attribute types that need to do that.
        """


    def delete(self, store, data, transactionID=None):
        """
        Called when the object is deleted from the store.
        """


    def copy(self):
        """
        Make a copy!
        """
        raise NotImplementedError("Implement copy for %s" % self.__class__)


    def build(self):
        """
        Build a query
        """
        return self.schema["name"]


class MapAttribute(object):
    """
    Defines an attribute that is actually a map. This does not preserve order.
    """
    attributeName = None
    owner = None
    ownerKey = None
    simple = True

    def __init__(self, keyType, valueType, transmit=False, transient=False,
            editDataType=None):
        self.keyType = keyType.copy()
        self.keyType.schema["name"] = "key"
        self.valueType = valueType.copy()
        if isinstance(self.valueType, Attribute):
            self.valueType.schema["name"] = "value"
        elif isinstance(self.valueType, MapAttribute):
            self.valueType.setOffset(1)
            self.simple = False
        self.transmit = transmit
        self.transient = transient
        self.editDataType = editDataType


    def setOwner(self, owner):
        """
        Set the owner.
        """
        self.owner = owner
        if isinstance(self.valueType, MapAttribute):
            self.valueType.setOwner(owner)


    def setAttributeName(self, attributeName):
        """
        Set the attributeName.
        """
        self.attributeName = attributeName
        if isinstance(self.valueType, MapAttribute):
            self.valueType.setAttributeName(attributeName)


    def setOffset(self, offset):
        """
        Used with a map inside a map
        """
        self.keyType.schema["name"] += str(offset)
        if isinstance(self.valueType, Attribute):
            self.valueType.schema["name"] += str(offset)
        elif isinstance(self.valueType, MapAttribute):
            self.valueType.setOffset(offset + 1)


    def getTypeName(self):
        """
        Returns the composite typename for our generated schema.
        """
        return "%s_%s" % (self.owner, self.attributeName)


    def toStore(self, store, ownerID, pythonObject, owner, attrs=None,
            transactionID=None):
        """
        Intelligently update all items in the join table based on the previously
        stored value.
        """
        if attrs is None:
            attrs = {}
        attrs["owner"] = ownerID
        if pythonObject is None:
            return
        if self.simple:
            keys = [value for key, value in attrs.items() if key != "owner"]
            if not keys:
                stateAttrName = "_" + self.attributeName + "_previousState"
            else:
                stateAttrName = "_" + self.attributeName + "_" + "_".join(
                    keys) + "_previousState"
            if owner._storedOn is not None and store in owner._storedOn:
                previousState = getattr(owner, stateAttrName, set([]))
            else:
                previousState = set([])
            currentState = []
            try:
                items = pythonObject.items()
            except AttributeError:
                # TODO: FAIL! This is here to reduce fallout a major bug found
                # where mapAttributes in parent classes were being lost.
                return
            for key, value in items:
                newAttrs = attrs.copy()
                newAttrs[self.keyType.schema["name"]] = self.keyType.toStore(
                    store, key, transactionID)
                newAttrs[self.valueType.schema["name"]] = self.valueType.toStore(
                    store, value, transactionID)
                currentState.append(tuple(newAttrs.items()))
            currentState = set(currentState)
            adds = currentState - previousState
            deletes = previousState - currentState
            for delete in deletes:
                params = [(key, "=", value) for key, value in delete]
                while len(params) > 1:
                    odd = len(params) % 2 == 1
                    oldLast = params[-1]
                    params = [(params[x], " and ", params[x + 1]) for x in range(0, len(params) - 1, 2)]
                    if odd:
                        params[-1] = (params[-1], " and ", oldLast)
                store.delete(self.getTypeName(), params[0],
                    transactionID=transactionID)
            for add in adds:
                store.new(self.getTypeName(), dict(add),
                    transactionID=transactionID)
            setattr(owner, stateAttrName, currentState)
        else:
            try:
                items = pythonObject.items()
            except AttributeError:
                # TODO: FAIL! This is here to reduce fallout a major bug found
                # where mapAttributes in parent classes were being lost.
                return
            for key, value in items:
                newAttrs = attrs.copy()
                newAttrs[self.keyType.schema["name"]] = self.keyType.toStore(
                    store, key, transactionID)
                self.valueType.owner = self.owner
                self.valueType.attributeName = self.attributeName
                self.valueType.toStore(store, ownerID, value, owner, newAttrs,
                    transactionID)


    def getFields(self):
        """
        Retrieves the fields we use
        """
        fields = [self.keyType.schema["name"]]
        if self.simple:
            fields.append(self.valueType.schema["name"])
        else:
            fields.extend(self.valueType.getFields())
        return fields


    def processResult(self, store, output, resultDict):
        """
        For a given result dict, insert a value into output corresponding to
        where in the hierarchy it goes.
        """
        keyName = self.keyType.schema["name"]
        key = resultDict[keyName]
        if self.simple:
            output[self.keyType.fromStore(
                store, key)] = self.valueType.fromStore(store,
                    resultDict[self.valueType.schema["name"]])
        else:
            if not output.has_key(key):
                output[key] = {}
            self.valueType.processResult(store, output[key], resultDict)


    def fromStore(self, store, ownerID, owner):
        """
        Retrieve all the items that match our ownerID from the join table.
        Then convert them to the correct type.
        """
        fields = self.getFields()
        results = store.query(self.getTypeName(), ("owner", "=", ownerID),
            fields)
        dicty = {}
        newState = []
        for result in results:
            self.processResult(store, dicty, result)
            newState.append([(key, value) for key, value in result.items() if key != "_schemaVersion"])
            newState[-1].insert(0, ('owner', ownerID))
            newState[-1] = tuple(newState[-1])
        setattr(owner, "_" + self.attributeName + "_previousState",
            set(newState))
        if not self.simple:
            dict2 = {}
            for key, value in dicty.iteritems():
                dict2[key] = self.valueType.finalize(value)
            return dict2
        return dicty


    def setStore(self, store, data):
        """
        Sets the store on the key and the value.
        """
        for key, value in data.items():
            self.keyType.setStore(store, key)
            self.valueType.setStore(store, value)


    def finalize(self, value):
        """
        Do any conversion needed after loading
        """
        return value


    def installingOn(self, store, transactionID=None):
        """
        When the attribute is installed, we need to register a new schema
        to store the values in this map (basically a join table)
        """
        store.registerSchema(self.getTypeName(), 1, self.getSchema(),
            transactionID=transactionID)


    def delete(self, store, data, ownerID, transactionID=None):
        """
        Called when the store is deleting our owner.
        """
        store.delete(self.getTypeName(), ("owner", "=", ownerID),
            transactionID=transactionID)
        for key, value in data.items():
            if isinstance(self.keyType, MapAttribute):
                self.keyType.delete(store, key, ownerID, transactionID)
            else:
                self.keyType.delete(store, key, transactionID)
            if isinstance(self.valueType, MapAttribute):
                self.valueType.delete(store, value, ownerID, transactionID)
            else:
                self.valueType.delete(store, value, transactionID)


    def getSchema(self):
        """
        Returns the schema needed to create this map in the store
        """
        attrs = []
        if self.ownerKey is not None:
            ownerSchema = self.ownerKey.schema.copy()
            ownerSchema["name"] = "owner"
            ownerSchema["index"] = True
            attrs.append(ownerSchema)
        if isinstance(self.valueType, Attribute):
            attrs += [self.keyType.schema, self.valueType.schema]
        elif isinstance(self.valueType, MapAttribute):
            attrs.append(self.keyType.schema)
            attrs.extend(self.valueType.getSchema())
        return attrs


    def copy(self):
        """
        Make a copy!
        """
        copy = MapAttribute(self.keyType, self.valueType, self.transmit,
            self.transient)
        copy.attributeName = self.attributeName
        copy.owner = self.owner
        copy.ownerKey = self.ownerKey
        copy.simple = self.simple
        return copy



class AttributeProperty(Attribute, property):
    """
    Allows the definition of a persistable attribute which is generated from
    a get/set method
    """
    def __init__(self, attrib, fget, fset=None, fdel=None, doc=None):
        property.__init__(self, fget, fset, fdel, doc)
        self.attrib = attrib
        self.schema = attrib.schema
        self.isID = attrib.isID
        self.autoSave = attrib.autoSave
        self.autoDirty = attrib.autoDirty
        self.partialSave = attrib.partialSave
        self.transient = attrib.transient
        self.transmit = attrib.transmit


    def installingOn(self, store, transactionID=None):
        """
        Called when the owner of the attribute is being installed on the store. 
        """
        return self.attrib.installingOn(store, transactionID=transactionID)


    def toStore(self, store, pythonObject, transactionID=None):
        """
        Called when a python value needs to be converted to a format that can
        be stored.
        """
        return self.attrib.toStore(store, pythonObject, transactionID)


    def fromStore(self, store, ownerID):
        """
        Retrieve all the items that match our ownerID from the join table.
        Then convert them to the correct type.
        """
        return self.attrib.fromStore(store, ownerID)


    def setStore(self, store, data):
        """
        Sets the store on the attrib.
        """
        self.attrib.setStore(store, data)


    def copy(self):
        """
        Make a copy!
        """
        return AttributeProperty(self.attrib, self.fget, self.fset, self.fdel)



class MapAttributeProperty(MapAttribute, property):
    """
    Allows the definition of a persistable attribute which is generated from
    a get/set method
    """
    def __init__(self, attrib, fget, fset=None, fdel=None, doc=None):
        property.__init__(self, fget, fset, fdel, doc)
        self.attrib = attrib


    def getOwner(self):
        return self.attrib.owner


    def setOwner(self, newOwner):
        self.attrib.owner = newOwner


    owner = property(getOwner, setOwner)


    def _getOwnerKey(self):
        return self.attrib.ownerKey


    def _setOwnerKey(self, newOwnerKey):
        self.attrib.ownerKey = newOwnerKey


    ownerKey = property(_getOwnerKey, _setOwnerKey)


    def getAttributeName(self):
        return self.attrib.attributeName


    def setAttributeName(self, newAttributeName):
        self.attrib.attributeName = newAttributeName


    attributeName = property(getAttributeName, setAttributeName)


    def _getTransmit(self):
        return self.attrib.transmit


    def _setTransmit(self, value):
        self.attrib.transmit = value


    transmit = property(_getTransmit, _setTransmit)


    def _getTransient(self):
        return self.attrib.transient


    def _setTransient(self, value):
        self.attrib.transient = value


    transient = property(_getTransient, _setTransient)


    def installingOn(self, store, transactionID=None):
        """
        Called when the owner of the attribute is being installed on the store. 
        """
        return self.attrib.installingOn(store, transactionID=transactionID)


    def toStore(self, store, ownerID, pythonObject, owner, attrs=None,
            transactionID=None):
        """
        Remove all items in the join table that match the owner and then write
        in new ones. Admittedly, this is inefficient.
        """
        return self.attrib.toStore(store, ownerID, pythonObject, owner, attrs,
            transactionID)


    def fromStore(self, storeObject, ownerID, owner):
        """
        Called when a value retrieved from the store needs to be converted back
        into a python value.
        """
        return self.attrib.fromStore(storeObject, ownerID, owner)


    def setStore(self, store, data):
        """
        Sets the store on the attrib.
        """
        self.attrib.setStore(store, data)


    def delete(self, store, data, ownerID, transactionID=None):
        """
        Do the same on the attrib
        """
        self.attrib.delete(store, data, ownerID, transactionID)


    def copy(self):
        """
        Make a copy!
        """
        return MapAttributeProperty(self.attrib, self.fget, self.fset,
            self.fdel)



class Float(Attribute):
    """
    Defines an attribute stored as a float in the database
    """
    def __init__(self, default=None, autoSave=False,
            partialSave=False, autoDirty=False, createIndex=False,
            transmit=False, transient=False, editDataType=None):
        Attribute.__init__(self, autoSave=autoSave,
            partialSave=partialSave, autoDirty=autoDirty,
            createIndex=createIndex, transmit=transmit, transient=transient,
            editDataType=editDataType)
        self.schema["default"] = default


    def toStore(self, _store, pythonObject, transactionID=None):
        """
        Stringify the float.
        """
        if pythonObject is None:
            return None
        return str(pythonObject)


    def fromStore(self, _store, dbObject):
        """
        Make sure the store returned a float and pass it along.
        """
        if dbObject is None:
            return None
        return float(dbObject)


    def copy(self):
        """
        Make a copy!
        """
        return Float(self.schema["default"], self.autoSave, self.partialSave,
            self.autoDirty, createIndex=self.createIndex,
            transmit=self.transmit, transient=self.transient)



class Integer(Attribute):
    """
    Defines an attribute stored as an integer in the database
    """
    def __init__(self, default=None, autoSave=False,
            partialSave=False, autoDirty=False, createIndex=False,
            transmit=False, transient=False, editDataType=None):
        Attribute.__init__(self, autoSave=autoSave,
            partialSave=partialSave, autoDirty=autoDirty,
            createIndex=createIndex, transmit=transmit, transient=transient,
            editDataType=editDataType)
        self.schema["default"] = default


    def toStore(self, _store, pythonObject, transactionID=None):
        """
        Stringify the int.
        """
        if pythonObject is None:
            return None
        return str(pythonObject)


    def fromStore(self, _store, dbObject):
        """
        Make sure the store returned an int and pass it along.
        """
        if dbObject is None:
            return None
        return int(dbObject)


    def copy(self):
        """
        Make a copy!
        """
        return Integer(self.schema["default"], self.autoSave, self.partialSave,
            self.autoDirty, createIndex=self.createIndex,
            transmit=self.transmit, transient=self.transient)



class Text(Attribute):
    """
    Defines a string based attribute
    """
    def __init__(self, maxLength=None, default=None, autoSave=False,
            partialSave=False, autoDirty=False, createIndex=False,
            transmit=False, transient=False, editDataType=None):
        Attribute.__init__(self, autoSave=autoSave,
            partialSave=partialSave, autoDirty=autoDirty,
            createIndex=createIndex, transmit=transmit, transient=transient,
            editDataType=editDataType)
        self.schema["maxLength"] = maxLength
        self.schema["default"] = default


    def toStore(self, _store, pythonObject, transactionID=None):
        """
        Just format this as a string. Really, we want to escape it properly.
        """
        if pythonObject is None:
            return None
        return str(pythonObject)


    def fromStore(self, _store, dbObject):
        """
        Make sure the store returned a string and pass it along.
        """
        if dbObject is None:
            return None
        return str(dbObject)


    def copy(self):
        """
        Make a copy!
        """
        return Text(self.schema["maxLength"], self.schema["default"],
            self.autoSave, self.partialSave, self.autoDirty, createIndex=self.createIndex,
            transmit=self.transmit, transient=self.transient)


    def endswith(self, other):
        """
        Returns an attribute comparison.
        """
        return AttributeComparison(self, " glob ", "*" + other)


    def startswith(self, other):
        """
        Returns an attribute comparison that will be true if 
        """
        return AttributeComparison(self, " glob ", other + "*")


    def contains(self, other):
        """
        Returns an attribute comparison that will be true if other is in the
        queried field.
        """
        return AttributeComparison(self, " glob ", "*" + other + "*")


    def matches(self, other):
        """
        Returns an attribute comparison. Match using a glob, so * for wildcard
        """
        return AttributeComparison(self, " glob ", other)



class IntVector(Text):
    """
    Defines an attribute set by a MV3D ID tuple (2 ints)
    """
    def __init__(self, default=None, autoSave=False,
            partialSave=False, autoDirty=False, createIndex=False,
            transmit=False, transient=False, editDataType=None):
        Text.__init__(self, default=default, autoSave=autoSave,
            partialSave=partialSave, autoDirty=autoDirty,
            createIndex=createIndex, transmit=transmit, transient=transient,
            editDataType=editDataType)
        self.schema["type"] = "Text"


    def toStore(self, _store, pythonObject, transactionID=None):
        """
        Just format this as a string. Really, we want to escape it properly.
        """
        if pythonObject is None:
            return None
        return ",".join([str(obj) for obj in pythonObject])


    def fromStore(self, _store, dbObject):
        """
        Make sure the store returned a string and pass it along.
        """
        if dbObject is None:
            return None
        return tuple([int(num) for num in dbObject.split(",")])


    def copy(self):
        """
        Make a copy!
        """
        return IntVector(self.schema["default"], self.autoSave,
            self.partialSave, self.autoDirty, createIndex=self.createIndex,
            transmit=self.transmit, transient=self.transient)



class Class(Text):
    """
    Defines an attribute which turns into a class when unpersisted.
    """
    def __init__(self, default=None, autoSave=False,
            partialSave=False, autoDirty=False, createIndex=False,
            transmit=False, transient=False, editDataType=None):
        Text.__init__(self, default=default, autoSave=autoSave,
            partialSave=partialSave, autoDirty=autoDirty,
            createIndex=createIndex, transmit=transmit, transient=transient,
            editDataType=editDataType)
        self.schema["type"] = "Text"


    def toStore(self, _store, pythonObject, transactionID=None):
        """
        Just format this as a string. Really, we want to escape it properly.
        """
        if pythonObject is None:
            return None
        return ".".join([pythonObject.__module__, pythonObject.__name__])


    def fromStore(self, _store, dbObject):
        """
        Make sure the store returned a string and pass it along.
        """
        if dbObject is None:
            return None
        return getClass(dbObject)


    def copy(self):
        """
        Make a copy!
        """
        return Class(self.schema["default"], self.autoSave,
            self.partialSave, self.autoDirty, createIndex=self.createIndex,
            transmit=self.transmit, transient=self.transient)



class Enumeration(Text):
    """
    Defines an attribute which represents an enum item. It is saved to the
    database as a string so that changes to the enum numbers do not affect
    anything.
    """
    def __init__(self, enumClass, default=None, autoSave=False,
            partialSave=False, autoDirty=False, createIndex=False,
            transmit=False, transient=False, editDataType=None):
        Text.__init__(self, default=default, autoSave=autoSave,
            partialSave=partialSave, autoDirty=autoDirty,
            createIndex=createIndex, transmit=transmit, transient=transient,
            editDataType=editDataType)
        self.schema["type"] = "Text"
        self.enumClass = enumClass


    def toStore(self, _store, pythonObject, transactionID=None):
        """
        Just format this as a string. Really, we want to escape it properly.
        """
        if pythonObject is None:
            return None
        return self.enumClass[pythonObject]


    def fromStore(self, _store, dbObject):
        """
        Make sure the store returned a string and pass it along.
        """
        if dbObject is None:
            return None
        return getattr(self.enumClass, dbObject)


    def copy(self):
        """
        Make a copy!
        """
        return Class(self.schema["default"], self.autoSave,
            self.partialSave, self.autoDirty, createIndex=self.createIndex,
            transmit=self.transmit, transient=self.transient)



class IDTuple(IntVector):
    """
    These are pretty much the same thing. We'll probably want to make sure that
    there's exactly 2 elements in it at some point, but meh.
    """
    def toStore(self, _store, pythonObject, transactionID=None):
        """
        Just format this as a string. Really, we want to escape it properly.
        """
        if isinstance(pythonObject, int):
            return str(pythonObject)
        return IntVector.toStore(self, _store, pythonObject)


    def fromStore(self, _store, dbObject):
        """
        Make sure the store returned a string and pass it along.
        """
        if dbObject and not "," in dbObject:
            return int(dbObject)
        if not dbObject:
            return None
        return IntVector.fromStore(self, _store, dbObject)


    def copy(self):
        """
        Make a copy!
        """
        return IDTuple(self.schema["default"], self.autoSave,
            self.partialSave, self.autoDirty, createIndex=self.createIndex,
            transmit=self.transmit, transient=self.transient)



class FloatVector(Text):
    """
    Defines an attribute set by a vector (2+ floats)
    """
    def __init__(self, default=None, autoSave=False,
            partialSave=False, autoDirty=False, createIndex=False,
            transmit=False, transient=False, editDataType=None):
        Text.__init__(self, default=default, autoSave=autoSave,
            partialSave=partialSave, autoDirty=autoDirty,
            createIndex=createIndex, transmit=transmit, transient=transient,
            editDataType=editDataType)
        self.schema["type"] = "Text"

    def toStore(self, _store, pythonObject, transactionID=None):
        """
        Just format this as a string. Really, we want to escape it properly.
        """
        if pythonObject is None:
            return None
        return ",".join([str(obj) for obj in pythonObject])


    def fromStore(self, _store, dbObject):
        """
        Make sure the store returned a string and pass it along.
        """
        if dbObject is None:
            return None
        if dbObject == "":
            return []
        return tuple([float(num) for num in dbObject.split(",")])


    def copy(self):
        """
        Make a copy!
        """
        return FloatVector(self.schema["default"], self.autoSave,
            self.partialSave, self.autoDirty, createIndex=self.createIndex,
            transmit=self.transmit, transient=self.transient)



class Stringable(Text):
    """
    Defines an attribute that can transform to a string and back using
    .toString and .fromString. Must also have a constructor that takes no
    arguments.
    """
    def __init__(self, cls, maxLength=None, autoSave=False,
            partialSave=False, autoDirty=False, createIndex=False,
            transmit=False, transient=False, editDataType=None):
        Text.__init__(self, maxLength=maxLength, autoSave=autoSave,
            partialSave=partialSave, autoDirty=autoDirty,
            createIndex=createIndex, transmit=transmit, transient=transient,
            editDataType=editDataType)
        self.cls = cls


    def toStore(self, _store, pythonObject, transactionID=None):
        """
        Just format this as a string. Really, we want to escape it properly.
        """
        if pythonObject is None:
            return None
        return pythonObject.toString()


    def fromStore(self, _store, dbObject):
        """
        Make sure the store returned a string and pass it along.
        """
        if dbObject is None:
            return None
        return self.cls().fromString(dbObject)


    def copy(self):
        """
        Make a copy!
        """
        return Stringable(self.cls, self.schema["maxLength"], self.autoSave,
            self.partialSave, self.autoDirty, createIndex=self.createIndex,
            transmit=self.transmit, transient=self.transient)




class UUID(Attribute):
    """
    Defines a UUID based attribute
    """
    def __init__(self, isID=False, autoSave=False,
            partialSave=False, autoDirty=False, createIndex=False,
            transmit=False, transient=False, editDataType=None):
        Attribute.__init__(self, isID, autoSave=autoSave,
            partialSave=partialSave, autoDirty=autoDirty,
            createIndex=createIndex, transmit=transmit, transient=transient,
            editDataType=editDataType)


    def toStore(self, _store, pythonObject, transactionID=None):
        """
        Just format this as a string. Really, we want to escape it properly.
        """
        if pythonObject is None or pythonObject is self:
            return None
        return pythonObject.hex


    def fromStore(self, _store, dbObject):
        """
        Make sure the store returned a string and pass it along.
        """
        if dbObject is None:
            return None
        return uuid.UUID(dbObject)

    def copy(self):
        """
        Make a copy!
        """
        return UUID(self.isID, self.autoSave, self.partialSave,
            self.autoDirty, createIndex=self.createIndex,
            transmit=self.transmit, transient=self.transient)



class Boolean(Attribute):
    """
    Defines a true/false attribute
    """
    def __init__(self, default=None, autoSave=False, partialSave=False,
            autoDirty=False, createIndex=False,
            transmit=False, transient=False, editDataType=None):
        Attribute.__init__(self, autoSave=autoSave,
            partialSave=partialSave, autoDirty=autoDirty,
            createIndex=createIndex, transmit=transmit, transient=transient,
            editDataType=editDataType)
        self.schema["default"] = default


    def toStore(self, _store, pythonObject, transactionID=None):
        """
        Just format this as a string. Really, we want to escape it properly.
        """
        return pythonObject


    def fromStore(self, _store, dbObject):
        """
        Make sure the store returned a string and pass it along.
        """
        return dbObject

    def copy(self):
        """
        Make a copy!
        """
        return Boolean(self.schema["default"], self.autoSave, self.partialSave,
            self.autoDirty, createIndex=self.createIndex,
            transmit=self.transmit, transient=self.transient)




class Reference(Attribute):
    """
    Defines an attribute that is a reference to another object of a specific
    type
    """
    def __init__(self, targetClass, autoLoad=True, autoSave=False,
            partialSave=False, autoDirty=False, createIndex=False,
            transmit=False, transient=False, soleOwner=False,
            editDataType=None):
        Attribute.__init__(self, autoSave=autoSave,
            partialSave=partialSave, autoDirty=autoDirty,
            createIndex=createIndex, transmit=transmit, transient=transient,
            editDataType=editDataType)
        refSchema = targetClass._idAttribute.schema.copy()
        if refSchema.has_key("isID"):
            del refSchema["isID"]
        if refSchema.has_key("owner"):
            del refSchema["owner"]
        if refSchema.has_key("name"):
            del refSchema["name"]
        self.schema.update(refSchema)
        self.targetClass = targetClass
        self.autoLoad = autoLoad
        self.soleOwner = soleOwner


    def toStore(self, store, pythonObject, transactionID=None):
        """
        Saves the referenced item if it hasn't been yet. Otherwise, return its
        id.
        """
        if pythonObject is None:
            return None
        if isinstance(pythonObject, uuid.UUID):
            return pythonObject.hex
        if pythonObject.store is None or store != pythonObject.store:
            pythonObject.save(store, transactionID=transactionID)
        return pythonObject.getFormattedID()


    def fromStore(self, store, dbObject):
        """
        Reads in the referenced item and returns it.
        """
        if not self.autoLoad:
            return self.targetClass._idAttribute.fromStore(store, dbObject)
        if dbObject is None:
            return None
        return self.targetClass.load(store,
            self.targetClass._idAttribute.fromStore(store, dbObject))


    def setStore(self, store, data):
        """
        Sets the store on the attrib.
        """
        if isinstance(data, uuid.UUID) or data is None:
            return
        data.setStore(store)


    def installingOn(self, store, transactionID=None):
        """
        Called when this attribute is being installed on a store. We will
        also have to install the target classes schema on the store.
        """
        self.targetClass.installOn(store, transactionID=transactionID)


    def delete(self, _store, data, transactionID=None):
        """
        Called when the object is deleted from the store.
        """
        if data is not None and self.soleOwner:
            data.delete(transactionID=transactionID)


    def copy(self):
        """
        Make a copy!
        """
        return Reference(self.targetClass, self.autoLoad, self.autoSave,
            self.partialSave, self.autoDirty, createIndex=self.createIndex,
            transmit=self.transmit, transient=self.transient,
            soleOwner=self.soleOwner)



class UntypedReference(Attribute):
    """
    Defines an attribute that is a reference to another object or any type
    """
    def __init__(self, autoSave=False, partialSave=False, autoDirty=False, createIndex=False,
            transmit=False, transient=False, soleOwner=False,
            editDataType=None):
        Attribute.__init__(self, autoSave=autoSave,
            partialSave=partialSave, autoDirty=autoDirty,
            createIndex=createIndex, transmit=transmit, transient=transient,
            editDataType=editDataType)
        self.schema["type"] = "Text"
        self.soleOwner = soleOwner


    def toStore(self, store, pythonObject, transactionID=None):
        """
        Saves the referenced item if it hasn't been yet. Otherwise, return its
        id.
        """
        if pythonObject is None:
            return None
        if (pythonObject.store is None or store not in pythonObject._storedOn or
                pythonObject.getFormattedID() is None):
            pythonObject.save(store, transactionID=transactionID)
        cls = ".".join([pythonObject.__class__.__module__,
            pythonObject.__class__.__name__])
        return ",".join([cls, pythonObject.getFormattedID()])


    def fromStore(self, store, dbObject):
        """
        Reads in the referenced item and returns it.
        """
        if dbObject is None:
            return None
        cls, oid = dbObject.split(",")
        modName = ".".join(cls.split(".")[:-1])
        clsName = cls.split(".")[-1]
        module = __import__(modName, fromlist=[clsName])
        targetClass = getattr(module, clsName)
        try:
            return targetClass.load(store,
                targetClass._idAttribute.fromStore(store, oid))
        except ValueError, exc:
            if "matches loading" in str(exc):
                deferr()
                return None
            raise


    def setStore(self, store, data):
        """
        Sets the store on the attrib.
        """
        if data is None:
            return
        data.setStore(store)


    def installingOn(self, store, transactionID=None):
        """
        Called when this attribute is being installed on a store. We will
        install the target classes schema on the store at a later time.
        """
        #self.targetClass.installOn(store)


    def delete(self, _store, data, transactionID=None):
        """
        Called when the object is deleted from the store.
        """
        if data is not None and self.soleOwner:
            data.delete(transactionID=transactionID)


    def copy(self):
        """
        Make a copy!
        """
        return UntypedReference(self.autoSave, self.partialSave,
            self.autoDirty, createIndex=self.createIndex,
            transmit=self.transmit, transient=self.transient,
            soleOwner=self.soleOwner)



class List(MapAttribute):
    """
    Defines a list that can be stored. Order of items in the list is guaranteed.
    """
    def __init__(self, ofType, transmit=False, transient=False,
            editDataType=None):
        MapAttribute.__init__(self, Integer(), ofType, transmit=transmit,
            transient=transient, editDataType=editDataType)


    def toStore(self, store, ownerID, pythonObject, owner, attrs=None,
            transactionID=None):
        """
        Convert the list to a dictionary where the keys are the indexes and
        the values are.. the values.
        """
        MapAttribute.toStore(self, store, ownerID,
            self.toDict(pythonObject), owner, attrs,
            transactionID=transactionID)


    def toDict(self, pythonObject):
        if pythonObject is None:
            return
        return dict(enumerate(pythonObject))


    def fromStore(self, store, ownerID, owner):
        """
        Convert the dictionary from our base class to a list.
        """
        dicty = MapAttribute.fromStore(self, store, ownerID, owner)
        return self.finalize(dicty)


    def setStore(self, store, data):
        """
        Sets the store on the attrib.
        """
        for item in data:
            self.valueType.setStore(store, item)


    def delete(self, store, data, ownerID, transactionID=None):
        """
        Called when the owner is deleted from the store.
        """
        if data is not None:
            MapAttribute.delete(self, store, dict(enumerate(data)), ownerID,
                transactionID=transactionID)


    def finalize(self, value):
        """
        Do any conversion needed after loading. In this case, converting to
        a list. 
        """
        return [value[offset] for offset in range(len(value))]


    def copy(self):
        """
        Make a copy!
        """
        return List(self.valueType, transmit=self.transmit,
            transient=self.transient)



class AutoAttrib(Attribute):
    """
    Descriptor class to automatically handle setting an attribute dirty
    """
    saveQueue = []
    saveCall = None


    def __init__(self, name, dataType, autoSave, partialSave, autoDirty,
            updateObservers):
        Attribute.__init__(self)
        self.schema["name"] = name
        self.name = name
        self.dataType = dataType
        self.autoSave = autoSave
        self.partialSave = partialSave
        self.autoDirty = autoDirty
        self.updateObservers = updateObservers
        self.attrName = "_auto_%s" % self.name


    def __get__(self, obj, _type=None):
        """
        Simply returns the attribute or None
        """
        if obj is None:
            return self
#        print obj.__class__.__name__, self.attrName
        return getattr(obj, self.attrName, None)


    def __set__(self, obj, value):
        """
        Sets the attribute and saves or sets dirty as appropriate
        """
        setattr(obj, self.attrName, value)
        if hasattr(obj, "propertyChanged"):
            obj.propertyChanged(self.name)
        if self.autoDirty or self.partialSave:
            obj.dirty(self.name)
        if obj._loading:
            return
        if self.autoSave and obj.store is not None:
            if (AutoAttrib.saveQueue is not None and
                    not (obj, True) in AutoAttrib.saveQueue
                    and not (obj, False) in AutoAttrib.saveQueue):
                AutoAttrib.saveQueue.append((obj, True))
                if AutoAttrib.saveCall is None:
                    AutoAttrib.saveCall = reactor.callLater(0, #@UndefinedVariable
                        self._runQueue)
            elif AutoAttrib.saveQueue is None:
                obj.save(partial=self.partialSave)
        if self.updateObservers:
            obj.updateAllClients("%s" % self.name, value)


    @classmethod
    @timed
    def _runQueue(cls):
        """
        Run through the queued up items and save them all.
        """
        cls.saveCall = None
        count = 0
        while count < 10 and len(cls.saveQueue):
            obj, partial = cls.saveQueue.pop(0)
            if hasattr(obj, "save"):
                if obj.store is not None and obj.store.isOpen:
                    obj.save(partial=partial)
            count += 1
        if len(cls.saveQueue):
            cls.saveCall = reactor.callLater(0, #@UndefinedVariable
                cls._runQueue)


    @classmethod
    @timed
    def saveAll(cls, store):
        """
        Save all queued items in the given store
        """
        if cls.saveQueue is None:
            return
        for obj, partial in cls.saveQueue[:]:
            if obj.store == store:
                cls.saveQueue.remove((obj, partial))
                obj.save(partial=partial)


    def toStore(self, store, pythonObject, transactionID):
        """
        Called when a python value needs to be converted to a format that can
        be stored.
        """
        return self.dataType.toStore(store, pythonObject, transactionID)


    def setStore(self, store, data):
        """
        Sets the store on the attrib.
        """
        self.dataType.setStore(store, data)


    def copy(self):
        """
        Make a copy!
        """
        return AutoAttrib(self.name, self.dataType, self.autoSave,
            self.partialSave, self.autoDirty)



class Pickled(Attribute):
    """
    An attribute that is pickled and stored as a string. Using this is a sign
    of weakness. It can introduce odd errors into your code and will generally
    screw with your head. Also, you probably won't be able to do any meaningful
    queries against this field.
    """
    def __init__(self, isID=False, autoSave=False, partialSave=False,
            autoDirty=False, createIndex=False, transmit=False,
            transient=False, editDataType=None):
        Attribute.__init__(self, isID, autoSave=autoSave,
            partialSave=partialSave, autoDirty=autoDirty,
            createIndex=createIndex, transmit=transmit, transient=transient,
            editDataType=editDataType)


    def toStore(self, _store, pythonObject, transactionID):
        """
        Pickle the object if not None
        """
        if pythonObject is None:
            return None
        return pickle.dumps(pythonObject)


    def fromStore(self, _store, dbObject):
        """
        Depickle the object if not None
        """
        if dbObject is None:
            return None
        return pickle.loads(str(dbObject))


    def copy(self):
        """
        Make a copy!
        """
        return Pickled(self.isID, self.autoSave, self.partialSave,
            self.autoDirty, createIndex=self.createIndex,
            transmit=self.transmit, transient=self.transient)



class Referenced(object):
    """
    Base class type which causes subclasses to include a hidden reference
    attribute pointing to the data stored by this class.
    """
    def __init__(self):
        pass


    def meta(self, owner, base, persistAttrs):
        persistAttrs["_%s_Base" % base.__name__] = Reference(base,
            autoLoad=False)
        persistAttrs["_%s_Base" % base.__name__].schema["name"] = ("_%s_Base" %
            base.__name__)
        persistAttrs["_%s_Base" % base.__name__].schema["owner"] = owner


    def installOn(self, store, _item, base, _schema, transactionID=None):
        """
        Apply this base class to another class
        """
        base.installOn(store, transactionID)
        #schema.append(Reference(base).schema)
        #schema[-1]["name"] = "_%s_Base" % base.__name__


    def new(self, store, item, base, _persistData, parentPersistData):
        setattr(item, "_%s_Base" % base.__name__, getattr(item,
            base._idAttribute.schema["name"]))
        parentPersistData["_%s_Base" % base.__name__] = (
            base._idAttribute.toStore(
            store, getattr(item,
            base._idAttribute.schema["name"])))


    def update(self, store, item, base, persistData):
        pass


    def load(self, cls, store, uid, obj, baseClass, useCache):
        cls.load(store, uid, obj, baseClass, useCache=useCache)


    def getStoreID(self, item, base):
        return getattr(item, "_%s_Base" % base.__name__)



class Inclusive(object):
    """
    Base class type which causes subclasses to just include all the attributes
    in their definition. Version control will be hairy here.
    """
    def meta(self, _owner, base, persistAttrs):
        persistAttrs.update(getattr(base, "_%s__persistAttributes" %
            base.__name__))


    def installOn(self, store, item, base, schema, transactionID=None):
        """
        Apply this base class to another class
        """


    def new(self, store, item, base, persistData, parentPersistData):
        pass


    def update(self, store, item, base, persistData):
        pass


    def load(self, cls, store, uid, obj, baseClass, useCache):
        pass


    def getStoreID(self, _item, _base):
        return None



class SharedID(object):
    """
    Base class type which creates data in a new table but links based on
    a shared id. All subclasses of the base must use the same type/name of id.
    """
    def meta(self, owner, base, persistAttrs):
        pass


    def installOn(self, store, _item, base, _schema, transactionID=None):
        """
        Apply this base class to another class
        """
        base.installOn(store, transactionID=transactionID)


    def new(self, store, item, base, persistData, _parentPersistData):
        idName = base._idAttribute.schema["name"]
        setattr(item, idName, item.getStoreID())
        persistData[idName] = base._idAttribute.toStore(
            store, item.getStoreID())


    def update(self, store, item, base, persistData):
        pass


    def load(self, cls, store, uid, obj, baseClass, useCache):
        cls.load(store, uid, obj, baseClass, useCache=useCache)


    def getStoreID(self, item, _base):
        return item.getStoreID()



def clearAllCachedItems():
    """
    Clear all cached items. Mostly useful for debugging and tests.
    """
    for cls in PersistableMeta.persistClasses:
        cls._cache = {}



class _Persistable(object):
    pass



class PersistableMeta(type):
    """
    Metaclass for persistable items that sets up their attributes.
    """
    persistClasses = []

    def __new__(cls, name, bases, dictionary):
        persistAttrs = {}
        mapAttrs = {}
        # check for schemaVersion problems
        if not "_schemaVersion" in dictionary:
            if not Persistable in bases:
                raise TypeError("Class %s doesn't define _schemaVersion!" %
                    name)
        isCacheable = False
        for base in bases:
            if base.__name__ == "Cacheable":
                isCacheable = True
        if not dictionary.has_key("_typeName"):
            dictionary["_typeName"] = name
        if not dictionary.has_key("_baseType"):
            dictionary["_baseType"] = None

        typeName = dictionary["_typeName"]
        attrs = dictionary.items()
        try:
            if dictionary["allowedState"] is not None:
                allowedState = list(dictionary["allowedState"])
            else:
                allowedState = []
        except KeyError:
            allowedState = []
        for base in bases:
            if hasattr(base, "allowedState"):
                allowedState.extend(base.allowedState or [])
        allowedState = list(set(allowedState))
        baseClasses = []
        allAttrs = {}
        for base in bases:
            if issubclass(base, _Persistable) and base.__name__ not in [
                    "Persistable", "_Persistable"]:
                baseClasses.append(base)
                if base._baseType is not None:
                    base._baseType.meta(name, base, persistAttrs)
            elif (not issubclass(base, _Persistable) and
                    base.__name__ != "_Persistable"):
                # include attributes from it
                def isAttribute(thing):
                    return isinstance(thing, (Attribute, MapAttribute))
                for attrName, item in getmembers(base, isAttribute):
                    attrs.append([attrName, item.copy()])
            if hasattr(base, "allowedState") and base.allowedState is not None:
                allowedState.extend(base.allowedState)
            if hasattr(base, "_attributes") and base._attributes:
                allAttrs.update(base._attributes)
        for attrName, item in attrs:
            if isinstance(item, Attribute):
                if item.isID:
                    dictionary["_idAttribute"] = item
                if item.autoSave or item.autoDirty or item.partialSave:
                    dictionary[attrName] = AutoAttrib(attrName, item,
                        item.autoSave, item.partialSave, item.autoDirty,
                        item.transmit and isCacheable)
                    dictionary[attrName].schema["owner"] = name
                if item.transmit:
                    if isCacheable and not dictionary.has_key(
                            "observe_%s" % attrName):
                        def mkObserved(attr):
                            def observed(self, value):
                                """ Updates the value """
                                setattr(self, attr, value)
                            return observed
                        dictionary["observe_%s" % attrName] = mkObserved(attrName)
                    allowedState.append(attrName)
                item.schema["owner"] = name
                item.schema["name"] = attrName
                allAttrs[attrName] = item
                if not item.transient:
                    persistAttrs[attrName] = item
            elif isinstance(item, MapAttribute):
                item.setOwner(name)
                item.setAttributeName(attrName)
                if item.transmit:
                    allowedState.append(attrName)
                    if isCacheable and not dictionary.has_key(
                            "observe_%s" % attrName):
                        def mkObserved(attr):
                            def observed(self, value):
                                """ Updates the value """
                                setattr(self, attr, value)
                            return observed
                        dictionary["observe_%s" % attrName] = mkObserved(
                            attrName)
                if not item.transient:
                    mapAttrs[attrName] = item
                allAttrs[attrName] = item
        if dictionary.get("_idAttribute") is not None:
            key = dictionary["_idAttribute"]
        else:
            key = UUID(isID=True)
            key.schema["owner"] = name
            key.schema["name"] = "%s_id" % typeName
            dictionary["_idAttribute"] = key
            dictionary[key.schema["name"]] = key
            persistAttrs[key.schema["name"]] = key
        for attrName, item in mapAttrs.items():
            item.ownerKey = key
        dictionary["_%s__persistAttributes" % name] = persistAttrs
        dictionary["_%s__mapAttrs" % name] = mapAttrs
        dictionary["_baseClasses"] = baseClasses
        dictionary["_attributes"] = allAttrs
        dictionary["_cache"] = {}
        if len(allowedState):
            allowedState = list(set(allowedState))
            dictionary["allowedState"] = allowedState
#        print name, "allowedState", allowedState
        newClass = type.__new__(cls, name, bases, dictionary)
        cls.persistClasses.append(newClass)
        return newClass



class Persistable(_Persistable):
    """
    Defines an object that can be persisted.
    """
    __metaclass__ = PersistableMeta
    __persistAttributes = None
    __mapAttrs = None
    _idAttribute = "_storeId"
    _baseClasses = None
    _typeName = None
    _schemaVersion = 1
    _dirtyFields = None
    _loading = False
    _storeId = None
    _storedOn = None
    _cache = None
    store = None


    def __init__(self, store=None, **kwargs):
        self._storedOn = []
        allowedArgs = self.resetAttributes(kwargs)
        store = store
        if len(kwargs) != len(allowedArgs):
            for name in kwargs.keys():
                if not name in allowedArgs:
                    raise TypeError("'%s' is an invalid argument for %s" %
                        (name, self.__class__.__name__))
        for name, attr in kwargs.items():
            setattr(self, name, attr)


    def resetAttributes(self, kwargs, cls=None, allowedArgs=None):
        """
        Reset attributes to their defaults
        """
        if allowedArgs is None:
            allowedArgs = []
        cls = cls or self.__class__
        items = getattr(cls, "_%s__persistAttributes" %
                cls.__name__).items()
        mapAttrs = getattr(cls, "_%s__mapAttrs" % cls.__name__).items()
        for name, attr in items:
            if kwargs.has_key(name):
                allowedArgs.append(name)
            setattr(self, name, attr.schema.get("default", None))
        for name, attr in mapAttrs:
            if kwargs.has_key(name):
                allowedArgs.append(name)
            setattr(self, name, None)
        for baseClass in cls._baseClasses:
            self.resetAttributes(kwargs, baseClass, allowedArgs)
        return allowedArgs


    @classmethod
    def installOn(cls, store, transactionID=None):
        """
        Install the schema for this type on the specified store. Only needs to
        be done once on the store and then again only if the version changes.
        """
        attrs = []
        for base in cls._baseClasses:
            if base._baseType is not None:
                base._baseType.installOn(store, cls, base, attrs, transactionID)
        items = getattr(cls, "_%s__persistAttributes" % cls.__name__)
        mapAttrs = getattr(cls, "_%s__mapAttrs" % cls.__name__).items()
        for _name, attr in items.items():
            attrs.append(attr.schema)
            attr.installingOn(store, transactionID)
        if not items.has_key(cls._idAttribute.schema["name"]):
            attr = cls._idAttribute
            attrs.append(attr.schema)
        for _name, attr in mapAttrs:
            attr.installingOn(store, transactionID)
        attrs.append(dict(name="owningObject", type="Text", owner=cls.__name__,
            index=True))
        store.registerSchema(cls._typeName, cls._schemaVersion,
            attrs, transactionID=transactionID)


    def getPersistData(self, store, cls=None, selectAttributes=None,
            transactionID=None):
        """
        Retrieves the persistable data from this item.
        """
        cls = cls or self.__class__
        idValue = getattr(self, cls._idAttribute.schema["name"])
        if idValue is None or idValue is cls._idAttribute:
            setattr(self, cls._idAttribute.schema["name"], uuid.uuid1())
        data = {}
        items = getattr(self, "_%s__persistAttributes" % cls.__name__)
        for name, pType in items.items():
            if selectAttributes is not None and name not in selectAttributes:
                continue
            try:
                data[name] = pType.toStore(store, getattr(self, name),
                    transactionID)
            except Exception:
                # add some helpful info to the exception
                ext, exc, tb = sys.exc_info()
                exc.message = "%s.%s: %s" % (cls.__name__, name, exc.message)
                exc.args = (exc.message,)
                raise ext, exc, tb
        if (not items.has_key(cls._idAttribute.schema["name"]) and
                selectAttributes is None):
            data[cls._idAttribute.schema["name"]] = cls._idAttribute.toStore(
                store, self.getStoreID())
        if cls is not self.__class__:
            className = ".".join([self.__class__.__module__,
                self.__class__.__name__])
            data["owningObject"] = ",".join([className, self.getFormattedID()])
        return data


    def setPersistData(self, data, cls=None):
        """
        Sets the persisted attributes on this item based on the data passed
        in.
        """
        cls = cls or self.__class__
        upgraded = False
        incomingVersion = data["_schemaVersion"]
        if incomingVersion != cls._schemaVersion:
            data = cls.upgrade(data)
            upgraded = True
        items = getattr(self, "_%s__persistAttributes" % cls.__name__)
        for name, pType in items.items():
            try:
                setattr(self, name, pType.fromStore(self.store, data[name]))
            except Exception:
                # add some helpful info to the exception
                ext, exc, tb = sys.exc_info()
                exc.message = "%s.%s: %s" % (cls.__name__, name, exc.message)
                exc.args = (exc.message,)
                raise ext, exc, tb
        return upgraded


    def onLoad(self):
        """
        Redefine this in your subclass to get called back when this object
        is loaded.
        """


    @classmethod
    def upgrade(cls, oldData):
        """
        This method will be called when trying to retrieve an item of this
        type from the store that has a different version number than this
        type. It should return a dictionary with updated data.
        """
        if oldData["_schemaVersion"] == cls._schemaVersion:
            return oldData
        raise UpgradeError(oldData["_schemaVersion"], cls._schemaVersion,
            "No overriden upgrade method for class %s" % cls.__name__)


    def getStoreID(self, cls=None):
        """
        Returns the id of this item.
        """
        if cls is None:
            cls = self.__class__
        return getattr(self, cls._idAttribute.schema["name"])


    def setStoreID(self, storeID, cls=None):
        """
        Returns the id of this item.
        """
        if cls is None:
            cls = self.__class__
        setattr(self, cls._idAttribute.schema["name"], storeID)


    def getFormattedID(self):
        """
        Returns the id of this item formatted for the store.
        """
        return self._idAttribute.toStore(self.store, self.getStoreID())


    def setStore(self, store, cls=None):
        """
        Set the store on this item and all child items
        """
        if self.store == store:
            return
        cls = cls or self.__class__
        items = getattr(self, "_%s__persistAttributes" % cls.__name__)
        mapAttrs = getattr(cls, "_%s__mapAttrs" % cls.__name__).items()
        for name, item in items.items():
            item.setStore(store, getattr(self, name))
        for name, attr in mapAttrs:
            attr.setStore(store, getattr(self, name))
        for baseCls in cls._baseClasses:
            self.setStore(store, baseCls)
        self.store = store


    def onSave(self, store, partial, selectAttributes):
        """
        Redefine this on your subclass.
        """


    @timed
    def save(self, store=None, partial=False, selectAttributes=None,
            transactionID=None, forceUpdate=False):
        """
        Save this item to a store. If we were loaded from a store and a new one
        wasn't provided, then we will be saved to the same store. Otherwise,
        store on the specified store.
        """
        if self.store is None and store is None:
            return
        if store is None:
            store = self.store
        if self._storedOn is None:
            self._storedOn = []
        if not ((store is None and self.store is not None) or
                store in self._storedOn):
            self.installOn(store, transactionID)
        transactionID = store.startTransaction()
        try:
            if partial:
                if not selectAttributes:
                    selectAttributes = self._dirtyFields
                    self._dirtyFields = []
            self.onSave(store, partial, selectAttributes)
            if ((store is None and self.store is not None) or
                    store in self._storedOn) or forceUpdate:
                isUpdate = True
            else:
                isUpdate = False
            self._saveClass(self.__class__, store, selectAttributes, isUpdate,
                transactionID=transactionID)
            if not isUpdate:
                if self._storedOn is None:
                    self._storedOn = []
                if not store in self._storedOn:
                    self._storedOn.append(store)
        except:
            store.rollbackTransaction(transactionID)
            raise
        else:
            store.commitTransaction(transactionID)
        return self.getStoreID()


    @timed
    def _saveClass(self, cls, store, selectAttributes=None, isUpdate=False,
            parentPersistData=None, includeParents=True, transactionID=None):
        """
        Save attributes from one class and is recursively called to save
        attributes of all parent classes.
        """
        if hasattr(cls, "_baseType") and cls._baseType.__class__.__name__ == "Inclusive": # TODO: HACK
            return
        self.store = store
        persistData = self.getPersistData(store, cls, selectAttributes,
            transactionID=transactionID)
        mapAttrs = getattr(cls, "_%s__mapAttrs" % cls.__name__).items()
        for name, attr in mapAttrs:
            if selectAttributes is not None and not name in selectAttributes:
                continue
            try:
                attr.toStore(store, self.getFormattedID(), getattr(self,
                    name), self, transactionID=transactionID)
            except Exception:
                # add some helpful info to the exception
                ext, exc, tb = sys.exc_info()
                exc.message = "%s.%s: %s" % (cls.__name__, name, exc.message)
                exc.args = (exc.message,)
                raise ext, exc, tb
        if includeParents:
            for baseClass in cls._baseClasses:
                self._saveClass(baseClass, store, selectAttributes, isUpdate,
                    persistData, transactionID=transactionID)
        if not len(persistData):
            return
        if isUpdate:
            if parentPersistData is not None:
                cls._baseType.update(store, self, cls, persistData)
            store.update(cls._typeName, (cls._idAttribute ==
                self.getStoreID(cls)).build(), persistData,
                transactionID=transactionID)
        else:
            if parentPersistData is not None:
                cls._baseType.new(store, self, cls, persistData,
                    parentPersistData)
            store.new(cls._typeName, persistData, transactionID=transactionID)
        self.__class__._cache[self.getStoreID(cls)] = self


    def queueSave(self, selectAttributes=None):
        """
        Queue a save of this object.
        """
        if selectAttributes is not None:
            for attrib in selectAttributes:
                self.dirty(attrib)
        if (AutoAttrib.saveQueue is not None and
                not (self, True) in AutoAttrib.saveQueue
                and not (self, False) in AutoAttrib.saveQueue):
            AutoAttrib.saveQueue.append((self, True))
            if AutoAttrib.saveCall is None:
                AutoAttrib.saveCall = reactor.callLater(0, #@UndefinedVariable
                    AutoAttrib._runQueue)
        elif AutoAttrib.saveQueue is None:
            partial = len(selectAttributes) > 0
            self.save(partial=partial, selectAttributes=selectAttributes)


    @timed
    def delete(self, transactionID=None):
        """
        Delete this item from the store. This can not be undone.
        """
        if self.store is None:
            return
        self.onDelete()
        self._delClass(self.__class__, self.getStoreID(),
            transactionID=transactionID)
        del self.__class__._cache[self.getStoreID()]
        self._storedOn.remove(self.store)
        self.store = None


    def onDelete(self):
        """
        Called before this item is being deleted from the store.
        """


    def _delClass(self, cls, uid, specificVersion=None, includeParents=True,
            transactionID=None):
        """
        Delete a class specific portion of this instance and all parent classes
        """
        self.store.delete(cls._typeName, (
            cls._idAttribute.schema["name"],
            "=", getattr(cls, cls._idAttribute.schema[
            "name"]).toStore(self.store, uid)), specificVersion,
            transactionID=transactionID)
        items = getattr(self, "_%s__persistAttributes" % cls.__name__)
        mapAttrs = getattr(cls, "_%s__mapAttrs" % cls.__name__).items()
        if specificVersion is None: # ??
            for name, pType in items.items():
                pType.delete(self.store, getattr(self, name), transactionID)
            for name, attr in mapAttrs:
                attr.delete(self.store, getattr(self, name), cls.getFormattedID(
                    self), transactionID=transactionID)
        if not includeParents:
            return
        for baseClass in cls._baseClasses:
            uid = baseClass._baseType.getStoreID(self, baseClass)
            if uid is not None:
                self._delClass(baseClass, uid, transactionID=transactionID)


    @classmethod
    @timed
    def load(cls, store, iid, obj=None, clazz=None, result=None, useCache=True):
        """
        Retrieve a single item from the store with the specified ID. This will
        raise an exception if there is more or less than one item that matches
        the query.
        """
        clazz = clazz or cls
        if obj is None:
            if useCache and clazz._cache.has_key(iid):
                obj = clazz._cache[iid]
                obj.store = store
                if not obj._storedOn:
                    obj._storedOn = []
                if not store in obj._storedOn:
                    obj._storedOn.append(store)
                return obj
            try:
                obj = clazz()
            except TypeError:
                class _Dummy(object):
                    pass
                obj = _Dummy()
                obj.__class__ = clazz
            obj.store = store
            obj._loading = True
            cls._cache[iid] = obj
            objWasNone = True
        else:
            objWasNone = False

        try:
            if result is None:
                result = store.query(clazz._typeName,
                    (clazz._idAttribute.schema["name"], "=",
                    clazz._idAttribute.toStore(store, iid)))
                if len(result) != 1:
                    raise ValueError("Found %d matches loading %s with id %s" %
                        (len(result), clazz.__name__, iid))
                result = result[0]
            if objWasNone and result.get("owningObject") is not None:
                # top level object
                className = ".".join([clazz.__module__, clazz.__name__])
                ownClass, ownID = result["owningObject"].split(",")
                if className != ownClass:
                    # got to load him instead
                    ownModName = ".".join(ownClass.split(".")[:-1])
                    ownClassName = ownClass.split(".")[-1]
                    mod = __import__(ownModName, fromlist=[ownClassName])
                    del cls._cache[iid]
                    realClass = getattr(mod, ownClassName)
                    return realClass.load(store,
                        realClass._idAttribute.fromStore(store, ownID),
                        useCache=useCache)
            upgraded = obj.setPersistData(result, clazz)
            mapAttrs = getattr(cls, "_%s__mapAttrs" % clazz.__name__).items()
            for name, attr in mapAttrs:
                try:
                    value = attr.fromStore(store, obj.getFormattedID(), obj)
                    setattr(obj, name, value)
                except Exception:
                    # add some helpful info to the exception
                    ext, exc, tb = sys.exc_info()
                    exc.message = "%s.%s: %s" % (cls.__name__, name,
                        exc.message)
                    exc.args = (exc.message,)
                    raise ext, exc, tb
            if upgraded:
                clazz.installOn(obj.store)
                obj._saveClass(clazz, obj.store, None, False,
                    includeParents=False)
                incomingVersion = result["_schemaVersion"]
                obj._delClass(clazz, obj.getStoreID(clazz), incomingVersion,
                    includeParents=False)
            for baseClass in clazz._baseClasses:
                uid = baseClass._baseType.getStoreID(obj, baseClass)
                if uid is not None:
                    baseClass._baseType.load(cls, store, uid, obj, baseClass,
                        useCache=useCache)
            if clazz is cls:
                obj.onLoad()
            return obj
        finally:
            obj._loading = False
            obj._storedOn = [store]


    @classmethod
    @timed
    def query(cls, store, query=None, offset=0, limit= -1, order=None,
            fields=None, useCache=True):
        """
        Finds all objects of this type that match the query sent in. Returns
        a list of them ready for use. The query can be created by using normal
        Python expressions and referencing the classes Attributes to specify
        what to query for. Here's an example:

        Thing.query(store, (Thing.age > 12) & (Thing.name != "trash"))

        One thing to note is that the binary & and | operators generally need
        their operands wrapped in parenthesis due to where they stand in
        order of operations.
        """
        return list(cls.xquery(store, query, offset, limit, order, fields,
            useCache))


    @classmethod
    @timed
    def xquery(cls, store, query=None, offset=0, limit= -1, order=None,
            fields=None, useCache=True):
        """
        Finds all objects of this type that match the query sent in. Returns
        a generator. The query can be created by using normal
        Python expressions and referencing the classes Attributes to specify
        what to query for. Here's an example:

        Thing.query(store, (Thing.age > 12) & (Thing.name != "trash"))

        One thing to note is that the binary & and | operators generally need
        their operands wrapped in parenthesis due to where they stand in
        order of operations.
        """
        if query is not None and not isinstance(query, tuple):
            query = query.build()
        if fields is not None:
            originalFields = fields
            fields = [field.schema["name"] for field in fields]
        results = store.query(cls._typeName, query, fields=fields,
            offset=offset, limit=limit, order=order)
        if fields is not None:
            for result in results:
                newResult = {}
                for field in originalFields:
                    newResult[field] = field.fromStore(store,
                        result[field.schema["name"]])
                yield newResult
            return
        for result in results:
            iid = cls._idAttribute.fromStore(store, result[
                cls._idAttribute.schema["name"]])
            yield cls.load(store, iid, result=result,
                useCache=useCache)


    @classmethod
    @timed
    def count(cls, store, query=None, offset=0, limit= -1, order=None):
        """
        Finds all objects of this type that match the query sent in. Returns
        a count of the number of matches. See query() for query syntax
        """
        if query is not None:
            query = query.build()
        results = store.query(cls._typeName, query, offset=offset, limit=limit,
            order=order, count=True)
        if not len(results):
            return 0
        return sum([result["count(*)"] for result in results])



    def existsOn(self, store):
        """
        Returns true if this has been stored on the specified store. Does an
        exhaustive check.
        """
        return self.count(store, self._idAttribute == self.getStoreID()) == 1


    @classmethod
    @timed
    def get(cls, store, query=None, useCache=True):
        """
        Similar to query, but the query is expected to return one result which
        is then returned 
        """
        res = cls.query(store, query, useCache=useCache)
        if len(res) != 1:
            raise ValueError("Expected a single result, got %d. Query: %s" % (
                len(res), query))
        return res[0]


    @classmethod
    @timed
    def first(cls, store, query=None, offset=0, order=None):
        """
        Similar to get, but doesn't error and just returns the first item or
        None
        """
        res = cls.query(store, query, offset=offset, limit=1, order=order)
        if not len(res):
            return
        return res[0]


    @timed
    def dirty(self, property):
        """
        Sets a specified property dirty
        """
        if self._dirtyFields is None:
            self._dirtyFields = []
        elif property in self._dirtyFields:
            return
        self._dirtyFields.append(property)


    def matches(self, query):
        """
        Returns true if this object matches the query.
        """
        op2op = {
            "<":operator.lt,
            "<=":operator.le,
            ">":operator.gt,
            ">=":operator.ge,
            "=":operator.eq,
            "!=":operator.ne,
            " glob ":fnmatch,
        }
        if isinstance(query, tuple):
            left, op, right = query
            if isinstance(left, tuple):
                left = self.matches(left)
            elif isinstance(left, str):
                try:
                    raw = getattr(self, left)
                    left = getattr(self.__class__, left).toStore(None, raw)
                except AttributeError:
                    pass
            if isinstance(right, tuple):
                right = self.matches(right)
            elif isinstance(right, str):
                try:
                    raw = getattr(self, right)
                    right = getattr(self.__class__, right).toStore(None, raw)
                except AttributeError:
                    pass
            if op == " and ":
                return left and right
            if op == " or ":
                return left or right
            return op2op[op](left, right)
        if isinstance(query.left, AttributeComparison):
            left = self.matches(query.left)
        elif isinstance(query.left, Attribute):
            left = getattr(self, query.left.schema["name"])
            left = query.left.toStore(None, left)
        else:
            left = query.left
        if isinstance(query.right, AttributeComparison):
            right = self.matches(query.right)
        elif isinstance(query.right, Attribute):
            right = getattr(self, query.right.schema["name"])
            right = query.right.toStore(None, right)
        else:
            right = query.right
        if query.operator == " and ":
            return left and right
        if query.operator == " or ":
            return left or right
        return op2op[query.operator](left, right)


    def getAttributesByType(self, typeOrTypes):
        """
        Returns a list of attribute on this object that match the given type or
        types.
        """
        attrs = set()
        for name, dataType in self.__persistAttributes.iteritems():
            if isinstance(dataType, typeOrTypes):
                attrs.add(name)
        for cls in self._baseClasses:
            attrs.update(cls.getReferenceAttributes(self, typeOrTypes))
        return list(attrs)