# -*- test-case-name: mv3d.test.service.test_simservice -*- # Copyright (C) 2006-2012 Mortal Coil Games # See LICENSE for details. """ Class and Code for Simulation Server """ import os from twisted.internet import defer from twisted.internet.defer import inlineCallbacks, returnValue from twisted.spread import pb from twisted.application.service import Service from twisted.python.log import deferr from zope.interface import implements #@UnresolvedImport import mv3d from mv3d.util.classgen import ClassGenerator from mv3d.util.modifier import Modifiable from mv3d.util.conductor import parseInterfaceConfig, parsePermissionConfig from mv3d.net.security import Securable, requirePermissions from mv3d.net.client import ServiceLoc from mv3d.net.ha import ShardedPool, PoolMember from mv3d.server.service import checkServicePermissions, viewed from mv3d.server.cluster import LookUpLocator, WithPools, ViewWithPools from mv3d.server.iserver import ISimulationService from mv3d.server.persist import SQLiteStore, Freezer, FrozenItem from mv3d.util.guide import ChangeNotifier, NotifierProperty from mv3d.util.profiler import timed from tempfile import mkstemp class SimulationError(Exception): """ An exception for simulation problems """ class SimCluster(ShardedPool): """ """ _schemaVersion = 1 def __init__(self, local, uid=None): ShardedPool.__init__(self, local, uid, mechanism=LookUpLocator()) class SimulationService(Service, Securable, Modifiable, WithPools): """ The simulation service does the grunt work of pushing the simulation forward. It contains a list of items that it simulates. Items can be areas, objects or whatever. """ implements(ISimulationService) classModifiers = dict( web=[ClassGenerator("mv3d.server.editor.Stats")], ige=[ClassGenerator("mv3d.server.ige.server.Select"), ClassGenerator("mv3d.server.ige.server.Add"), ClassGenerator("mv3d.server.ige.server.Factory")], guide=[ClassGenerator("mv3d.server.sim.SimConfig")] ) storeInterval = 60 publicLocation = None def __init__(self): Securable.__init__(self) Modifiable.__init__(self) self.store = SQLiteStore() self.store.open() self.items = {} self.interfaces = {} self.realms = {} self.toIterate = [] self.config = {} def configure(self, name, cfg): """ Configure this interface based on the named section in the config file. """ self.config.update(dict(cfg.items(name))) if cfg.has_option(name, "grantPermissions"): parsePermissionConfig(cfg.get(name, "grantPermissions"), self, True) if cfg.has_option(name, "denyPermissions"): parsePermissionConfig(cfg.get(name, "denyPermissions"), self, False) if cfg.has_option(name, "store"): self.store = SQLiteStore() fname = cfg.get(name, "store") if os.path.abspath(fname) != fname and not fname.startswith(":"): fname = os.path.join(self.parent.dataRoot, fname) dirname = os.path.dirname(fname) if not os.path.exists(dirname): os.makedirs(dirname) self.store.open(fname) if cfg.has_option(name, "interfaces"): parseInterfaceConfig(cfg.get(name, "interfaces"), self) if cfg.has_option(name, "publicLocation"): self.publicLocation = ServiceLoc(cfg.get(name, "publicLocation")) @inlineCallbacks def startService(self): """ Initialize the interface, start the coiterator, and join any realms needed. """ yield self.loadPools() @inlineCallbacks def stopService(self): """ Leave any joined realms. """ yield self.onShutdown() self.store.close() @timed def iterate(self, _time): """ Loop through all items that require iteration. By raising StopIteration in their iterate method, items can be removed from the list. To remove an item from the iterate list outside of its iterate method, use removeIteration. """ for realm in self.realms.values(): try: realm[1].iterate(_time) except Exception: deferr() for item in self.toIterate[:]: try: item.iterate(_time) except StopIteration: self.toIterate.remove(item) except Exception: try: self.toIterate.remove(item) except ValueError: pass deferr() @timed def requestIteration(self, item): """ Add an item to our iteration loop. Be careful about doing this. Items should not be iterated if: * They are static (unless specifically required) * They are disabled * They are enabled and do not have any observers or any code that needs to run each iteration. * When in doubt, don't iterate it. """ assert item not in self.toIterate self.toIterate.append(item) @timed def removeIteration(self, item): """ Remove an item from the iteration loop. When in doubt, remove it from the loop. """ try: self.toIterate.remove(item) except ValueError: pass def registerItem(self, item): """ Register a local item on this sim service. All items including cached ones should be registered in this way. This ensures that queries for items first search the local items before contacting remote servers. This will also make sure that the item's realm is cached locally. """ iid = item.getID() assert not self.items.has_key(iid) assert self.realms.has_key(iid[0]) self.items[iid] = item if item.requiresIteration(): self.requestIteration(item) def removeItem(self, itemID): """ Remove a previously registered item from the service. Stops iterating the item if it had previously requested iteration. """ if self.items[itemID] in self.toIterate: self.toIterate.remove(self.items[itemID]) del self.items[itemID] @inlineCallbacks @timed def getRealm(self, realmID): """ Cache a realm locally if it isn't already here. """ if self.realms.has_key(realmID): returnValue(self.realms[realmID][1]) # pylint: disable-msg=E1101 dsvc = yield self.parent.getOneService(self.parent.directoryServices) # pylint: enable-msg=E1101 locator = yield dsvc.getItem("Realms", realmID) holder = yield locator.getItem(self.parent) if locator.foundLocal: realm = holder.realm holder = PoolMember(holder, 0, locator.locations[0]) else: realm = yield holder.pool.callRemote("getRealm") self.realms[realmID] = (holder, realm) realm.setupWorld() returnValue(self.realms[realmID][1]) def getLocalRealm(self, realmID): """ Returns a realm if we have it locally. """ return self.realms[realmID][1] @inlineCallbacks @timed def getItem(self, itemID): """ Check locally for an item and if not found, get it off of the server it lives on and cache it locally. """ if self.items.has_key(itemID): returnValue(self.items[itemID]) yield self.getRealm(itemID[0]) realm = self.realms[itemID[0]][0] locator = yield realm.callOnPool("getItem", itemID) self.items[itemID] = yield locator.getItem(self.parent) returnValue(self.items[itemID]) def getLocalItem(self, itemID): """ Only returns local items. """ return self.items[itemID] @inlineCallbacks def getRealmInfo(self, realmID): """ Returns the name of the realm currently, but could return a dict with a desc and other things (current time/weather/etc). """ realm = yield self.getRealm(realmID) returnValue(realm.getName()) def getStatistics(self): """ Return a dict of stats """ res = dict(totalItems=len(self.items), iteratingItems=len(self.toIterate)) return res def getTotalItems(self): """ Return the total number of local items """ return len(self.items) def getIteratingItems(self): """ Return the total number of iterating items """ return len(self.toIterate) def registerStats(self, stats): """ Register available stats with the visualizer """ stats.addStat("simulatingItems", 1, self.getTotalItems) stats.addStat("iteratingItems", 1, self.getIteratingItems) def getConfig(self): """ Returns a dict with the config of this sim. """ return self.config def setStore(self, store): """ Sets the datastore this sim uses """ if store != self.store.filename: self.store.close() self.store = SQLiteStore() self.store.open(store) def setStoreInterval(self, interval): """ Sets the time between storing items in the store """ if interval == self.storeInterval: return self.storeInterval = interval @inlineCallbacks def joinCluster(self, location, simClusterID): """ Join this Sim server to a SimCluster. """ if not self.pools or (self.pools and not self.pools.has_key( simClusterID)): pool = SimCluster(self.publicLocation, uid=simClusterID) self.addPool(pool) else: pool = self.pools[simClusterID] if len(pool.getMembers()) != 1: raise SimulationError("Can't join a cluster that is already" "active!") yield pool.join(self.parent, location) def newCluster(self): """ Create a new Sim cluster. """ pool = SimCluster(self.publicLocation) self.addPool(pool) return pool.uid @inlineCallbacks def newMasterArea(self, cgen, realmID, simCluster=None): """ Create a new area using the specified class generator in the given realm. It should be part of the simCluster. If simCluster is None, then if we are part of a cluster, it'll be added to that. Otherwise, a new one will be created. """ # ?? possibly need to add it as an item? if simCluster is None: if self.pools is None: self.pools = {} for cluster in self.pools.values(): if isinstance(cluster, SimCluster): break else: print "Creating cluster!" cluster = SimCluster("self/Sim") self.addPool(cluster) else: cluster = self.pools[simCluster] yield self.getRealm(realmID) realmHolder, _realm = self.realms[realmID] aid = yield realmHolder.callOnPool("newItem", self.parent, cluster.uid, cluster.getMembers()) area = yield cgen.construct("self/Sim", aid) self.addPool(area) yield cluster.addMasterPool(self.parent, ["self/Sim"], aid) yield cluster.updateMechanism("addItem", aid, aid) yield area.create(self, cluster.uid, (0, 0, 0), 10000) returnValue(aid) @inlineCallbacks def freezeArea(self, aid, includeObjects=True, positionOffset=None): """ Freeze a whole area and return the data. """ area = yield self.getItem(aid) handle, tmpFile = mkstemp() newStore = SQLiteStore() newStore.open(tmpFile) freezer = Freezer(newStore) freezer.freezeArea(area, includeObjects, positionOffset) newStore.close() fil = os.fdopen(handle, "r") data = fil.read() fil.close() os.remove(tmpFile) returnValue(data) @inlineCallbacks def freezeObjects(self, itemIDs, positionOffset=None): """ Freeze selected items with a position offset (optional) and return the data file. """ handle, tmpFile = mkstemp() newStore = SQLiteStore() newStore.open(tmpFile) try: freezer = Freezer(newStore) for itemID in itemIDs: item = yield self.getItem(itemID) freezer.freezeObject(item, positionOffset) newStore.close() fil = os.fdopen(handle, "r") data = fil.read() fil.close() returnValue(data) finally: try: newStore.close() except: pass try: fil.close() except: pass try: os.remove(tmpFile) except: pass @inlineCallbacks def thawObjects(self, data, areaID, positionOffset=None): """ Thaw all the objects in the file. Returns a list of object ids """ handle, tmpFile = mkstemp() fil = os.fdopen(handle, "wb") try: fil.write(data) fil.close() newStore = SQLiteStore() newStore.open(tmpFile) freezer = Freezer(newStore) iids = [] area = yield self.getItem(areaID) for fitem in FrozenItem.query(newStore): iids.append((yield freezer.thawObject(fitem, area, positionOffset))) newStore.close() returnValue(iids) finally: try: newStore.close() except: pass try: fil.close() except: pass try: os.remove(tmpFile) except: pass def getLoad(self): """ Returns the load index of this sim which takes into account several things. """ load = len(self.items) / 100.0 load += len(self.toIterate) / 25.0 parLoad = self.parent.getLoad() if parLoad > 0.0: load += 1.0 / parLoad else: load += 10000.0 # we're not even iterating so very high load return load def setPublicLocation(self, location): """ Sets the public location of the sim """ self.publicLocation = ServiceLoc(location) @requirePermissions("reference") def getInterface(self, _client, protocol): """ Hand out public interfaces """ return self.interfaces[protocol] def isLocal(self): """ We are local so return true """ return True class SimServiceView(pb.Viewable, ViewWithPools): """ This type of service does the majority of the work simulating the virtual world. """ def __init__(self, service): self.service = service def getProtocol(self): """ Return what protocol we implement """ return "pb" @checkServicePermissions("modify") @viewed def view_addMasterItem(self, client, cg, i): """ Add a new master item remotely """ @checkServicePermissions("modify") @viewed def view_remItem(self, client, i): """ Remove an item via a remote call """ @checkServicePermissions("read") @viewed def view_listContents(self, con): """ Remotely list all objects on this server """ @checkServicePermissions("read") @viewed def view_getLoad(self, con): """ Remotely check the load of this server """ @checkServicePermissions("modify") @viewed def view_addSlaveItem(self, client, i, **kw): """ Remotely add a slave item to this server """ @checkServicePermissions("modify") def view_remSlaveItem(self, iid): """ Change a slave item to only be a cached item Returns a deferred. """ @checkServicePermissions("modify") @viewed def view_promoteSlaveItem(self, client, i): """ Remotely promote a slave item to master. """ @checkServicePermissions("modify") @viewed def view_demoteMasterItem(self, client, i, ns): """ Remotely demote a master item to a slave """ @checkServicePermissions("read") @viewed def view_getAllInRealm(self, client, rid, pris=(1, 2, 3)): """ Return the list of top level objects we are simulating as a priority in pris in realm rid """ @checkServicePermissions("read") def view_getItem(self, con, i): """ Remotely get an item """ d = self.service.getItem(i) def gotItem(itm): if hasattr(itm, "getClassGenerator"): cg = itm.getClassGenerator() elif hasattr(itm, "getClassGenerator"): cg = itm.getClassGenerator() else: cg = ClassGenerator(constructor=itm) d = defer.maybeDeferred(con.readyClass, cg) d.addCallback(lambda _ignored: itm) return d d.addCallback(gotItem) return d @checkServicePermissions("read") @viewed def view_haveItem(self, client, i): """ Remotely ask if this server has an item """ @checkServicePermissions("read") @viewed def view_getStatistics(self, client): """ Retrieves the statistics from the realm server """ @checkServicePermissions("read") @viewed def view_getConfig(self, client): """ Returns the config for this sim. """ @checkServicePermissions("modify") @viewed def view_setStore(self, client, store): """ Sets the datastore this sim uses """ @checkServicePermissions("modify") @viewed def view_setStoreInterval(self, client, interval): """ Sets the time between storing items in the store """ @checkServicePermissions("modify") @viewed def view_setPublicLocation(self, client, location): """ Sets the public location of the sim """ @checkServicePermissions("modify") @viewed def view_newMasterArea(self, client, cgen, realmID, simCluster=None): """ Create a new area using the specified class generator in the given realm. It should be part of the simCluster. If simCluster is None, then if we are part of a cluster, it'll be added to that. Otherwise, a new one will be created. """ @checkServicePermissions("read") @viewed def view_freezeArea(self, client, aid, includeObjects=True, positionOffset=None): """ Freeze a whole area optionally including objects and optionally setting their position offset. Returns the data of the frozen file. """ @checkServicePermissions("read") @viewed def view_freezeObjects(self, client, itemIDs, positionOffset=None): """ Freeze selected items with a position offset (optional) and return the data file. """ @checkServicePermissions("modify") @viewed def view_thawObjects(self, client, data, areaID, positionOffset=None): """ Thaw all the objects in the file. Returns a list of object ids """ @checkServicePermissions("modify") @viewed def view_joinCluster(self, client, location, simClusterID): """ Join this Sim server to a SimCluster. """ @checkServicePermissions("modify") @viewed def view_newCluster(self, client): """ Create a new Sim cluster. """ class SimConfig(ChangeNotifier): """ Guide interface for changing sim properties """ store = NotifierProperty("_store", "store/sim") publicLocation = NotifierProperty("_publicLocation", "self/Sim") window = None def __init__(self, sim): self.sim = sim self.getConfig() self.doneDeferred = defer.Deferred() @inlineCallbacks def getConfig(self): """ Called on startup to load everything """ config = yield self.sim.getConfig() self.store = config["store"] self.publicLocation = config.get("publicLocation") try: from mv3d.tools.guidewx import WxParser parser = WxParser() fname = os.path.abspath(os.path.join(os.path.dirname(mv3d.__file__), "..", "templates", "guide", "simconfig.xml")) self.window = parser.load(fname, self) except: deferr() self.window.show() def cancel(self, _obj, _property): """ Called when the user hits the cancel button """ if self.doneDeferred is not None: self.doneDeferred.callback(False) self.doneDeferred = None self.window.destroy() @inlineCallbacks def submit(self, _obj, _property): """ Called when the user hits the submit button. """ if self.doneDeferred is not None: yield self.sim.setStore(self.store) yield self.sim.setPublicLocation(self.publicLocation) self.doneDeferred.callback(True) self.doneDeferred = None self.window.destroy()