From 16bd44eb35be6bfe75c31782c052dd345b9023d5 Mon Sep 17 00:00:00 2001 From: Max Zwiessele Date: Fri, 14 Mar 2014 11:32:38 +0000 Subject: [PATCH] changes due to tests in parameterization --- GPy/core/parameterization/index_operations.py | 60 ++++--- GPy/core/parameterization/lists_and_dicts.py | 20 +-- GPy/core/parameterization/param.py | 79 ++++----- GPy/core/parameterization/parameter_core.py | 152 ++++++++++-------- 4 files changed, 151 insertions(+), 160 deletions(-) diff --git a/GPy/core/parameterization/index_operations.py b/GPy/core/parameterization/index_operations.py index f8f6ab5b..c22d8b6b 100644 --- a/GPy/core/parameterization/index_operations.py +++ b/GPy/core/parameterization/index_operations.py @@ -23,17 +23,16 @@ class ParameterIndexOperations(object): if constraints is not None: for t, i in constraints.iteritems(): self.add(t, i) - + def __getstate__(self): - return self._properties#, self._reverse - + return self._properties + def __setstate__(self, state): - self._properties = state[0] - # self._reverse = state[1] + self._properties = state def iteritems(self): return self._properties.iteritems() - + def items(self): return self._properties.items() @@ -42,7 +41,7 @@ class ParameterIndexOperations(object): def iterproperties(self): return self._properties.iterkeys() - + def shift_right(self, start, size): for ind in self.iterindices(): toshift = ind>=start @@ -58,29 +57,26 @@ class ParameterIndexOperations(object): ind[toshift] -= size if ind.size != 0: self._properties[v] = ind else: del self._properties[v] - + def clear(self): self._properties.clear() - + @property def size(self): - return reduce(lambda a,b: a+b.size, self.iterindices(), 0) - + return reduce(lambda a,b: a+b.size, self.iterindices(), 0) + def iterindices(self): return self._properties.itervalues() - + def indices(self): return self._properties.values() def properties_for(self, index): return vectorize(lambda i: [prop for prop in self.iterproperties() if i in self[prop]], otypes=[list])(index) - + def add(self, prop, indices): - try: - self._properties[prop] = combine_indices(self._properties[prop], indices) - except KeyError: - self._properties[prop] = indices - + self._properties[prop] = combine_indices(self._properties[prop], indices) + def remove(self, prop, indices): if prop in self._properties: diff = remove_indices(self[prop], indices) @@ -91,22 +87,22 @@ class ParameterIndexOperations(object): del self._properties[prop] return removed.astype(int) return numpy.array([]).astype(int) - + def update(self, parameter_index_view, offset=0): for i, v in parameter_index_view.iteritems(): self.add(i, v+offset) - + def copy(self): return ParameterIndexOperations(dict(self.iteritems())) - + def __getitem__(self, prop): return self._properties[prop] - + def __str__(self, *args, **kwargs): import pprint return pprint.pformat(dict(self._properties)) - + def combine_indices(arr1, arr2): return numpy.union1d(arr1, arr2) @@ -114,24 +110,22 @@ def remove_indices(arr, to_remove): return numpy.setdiff1d(arr, to_remove, True) def index_empty(index): - return numpy.size(index) == 0 + return numpy.size(index) == 0 class ParameterIndexOperationsView(object): def __init__(self, param_index_operations, offset, size): self._param_index_ops = param_index_operations self._offset = offset self._size = size - + def __getstate__(self): return [self._param_index_ops, self._offset, self._size] - def __setstate__(self, state): self._param_index_ops = state[0] self._offset = state[1] self._size = state[2] - def _filter_index(self, ind): return ind[(ind >= self._offset) * (ind < (self._offset + self._size))] - self._offset @@ -140,7 +134,7 @@ class ParameterIndexOperationsView(object): for i, ind in self._param_index_ops.iteritems(): ind2 = self._filter_index(ind) if ind2.size > 0: - yield i, ind2 + yield i, ind2 def items(self): return [[i,v] for i,v in self.iteritems()] @@ -151,7 +145,7 @@ class ParameterIndexOperationsView(object): def iterproperties(self): for i, _ in self.iteritems(): - yield i + yield i def shift_right(self, start, size): @@ -161,7 +155,7 @@ class ParameterIndexOperationsView(object): self._param_index_ops.shift_left(start+self._offset, size) self._offset -= size self._size -= size - + def clear(self): for i, ind in self.items(): self._param_index_ops.remove(i, ind+self._offset) @@ -198,7 +192,7 @@ class ParameterIndexOperationsView(object): def __getitem__(self, prop): ind = self._filter_index(self._param_index_ops[prop]) return ind - + def __str__(self, *args, **kwargs): import pprint return pprint.pformat(dict(self.iteritems())) @@ -206,8 +200,8 @@ class ParameterIndexOperationsView(object): def update(self, parameter_index_view, offset=0): for i, v in parameter_index_view.iteritems(): self.add(i, v+offset) - - + + def copy(self): return ParameterIndexOperations(dict(self.iteritems())) pass diff --git a/GPy/core/parameterization/lists_and_dicts.py b/GPy/core/parameterization/lists_and_dicts.py index 5b13b3b5..ca0589c9 100644 --- a/GPy/core/parameterization/lists_and_dicts.py +++ b/GPy/core/parameterization/lists_and_dicts.py @@ -5,21 +5,17 @@ Created on 27 Feb 2014 ''' from collections import defaultdict -class DefaultArrayDict(defaultdict): - def __init__(self): + +def intarray_default_factory(): + import numpy as np + return np.int_([]) + +class IntArrayDict(defaultdict): + def __init__(self, default_factory=None): """ Default will be self._default, if not set otherwise """ - defaultdict.__init__(self, self.default_factory) - -class SetDict(DefaultArrayDict): - def default_factory(self): - return set() - -class IntArrayDict(DefaultArrayDict): - def default_factory(self): - import numpy as np - return np.int_([]) + defaultdict.__init__(self, intarray_default_factory) class ArrayList(list): """ diff --git a/GPy/core/parameterization/param.py b/GPy/core/parameterization/param.py index cad20a8a..cd1ce39e 100644 --- a/GPy/core/parameterization/param.py +++ b/GPy/core/parameterization/param.py @@ -49,9 +49,6 @@ class Param(OptimizationHandlable, ObservableArray): obj._realshape_ = obj.shape obj._realsize_ = obj.size obj._realndim_ = obj.ndim - from lists_and_dicts import SetDict - obj._tied_to_me_ = SetDict() - obj._tied_to_ = [] obj._original_ = True obj._gradient_array_ = numpy.zeros(obj.shape, dtype=numpy.float64) return obj @@ -80,13 +77,11 @@ class Param(OptimizationHandlable, ObservableArray): self._parent_index_ = getattr(obj, '_parent_index_', None) self._default_constraint_ = getattr(obj, '_default_constraint_', None) self._current_slice_ = getattr(obj, '_current_slice_', None) - self._tied_to_me_ = getattr(obj, '_tied_to_me_', None) - self._tied_to_ = getattr(obj, '_tied_to_', None) self._realshape_ = getattr(obj, '_realshape_', None) self._realsize_ = getattr(obj, '_realsize_', None) self._realndim_ = getattr(obj, '_realndim_', None) self._original_ = getattr(obj, '_original_', None) - self._name = getattr(obj, 'name', None) + self._name = getattr(obj, '_name', None) self._gradient_array_ = getattr(obj, '_gradient_array_', None) self.constraints = getattr(obj, 'constraints', None) self.priors = getattr(obj, 'priors', None) @@ -106,10 +101,10 @@ class Param(OptimizationHandlable, ObservableArray): #=========================================================================== # Pickling operations #=========================================================================== - def __reduce_ex__(self): + def __reduce__(self): func, args, state = super(Param, self).__reduce__() return func, args, (state, - (self.name, + (self._name, self._parent_, self._parent_index_, self._default_constraint_, @@ -117,16 +112,16 @@ class Param(OptimizationHandlable, ObservableArray): self._realshape_, self._realsize_, self._realndim_, - self._tied_to_me_, - self._tied_to_, + self.constraints, + self.priors ) ) def __setstate__(self, state): super(Param, self).__setstate__(state[0]) state = list(state[1]) - self._tied_to_ = state.pop() - self._tied_to_me_ = state.pop() + self.priors = state.pop() + self.constraints = state.pop() self._realndim_ = state.pop() self._realsize_ = state.pop() self._realshape_ = state.pop() @@ -134,12 +129,13 @@ class Param(OptimizationHandlable, ObservableArray): self._default_constraint_ = state.pop() self._parent_index_ = state.pop() self._parent_ = state.pop() - self.name = state.pop() + self._name = state.pop() def copy(self, *args): constr = self.constraints.copy() priors = self.priors.copy() p = Param(self.name, self.view(numpy.ndarray).copy(), self._default_constraint_) + import ipdb;ipdb.set_trace() p.constraints = constr p.priors = priors return p @@ -180,21 +176,21 @@ class Param(OptimizationHandlable, ObservableArray): #=========================================================================== # Index Operations: #=========================================================================== - def _internal_offset(self): - internal_offset = 0 - extended_realshape = numpy.cumprod((1,) + self._realshape_[:0:-1])[::-1] - for i, si in enumerate(self._current_slice_[:self._realndim_]): - if numpy.all(si == Ellipsis): - continue - if isinstance(si, slice): - a = si.indices(self._realshape_[i])[0] - elif isinstance(si, (list,numpy.ndarray,tuple)): - a = si[0] - else: a = si - if a < 0: - a = self._realshape_[i] + a - internal_offset += a * extended_realshape[i] - return internal_offset + #def _internal_offset(self): + # internal_offset = 0 + # extended_realshape = numpy.cumprod((1,) + self._realshape_[:0:-1])[::-1] + # for i, si in enumerate(self._current_slice_[:self._realndim_]): + # if numpy.all(si == Ellipsis): + # continue + # if isinstance(si, slice): + # a = si.indices(self._realshape_[i])[0] + # elif isinstance(si, (list,numpy.ndarray,tuple)): + # a = si[0] + # else: a = si + # if a < 0: + # a = self._realshape_[i] + a + # internal_offset += a * extended_realshape[i] + # return internal_offset def _raveled_index(self, slice_index=None): # return an index array on the raveled array, which is formed by the current_slice @@ -204,6 +200,9 @@ class Param(OptimizationHandlable, ObservableArray): if ind.ndim < 2: ind = ind[:, None] return numpy.asarray(numpy.apply_along_axis(lambda x: numpy.sum(extended_realshape * x), 1, ind), dtype=int) + def _raveled_index_for(self, obj): + return self._raveled_index() + def _expand_index(self, slice_index=None): # this calculates the full indexing arrays from the slicing objects given by get_item for _real..._ attributes # it basically translates slices to their respective index arrays and turns negative indices around @@ -224,6 +223,11 @@ class Param(OptimizationHandlable, ObservableArray): return numpy.r_[a] return numpy.r_[:b] return itertools.imap(f, itertools.izip_longest(slice_index[:self._realndim_], self._realshape_, fillvalue=slice(self.size))) + #=========================================================================== + # Constrainable + #=========================================================================== + def _ensure_fixes(self): + if not self._has_fixes(): self._fixes_ = numpy.ones(self._realsize_, dtype=bool) #=========================================================================== # Convenience @@ -239,7 +243,6 @@ class Param(OptimizationHandlable, ObservableArray): #round.__doc__ = numpy.round.__doc__ def _get_original(self, param): return self - #=========================================================================== # Printing -> done #=========================================================================== @@ -266,23 +269,11 @@ class Param(OptimizationHandlable, ObservableArray): return [' '.join(map(lambda c: str(c[0]) if c[1].size == self._realsize_ else "{" + str(c[0]) + "}", self.priors.iteritems()))] @property def _ties_str(self): - return [t._short() for t in self._tied_to_] or [''] + return [''] def __repr__(self, *args, **kwargs): name = "\033[1m{x:s}\033[0;0m:\n".format( x=self.hierarchy_name()) return name + super(Param, self).__repr__(*args, **kwargs) - def _ties_for(self, rav_index): - # size = sum(p.size for p in self._tied_to_) - ties = numpy.empty(shape=(len(self._tied_to_), numpy.size(rav_index)), dtype=Param) - for i, tied_to in enumerate(self._tied_to_): - for t, ind in tied_to._tied_to_me_.iteritems(): - if t._parent_index_ == self._parent_index_: - matches = numpy.where(rav_index[:, None] == t._raveled_index()[None, :]) - tt_rav_index = tied_to._raveled_index() - ind_rav_matches = numpy.where(tt_rav_index == numpy.array(list(ind)))[0] - if len(ind) != 1: ties[i, matches[0][ind_rav_matches]] = numpy.take(tt_rav_index, matches[1], mode='wrap')[ind_rav_matches] - else: ties[i, matches[0]] = numpy.take(tt_rav_index, matches[1], mode='wrap') - return map(lambda a: sum(a, []), zip(*[[[tie.flatten()] if tx != None else [] for tx in t] for t, tie in zip(ties, self._tied_to_)])) def _indices(self, slice_index=None): # get a int-array containing all indices in the first axis. if slice_index is None: @@ -322,8 +313,8 @@ class Param(OptimizationHandlable, ObservableArray): ravi = self._raveled_index(filter_) if constr_matrix is None: constr_matrix = self.constraints.properties_for(ravi) if prirs is None: prirs = self.priors.properties_for(ravi) - if ties is None: ties = self._ties_for(ravi) - ties = [' '.join(map(lambda x: x._short(), t)) for t in ties] + if ties is None: ties = [['N/A']]*self.size + ties = [' '.join(map(lambda x: x, t)) for t in ties] if lc is None: lc = self._max_len_names(constr_matrix, __constraints_name__) if lx is None: lx = self._max_len_values() if li is None: li = self._max_len_index(indices) diff --git a/GPy/core/parameterization/parameter_core.py b/GPy/core/parameterization/parameter_core.py index 51b6cddf..d2f066ff 100644 --- a/GPy/core/parameterization/parameter_core.py +++ b/GPy/core/parameterization/parameter_core.py @@ -16,7 +16,7 @@ Observable Pattern for patameterization from transformations import Transformation, Logexp, NegativeLogexp, Logistic, __fixed__, FIXED, UNFIXED import numpy as np -__updated__ = '2014-03-13' +__updated__ = '2014-03-14' class HierarchyError(Exception): """ @@ -31,7 +31,71 @@ def adjust_name_for_printing(name): return name.replace(" ", "_").replace(".", "_").replace("-", "_m_").replace("+", "_p_").replace("!", "_I_").replace("**", "_xx_").replace("*", "_x_").replace("/", "_l_").replace("@",'_at_') return '' -class Observable(object): +class InterfacePickleFunctions(object): + def __init__(self, *a, **kw): + super(InterfacePickleFunctions, self).__init__() + + def _getstate(self): + """ + Returns the state of this class in a memento pattern. + The state must be a list-like structure of all the fields + this class needs to run. + + See python doc "pickling" (`__getstate__` and `__setstate__`) for details. + """ + raise NotImplementedError, "To be able to use pickling you need to implement this method" + def _setstate(self, state): + """ + Set the state (memento pattern) of this class to the given state. + Usually this is just the counterpart to _getstate, such that + an object is a copy of another when calling + + copy = .__new__(*args,**kw)._setstate(._getstate()) + + See python doc "pickling" (`__getstate__` and `__setstate__`) for details. + """ + raise NotImplementedError, "To be able to use pickling you need to implement this method" + +class Pickleable(object): + """ + Make an object pickleable (See python doc 'pickling'). + + This class allows for pickling support by Memento pattern. + _getstate returns a memento of the class, which gets pickled. + _setstate() (re-)sets the state of the class to the memento + """ + def __init__(self, *a, **kw): + super(Pickleable, self).__init__() + #=========================================================================== + # Pickling operations + #=========================================================================== + def pickle(self, f, protocol=-1): + """ + :param f: either filename or open file object to write to. + if it is an open buffer, you have to make sure to close + it properly. + :param protocol: pickling protocol to use, python-pickle for details. + """ + import cPickle + if isinstance(f, str): + with open(f, 'w') as f: + cPickle.dump(self, f, protocol) + else: + cPickle.dump(self, f, protocol) + def __getstate__(self): + if self._has_get_set_state(): + return self._getstate() + return self.__dict__ + def __setstate__(self, state): + if self._has_get_set_state(): + self._setstate(state) + # TODO: maybe parameters_changed() here? + return + self.__dict__ = state + def _has_get_set_state(self): + return '_getstate' in vars(self.__class__) and '_setstate' in vars(self.__class__) + +class Observable(InterfacePickleFunctions): """ Observable pattern for parameterization. @@ -41,7 +105,7 @@ class Observable(object): """ _updated = True def __init__(self, *args, **kwargs): - super(Observable, self).__init__() + super(Observable, self).__init__(*args, **kwargs) self._observer_callables_ = [] def add_observer(self, observer, callble, priority=0): @@ -89,68 +153,16 @@ class Observable(object): ins += 1 self._observer_callables_.insert(ins, (p, o, c)) -class Pickleable(object): - """ - Make an object pickleable (See python doc 'pickling'). - - This class allows for pickling support by Memento pattern. - _getstate returns a memento of the class, which gets pickled. - _setstate() (re-)sets the state of the class to the memento - """ - #=========================================================================== - # Pickling operations - #=========================================================================== - def pickle(self, f, protocol=-1): - """ - :param f: either filename or open file object to write to. - if it is an open buffer, you have to make sure to close - it properly. - :param protocol: pickling protocol to use, python-pickle for details. - """ - import cPickle - if isinstance(f, str): - with open(f, 'w') as f: - cPickle.dump(self, f, protocol) - else: - cPickle.dump(self, f, protocol) - def __getstate__(self): - if self._has_get_set_state(): - return self._getstate() - return self.__dict__ - def __setstate__(self, state): - if self._has_get_set_state(): - self._setstate(state) - # TODO: maybe parameters_changed() here? - return - self.__dict__ = state - def _has_get_set_state(self): - return '_getstate' in vars(self.__class__) and '_setstate' in vars(self.__class__) def _getstate(self): - """ - Returns the state of this class in a memento pattern. - The state must be a list-like structure of all the fields - this class needs to run. - - See python doc "pickling" (`__getstate__` and `__setstate__`) for details. - """ - raise NotImplementedError, "To be able to use pickling you need to implement this method" + return [self._observer_callables_] def _setstate(self, state): - """ - Set the state (memento pattern) of this class to the given state. - Usually this is just the counterpart to _getstate, such that - an object is a copy of another when calling - - copy = .__new__(*args,**kw)._setstate(._getstate()) - - See python doc "pickling" (`__getstate__` and `__setstate__`) for details. - """ - raise NotImplementedError, "To be able to use pickling you need to implement this method" + self._observer_callables_ = state.pop() #=============================================================================== # Foundation framework for parameterized and param objects: #=============================================================================== -class Parentable(object): +class Parentable(Observable): """ Enable an Object to have a parent. @@ -160,7 +172,7 @@ class Parentable(object): _parent_ = None _parent_index_ = None def __init__(self, *args, **kwargs): - super(Parentable, self).__init__() + super(Parentable, self).__init__(*args, **kwargs) def has_parent(self): """ @@ -284,13 +296,6 @@ class Indexable(object): """ raise NotImplementedError, "Need to be able to get the raveled Index" - def _internal_offset(self): - """ - The offset for this parameter inside its parent. - This has to account for shaped parameters! - """ - return 0 - def _offset_for(self, param): """ Return the offset of the param inside this parameterized object. @@ -308,7 +313,7 @@ class Indexable(object): raise NotImplementedError, "shouldnt happen, raveld index transformation required from non parameterization object?" -class Constrainable(Nameable, Indexable, Observable): +class Constrainable(Nameable, Indexable): """ Make an object constrainable with Priors and Transformations. TODO: Mappings!! @@ -367,21 +372,26 @@ class Constrainable(Nameable, Indexable, Observable): self._highest_parent_._set_unfixed(unconstrained) unfix = unconstrain_fixed - def _set_fixed(self, index): + def _ensure_fixes(self): + # Ensure that the fixes array is set: + # Parameterized: ones(self.size) + # Param: ones(self._realsize_ if not self._has_fixes(): self._fixes_ = np.ones(self.size, dtype=bool) + + def _set_fixed(self, index): + self._ensure_fixes() self._fixes_[index] = FIXED if np.all(self._fixes_): self._fixes_ = None # ==UNFIXED def _set_unfixed(self, index): - if not self._has_fixes(): self._fixes_ = np.ones(self.size, dtype=bool) - # rav_i = self._raveled_index_for(param)[index] + self._ensure_fixes() self._fixes_[index] = UNFIXED if np.all(self._fixes_): self._fixes_ = None # ==UNFIXED def _connect_fixes(self): fixed_indices = self.constraints[__fixed__] if fixed_indices.size > 0: - self._fixes_ = np.ones(self.size, dtype=bool) * UNFIXED + self._ensure_fixes() self._fixes_[fixed_indices] = FIXED else: self._fixes_ = None