From f71af385056aa1948f62a04a32c7791d6c20a1ab Mon Sep 17 00:00:00 2001 From: Max Zwiessele Date: Thu, 13 Feb 2014 15:06:05 +0000 Subject: [PATCH 1/7] fixing now works, removing parameters needs fixing --- GPy/__init__.py | 1 + GPy/core/parameterization/index_operations.py | 15 ++- GPy/core/parameterization/parameter_core.py | 91 ++++++++----- GPy/core/parameterization/parameterized.py | 127 +++--------------- GPy/core/parameterization/transformations.py | 9 ++ GPy/kern/parts/rbf.py | 2 +- GPy/plotting/matplot_dep/visualize.py | 2 +- 7 files changed, 100 insertions(+), 147 deletions(-) diff --git a/GPy/__init__.py b/GPy/__init__.py index 41369a50..94128dfa 100644 --- a/GPy/__init__.py +++ b/GPy/__init__.py @@ -4,6 +4,7 @@ import warnings warnings.filterwarnings("ignore", category=DeprecationWarning) import core +from core.parameterization import transformations import models import mappings import inference diff --git a/GPy/core/parameterization/index_operations.py b/GPy/core/parameterization/index_operations.py index d2549dad..091b6372 100644 --- a/GPy/core/parameterization/index_operations.py +++ b/GPy/core/parameterization/index_operations.py @@ -57,9 +57,11 @@ class ParameterIndexOperations(object): You can give an offset to set an offset for the given indices in the index array, for multi-param handling. ''' - def __init__(self): + def __init__(self, constraints=None): self._properties = IntArrayDict() - #self._reverse = collections.defaultdict(list) + if constraints is not None: + for t, i in constraints.iteritems(): + self.add(t, i) def __getstate__(self): return self._properties#, self._reverse @@ -191,7 +193,7 @@ class ParameterIndexOperationsView(object): def indices(self): - [ind for ind in self.iterindices()] + return [ind for ind in self.iterindices()] def properties_for(self, index): @@ -206,6 +208,8 @@ class ParameterIndexOperationsView(object): removed = self._param_index_ops.remove(prop, indices+self._offset) if removed.size > 0: return removed - self._size + 1 + if self[prop].size == 0: + del self[prop] return removed @@ -222,6 +226,9 @@ class ParameterIndexOperationsView(object): def update(self, parameter_index_view): for i, v in parameter_index_view.iteritems(): self.add(i, v) - + + + def copy(self): + return ParameterIndexOperations(dict(self.iteritems())) pass diff --git a/GPy/core/parameterization/parameter_core.py b/GPy/core/parameterization/parameter_core.py index 6afea470..cfee60bd 100644 --- a/GPy/core/parameterization/parameter_core.py +++ b/GPy/core/parameterization/parameter_core.py @@ -1,7 +1,7 @@ # Copyright (c) 2012, GPy authors (see AUTHORS.txt). # Licensed under the BSD 3-clause license (see LICENSE.txt) -from transformations import Transformation, Logexp, NegativeLogexp, Logistic +from transformations import Transformation, Logexp, NegativeLogexp, Logistic, __fixed__, FIXED, UNFIXED __updated__ = '2013-12-16' @@ -10,11 +10,6 @@ def adjust_name_for_printing(name): return name.replace(" ", "_").replace(".", "_").replace("-","").replace("+","").replace("!","").replace("*","").replace("/","") return '' -#=============================================================================== -# Printing: -__fixed__ = "fixed" -#=============================================================================== - class Observable(object): _observers_ = {} def add_observer(self, observer, callble): @@ -119,6 +114,14 @@ class Indexable(object): def _offset_for(self, param): raise NotImplementedError, "shouldnt happen, offset required from non parameterization object?" + + def _raveled_index_for(self, param): + """ + get the raveled index for a param + that is an int array, containing the indexes for the flattened + param inside this parameterized logic. + """ + raise NotImplementedError, "shouldnt happen, raveld index transformation required from non parameterization object?" class Constrainable(Nameable, Indexable, Parameterizable): def __init__(self, name, default_constraint=None): @@ -126,6 +129,9 @@ class Constrainable(Nameable, Indexable, Parameterizable): self._default_constraint_ = default_constraint from index_operations import ParameterIndexOperations self.constraints = ParameterIndexOperations() + if self._default_constraint_ is not None: + self.constrain(self._default_constraint_) + #=========================================================================== # Fixing Parameters: #=========================================================================== @@ -138,17 +144,40 @@ class Constrainable(Nameable, Indexable, Parameterizable): if value is not None: self[:] = value self.constrain(__fixed__, warning=warning) - self._highest_parent_._set_fixed(self._raveled_index()) + rav_i = self._highest_parent_._raveled_index_for(self) + self._highest_parent_._set_fixed(rav_i) fix = constrain_fixed + def unconstrain_fixed(self): """ This parameter will no longer be fixed. """ unconstrained = self.unconstrain(__fixed__) - import ipdb;ipdb.set_trace() - self._highest_parent_._set_unfixed(unconstrained) - + self._highest_parent_._set_unfixed(unconstrained) unfix = unconstrain_fixed + + def _set_fixed(self, index): + import numpy as np + if not self._has_fixes(): self._fixes_ = np.ones(self.size, dtype=bool) + self._fixes_[index] = FIXED + if np.all(self._fixes_): self._fixes_ = None # ==UNFIXED + + def _set_unfixed(self, index): + import numpy as np + if not self._has_fixes(): self._fixes_ = np.ones(self.size, dtype=bool) + #rav_i = self._raveled_index_for(param)[index] + self._fixes_[index] = UNFIXED + if np.all(self._fixes_): self._fixes_ = None # ==UNFIXED + + def _connect_fixes(self): + import numpy as np + fixed_indices = self.constraints[__fixed__] + if fixed_indices.size > 0: + self._fixes_ = np.ones(self.size, dtype=bool) * UNFIXED + self._fixes_[fixed_indices] = FIXED + else: + self._fixes_ = None + #=========================================================================== # Constrain operations -> done #=========================================================================== @@ -174,18 +203,29 @@ class Constrainable(Nameable, Indexable, Parameterizable): self._set_params(transform.initialize(self._get_params()), update=False) reconstrained = self.unconstrain() self.constraints.add(transform, self._raveled_index()) - if reconstrained.size > 0: + if warning and reconstrained.size > 0: print "WARNING: reconstraining parameters {}".format(self.parameter_names) if update: self._highest_parent_.parameters_changed() - # if self.has_parent(): - # self._highest_parent_._add_constrain(self, transform, warning) - # else: - # for p in self._parameters_: - # self._add_constrain(p, transform, warning) - # if update: - # self.parameters_changed() + def unconstrain(self, *transforms): + """ + :param transforms: The transformations to unconstrain from. + + remove all :py:class:`GPy.core.transformations.Transformation` + transformats of this parameter object. + """ + if len(transforms) == 0: + transforms = self.constraints.properties() + import numpy as np + removed = np.empty((0,),dtype=int) + for t in transforms: + unconstrained = self.constraints.remove(t, self._raveled_index()) + removed = np.union1d(removed, unconstrained) + if t is __fixed__: + self._highest_parent_._set_unfixed(unconstrained) + return removed + def constrain_positive(self, warning=True, update=True): """ :param warning: print a warning if re-constraining parameters. @@ -211,21 +251,6 @@ class Constrainable(Nameable, Indexable, Parameterizable): """ self.constrain(Logistic(lower, upper), warning=warning, update=update) - def unconstrain(self, *transforms): - """ - :param transforms: The transformations to unconstrain from. - - remove all :py:class:`GPy.core.transformations.Transformation` - transformats of this parameter object. - """ - if len(transforms) == 0: - transforms = self.constraints.properties() - import numpy as np - removed = np.empty((0,),dtype=int) - for t in transforms: - removed = np.union1d(removed, self.constraints.remove(t, self._raveled_index())) - return removed - def unconstrain_positive(self): """ Remove positive constraint of this parameter. diff --git a/GPy/core/parameterization/parameterized.py b/GPy/core/parameterization/parameterized.py index a30a1e3b..80bf2959 100644 --- a/GPy/core/parameterization/parameterized.py +++ b/GPy/core/parameterization/parameterized.py @@ -8,15 +8,10 @@ import cPickle import itertools from re import compile, _pattern_type from param import ParamConcatenation, Param -from parameter_core import Constrainable, Pickleable, Observable, adjust_name_for_printing, Gradcheckable, __fixed__ +from parameter_core import Constrainable, Pickleable, Observable, adjust_name_for_printing, Gradcheckable +from transformations import __fixed__, FIXED, UNFIXED from array_core import ParamList -#=============================================================================== -# constants -FIXED = False -UNFIXED = True -#=============================================================================== - class Parameterized(Constrainable, Pickleable, Observable, Gradcheckable): """ Parameterized class @@ -71,37 +66,6 @@ class Parameterized(Constrainable, Pickleable, Observable, Gradcheckable): self._added_names_ = set() del self._in_init_ - #=========================================================================== - # Parameter connection for model creation: - #=========================================================================== -# def set_as_parameter(self, name, array, gradient, index=None, gradient_parent=None): -# """ -# :param name: name of the param (in print and plots), can be callable without parameters -# :type name: str, callable -# :param array: array which the param consists of -# :type array: array-like -# :param gradient: gradient method of the param -# :type gradient: callable -# :param index: (optional) index of the param when printing -# -# (:param gradient_parent: connect these parameters to this class, but tell -# updates to highest_parent, this is needed when parameterized classes -# contain parameterized classes, but want to access the parameters -# of their children) -# -# -# Set array (e.g. self.X) as param with name and gradient. -# I.e: self.set_as_parameter('curvature', self.lengthscale, self.dK_dlengthscale) -# -# Note: the order in which parameters are added can be adjusted by -# giving an index, of where to put this param in printing -# """ -# if index is None: -# self._parameters_.append(Param(name, array, gradient)) -# else: -# self._parameters_.insert(index, Param(name, array, gradient)) -# self._connect_parameters(gradient_parent=gradient_parent) - def _has_fixes(self): return hasattr(self, "_fixes_") and self._fixes_ is not None @@ -118,53 +82,22 @@ class Parameterized(Constrainable, Pickleable, Observable, Gradcheckable): # if param.has_parent(): # raise AttributeError, "parameter {} already in another model, create new object (or copy) for adding".format(param._short()) if param in self._parameters_ and index is not None: - # make sure fixes and constraints are indexed right - if self._has_fixes(): - param_slice = slice(self._offset_for(param), self._offset_for(param) + param.size) - dest_index = sum((p.size for p in self._parameters_[:index])) - dest_slice = slice(dest_index, dest_index + param.size) - fixes_param = self._fixes_[param_slice].copy() - self._fixes_[param_slice] = self._fixes_[dest_slice] - self._fixes_[dest_slice] = fixes_param - - del self._parameters_[param._parent_index_] - self._parameters_.insert(index, param) + self.remove_parameter(param) + self.add_parameter(param, index) elif param not in self._parameters_: # make sure the size is set - if not hasattr(self, 'size'): - self.size = sum(p.size for p in self._parameters_) if index is None: self._parameters_.append(param) - - # make sure fixes and constraints are indexed right - if param._has_fixes(): fixes_param = param._fixes_.copy() - else: fixes_param = numpy.ones(param.size, dtype=bool) - if self._has_fixes(): self._fixes_ = np.r_[self._fixes_, fixes_param] - elif param._has_fixes(): self._fixes_ = np.r_[np.ones(self.size, dtype=bool), fixes_param] - else: start = sum(p.size for p in self._parameters_[:index]) self.constraints.shift(start, param.size) self._parameters_.insert(index, param) - - # make sure fixes and constraints are indexed right - if param._has_fixes(): fixes_param = param._fixes_.copy() - else: fixes_param = numpy.ones(param.size, dtype=bool) - ins = sum((p.size for p in self._parameters_[:index])) - if self._has_fixes(): self._fixes_ = np.r_[self._fixes_[:ins], fixes_param, self._fixes[ins:]] - elif not np.all(fixes_param): - self._fixes_ = np.ones(self.size + param.size, dtype=bool) - self._fixes_[ins:ins + param.size] = fixes_param self.size += param.size else: raise RuntimeError, """Parameter exists already added and no copy made""" self._connect_parameters() for p in self._parameters_: p._parent_changed(self) - if param._default_constraint_ is not None: - param.constrain(param._default_constraint_, False) - if self._has_fixes() and np.all(self._fixes_): # ==UNFIXED - self._fixes_ = None def add_parameters(self, *parameters): """ @@ -173,18 +106,20 @@ class Parameterized(Constrainable, Pickleable, Observable, Gradcheckable): """ [self.add_parameter(p) for p in parameters] - def remove_parameter(self, *names_params_indices): + def remove_parameter(self, param): """ - :param names_params_indices: mix of parameter_names, param objects, or indices - to remove from being a param of this parameterized object. - - note: if it is a string object it will not (!) be regexp-matched - automatically. + :param param: param object to remove from being a parameter of this parameterized object. """ - self._parameters_ = ParamList([p for p in self._parameters_ - if not (p._parent_index_ in names_params_indices - or p.name in names_params_indices - or p in names_params_indices)]) + if not param in self._parameters_: + raise RuntimeError, "Parameter {} does not belong to this object, remove parameters directly from their respective parents".format(param._short()) + del self._parameters_[param._parent_index_] + self.size -= param.size + constr = param.constraints.copy() + param.constraints.clear() + param.constraints = constr + param._direct_parent_ = None + param._parent_index_ = None + param._connect_fixes() self._connect_parameters() def _connect_parameters(self): @@ -214,6 +149,8 @@ class Parameterized(Constrainable, Pickleable, Observable, Gradcheckable): elif not (pname in not_unique): self.__dict__[pname] = p self._added_names_.add(pname) + self._connect_fixes() + #=========================================================================== # Pickling operations #=========================================================================== @@ -337,7 +274,7 @@ class Parameterized(Constrainable, Pickleable, Observable, Gradcheckable): self._added_names_.add(pname) self.__dict__[pname] = param #=========================================================================== - # Index Handling + # Indexable Handling #=========================================================================== def _backtranslate_index(self, param, ind): # translate an index in parameterized indexing into the index of param @@ -373,36 +310,10 @@ class Parameterized(Constrainable, Pickleable, Observable, Gradcheckable): #=========================================================================== # Fixing parameters: #=========================================================================== - def _set_fixed(self, param_or_index): - if not self._has_fixes(): self._fixes_ = numpy.ones(self.size, dtype=bool) - try: - param_or_index = self._raveled_index_for(param_or_index) - except AttributeError: - pass - self._fixes_[param_or_index] = FIXED - if numpy.all(self._fixes_): self._fixes_ = None # ==UNFIXED - def _set_unfixed(self, param_or_index): - if not self._has_fixes(): self._fixes_ = numpy.ones(self.size, dtype=bool) - try: - param_or_index = self._raveled_index_for(param_or_index) - except AttributeError: - pass - self._fixes_[param_or_index] = UNFIXED - for constr, ind in self.constraints.iteritems(): - if constr is __fixed__: - self._fixes_[ind] = FIXED - if numpy.all(self._fixes_): self._fixes_ = None # ==UNFIXED def _fixes_for(self, param): if self._has_fixes(): return self._fixes_[self._raveled_index_for(param)] return numpy.ones(self.size, dtype=bool)[self._raveled_index_for(param)] - # def _fix(self, param, warning=True): - # f = self._add_constrain(param, __fixed__, warning) - # self._set_fixed(f) - # def _unfix(self, param): - # if self._has_fixes(): - # f = self._remove_constrain(param, __fixed__) - # self._set_unfixed(f) #=========================================================================== # Convenience for fixed, tied checking of param: #=========================================================================== diff --git a/GPy/core/parameterization/transformations.py b/GPy/core/parameterization/transformations.py index e17da404..9582ad4a 100644 --- a/GPy/core/parameterization/transformations.py +++ b/GPy/core/parameterization/transformations.py @@ -6,8 +6,17 @@ import numpy as np from domains import _POSITIVE,_NEGATIVE, _BOUNDED import sys import weakref + _lim_val = -np.log(sys.float_info.epsilon) +#=============================================================================== +# Fixing constants +__fixed__ = "fixed" +FIXED = False +UNFIXED = True +#=============================================================================== + + class Transformation(object): domain = None _instance = None diff --git a/GPy/kern/parts/rbf.py b/GPy/kern/parts/rbf.py index 4b38bd0f..4a8ebd4d 100644 --- a/GPy/kern/parts/rbf.py +++ b/GPy/kern/parts/rbf.py @@ -53,7 +53,7 @@ class RBF(Kernpart): self.variance = Param('variance', variance, Logexp()) - self.lengthscale = Param('lengthscale', lengthscale) + self.lengthscale = Param('lengthscale', lengthscale, Logexp()) self.lengthscale.add_observer(self, self.update_lengthscale) self.update_lengthscale(self.lengthscale) diff --git a/GPy/plotting/matplot_dep/visualize.py b/GPy/plotting/matplot_dep/visualize.py index 99e8a0da..fb085f39 100644 --- a/GPy/plotting/matplot_dep/visualize.py +++ b/GPy/plotting/matplot_dep/visualize.py @@ -4,7 +4,6 @@ import GPy import numpy as np import matplotlib as mpl import time -import Image try: import visual visual_available = True @@ -324,6 +323,7 @@ class image_show(matplotlib_show): else: self.vals = 255*(self.vals - self.vals.min())/(self.vals.max() - self.vals.min()) if not self.palette == []: # applying using an image palette (e.g. if the image has been quantized) + from PIL import Image self.vals = Image.fromarray(self.vals.astype('uint8')) self.vals.putpalette(self.palette) # palette is a list, must be loaded before calling this function From 13212abd0b0b8fb40aebd367c16ccbc334aabeaa Mon Sep 17 00:00:00 2001 From: Max Zwiessele Date: Thu, 13 Feb 2014 16:44:45 +0000 Subject: [PATCH 2/7] parameter adding and removing now fully functional according to tests, including fixes --- GPy/core/parameterization/index_operations.py | 13 ++- GPy/core/parameterization/parameter_core.py | 11 ++- GPy/core/parameterization/parameterized.py | 14 ++- GPy/testing/parameterized_tests.py | 93 +++++++++++++++++++ 4 files changed, 123 insertions(+), 8 deletions(-) create mode 100644 GPy/testing/parameterized_tests.py diff --git a/GPy/core/parameterization/index_operations.py b/GPy/core/parameterization/index_operations.py index 091b6372..b816e05f 100644 --- a/GPy/core/parameterization/index_operations.py +++ b/GPy/core/parameterization/index_operations.py @@ -57,6 +57,7 @@ class ParameterIndexOperations(object): You can give an offset to set an offset for the given indices in the index array, for multi-param handling. ''' + _offset = 0 def __init__(self, constraints=None): self._properties = IntArrayDict() if constraints is not None: @@ -120,6 +121,14 @@ class ParameterIndexOperations(object): return removed.astype(int) return numpy.array([]).astype(int) + def update(self, parameter_index_view, offset=0): + for i, v in parameter_index_view.iteritems(): + self.add(i, v+offset) + + + def copy(self): + return ParameterIndexOperations(dict(self.iteritems())) + def __getitem__(self, prop): return self._properties[prop] @@ -223,9 +232,9 @@ class ParameterIndexOperationsView(object): import pprint return pprint.pformat(dict(self.iteritems())) - def update(self, parameter_index_view): + def update(self, parameter_index_view, offset=0): for i, v in parameter_index_view.iteritems(): - self.add(i, v) + self.add(i, v+offset) def copy(self): diff --git a/GPy/core/parameterization/parameter_core.py b/GPy/core/parameterization/parameter_core.py index cfee60bd..65504652 100644 --- a/GPy/core/parameterization/parameter_core.py +++ b/GPy/core/parameterization/parameter_core.py @@ -72,6 +72,13 @@ class Parentable(object): def has_parent(self): return self._direct_parent_ is not None + def _notify_parent_change(self): + for p in self._parameters_: + p._parent_changed(self) + + def _parent_changed(self): + raise NotImplementedError, "shouldnt happen, Parentable objects need to be able to change their parent" + @property def _highest_parent_(self): if self._direct_parent_ is None: @@ -182,11 +189,9 @@ class Constrainable(Nameable, Indexable, Parameterizable): # Constrain operations -> done #=========================================================================== def _parent_changed(self, parent): - c = self.constraints from index_operations import ParameterIndexOperationsView self.constraints = ParameterIndexOperationsView(parent.constraints, parent._offset_for(self), self.size) - self.constraints.update(c) - del c + self._fixes_ = None for p in self._parameters_: p._parent_changed(parent) diff --git a/GPy/core/parameterization/parameterized.py b/GPy/core/parameterization/parameterized.py index 80bf2959..a976eb93 100644 --- a/GPy/core/parameterization/parameterized.py +++ b/GPy/core/parameterization/parameterized.py @@ -87,17 +87,20 @@ class Parameterized(Constrainable, Pickleable, Observable, Gradcheckable): elif param not in self._parameters_: # make sure the size is set if index is None: + self.constraints.update(param.constraints, self.size) self._parameters_.append(param) else: start = sum(p.size for p in self._parameters_[:index]) self.constraints.shift(start, param.size) + self.constraints.update(param.constraints, start) self._parameters_.insert(index, param) self.size += param.size else: raise RuntimeError, """Parameter exists already added and no copy made""" self._connect_parameters() - for p in self._parameters_: - p._parent_changed(self) + self._notify_parent_change() + self._connect_fixes() + def add_parameters(self, *parameters): """ @@ -120,7 +123,13 @@ class Parameterized(Constrainable, Pickleable, Observable, Gradcheckable): param._direct_parent_ = None param._parent_index_ = None param._connect_fixes() + param._notify_parent_change() + pname = adjust_name_for_printing(param.name) + if pname in self._added_names_: + del self.__dict__[pname] self._connect_parameters() + #self._notify_parent_change() + self._connect_fixes() def _connect_parameters(self): # connect parameterlist to this parameterized object @@ -149,7 +158,6 @@ class Parameterized(Constrainable, Pickleable, Observable, Gradcheckable): elif not (pname in not_unique): self.__dict__[pname] = p self._added_names_.add(pname) - self._connect_fixes() #=========================================================================== # Pickling operations diff --git a/GPy/testing/parameterized_tests.py b/GPy/testing/parameterized_tests.py new file mode 100644 index 00000000..ff57606a --- /dev/null +++ b/GPy/testing/parameterized_tests.py @@ -0,0 +1,93 @@ +''' +Created on Feb 13, 2014 + +@author: maxzwiessele +''' +import unittest +import GPy +import numpy as np + +class Test(unittest.TestCase): + + def setUp(self): + self.rbf = GPy.kern.rbf(1) + self.white = GPy.kern.white(1) + from GPy.core.parameterization import Param + from GPy.core.parameterization.transformations import Logistic + self.param = Param('param', np.random.rand(25,2), Logistic(0, 1)) + + self.test1 = GPy.core.Parameterized("test model") + self.test1.add_parameter(self.white) + self.test1.add_parameter(self.rbf, 0) + self.test1.add_parameter(self.param) + + def test_add_parameter(self): + self.assertEquals(self.rbf._parent_index_, 0) + self.assertEquals(self.white._parent_index_, 1) + pass + + def test_fixes(self): + self.white.fix(warning=False) + self.test1.remove_parameter(self.test1.param) + self.assertTrue(self.test1._has_fixes()) + + from GPy.core.parameterization.transformations import FIXED, UNFIXED + self.assertListEqual(self.test1._fixes_.tolist(),[UNFIXED,UNFIXED,FIXED]) + + self.test1.add_parameter(self.white, 0) + self.assertListEqual(self.test1._fixes_.tolist(),[FIXED,UNFIXED,UNFIXED]) + + + def test_remove_parameter(self): + from GPy.core.parameterization.transformations import FIXED, UNFIXED, __fixed__ + self.white.fix() + self.test1.remove_parameter(self.white) + self.assertIs(self.test1._fixes_,None) + + self.assertListEqual(self.white._fixes_.tolist(), [FIXED]) + self.assertIs(self.white.constraints,self.white.white.constraints._param_index_ops) + self.assertEquals(self.white.white.constraints._offset, 0) + self.assertIs(self.test1.constraints, self.rbf.constraints._param_index_ops) + self.assertIs(self.test1.constraints, self.param.constraints._param_index_ops) + + self.test1.add_parameter(self.white, 0) + self.assertIs(self.test1.constraints, self.white.constraints._param_index_ops) + self.assertIs(self.test1.constraints, self.rbf.constraints._param_index_ops) + self.assertIs(self.test1.constraints, self.param.constraints._param_index_ops) + self.assertListEqual(self.test1.constraints[__fixed__].tolist(), [0]) + self.assertIs(self.white._fixes_,None) + self.assertListEqual(self.test1._fixes_.tolist(),[FIXED] + [UNFIXED] * 52) + self.test1.remove_parameter(self.white) + self.assertIs(self.test1._fixes_,None) + self.assertListEqual(self.white._fixes_.tolist(), [FIXED]) + self.assertIs(self.white.constraints,self.white.white.constraints._param_index_ops) + self.assertIs(self.test1.constraints, self.rbf.constraints._param_index_ops) + self.assertIs(self.test1.constraints, self.param.constraints._param_index_ops) + + def test_add_parameter_already_in_hirarchy(self): + self.test1.add_parameter(self.white._parameters_[0]) + + def test_default_constraints(self): + self.assertIs(self.rbf.rbf.variance.constraints._param_index_ops, self.rbf.constraints._param_index_ops) + self.assertIs(self.test1.constraints, self.rbf.constraints._param_index_ops) + self.assertListEqual(self.rbf.constraints.indices()[0].tolist(), range(2)) + from GPy.core.parameterization.transformations import Logexp + kern = self.rbf+self.white + self.assertListEqual(kern.constraints[Logexp()].tolist(), range(3)) + + def test_constraints(self): + self.rbf.constrain(GPy.transformations.Square(), False) + self.assertListEqual(self.test1.constraints[GPy.transformations.Square()].tolist(), range(2)) + self.assertListEqual(self.test1.constraints[GPy.transformations.Logexp()].tolist(), [2]) + + self.test1.remove_parameter(self.rbf) + self.assertListEqual(self.test1.constraints[GPy.transformations.Square()].tolist(), []) + + def test_constraints_views(self): + self.assertEqual(self.white.constraints._offset, 2) + self.assertEqual(self.rbf.constraints._offset, 0) + self.assertEqual(self.param.constraints._offset, 3) + +if __name__ == "__main__": + #import sys;sys.argv = ['', 'Test.test_add_parameter'] + unittest.main() \ No newline at end of file From c84372127f8165a3b8c28682d6ec060864d42055 Mon Sep 17 00:00:00 2001 From: Max Zwiessele Date: Thu, 13 Feb 2014 16:49:17 +0000 Subject: [PATCH 3/7] checkgrad was changing parameters --- GPy/core/model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/GPy/core/model.py b/GPy/core/model.py index f2632c93..2e2e63ed 100644 --- a/GPy/core/model.py +++ b/GPy/core/model.py @@ -470,6 +470,7 @@ class Model(Parameterized): ng = '%.6f' % float(numerical_gradient) grad_string = "{0:<{c0}}|{1:^{c1}}|{2:^{c2}}|{3:^{c3}}|{4:^{c4}}".format(formatted_name, r, d, g, ng, c0=cols[0] + 9, c1=cols[1], c2=cols[2], c3=cols[3], c4=cols[4]) print grad_string + self._set_params_transformed(x) return ret def input_sensitivity(self): From c78ddde6debb0c6cb2b591aa46313596423fc058 Mon Sep 17 00:00:00 2001 From: Max Zwiessele Date: Thu, 13 Feb 2014 20:24:45 +0000 Subject: [PATCH 4/7] gradcheck now fully functional --- GPy/core/model.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/GPy/core/model.py b/GPy/core/model.py index 2e2e63ed..1252198a 100644 --- a/GPy/core/model.py +++ b/GPy/core/model.py @@ -410,7 +410,13 @@ class Model(Parameterized): f1 = self.objective_function(x + dx) f2 = self.objective_function(x - dx) gradient = self.objective_function_gradients(x) - + + param_list = self._raveled_index_for(target_param) + if self._has_fixes(): + param_list = np.intersect1d(np.r_[:self.size][self._fixes_], param_list, True) + + + numerical_gradient = (f1 - f2) / (2 * dx) global_ratio = (f1 - f2) / (2 * np.dot(dx, np.where(gradient == 0, 1e-32, gradient))) @@ -448,14 +454,14 @@ class Model(Parameterized): ret = True for i, ind in enumerate(param_list): xx = x.copy() - xx[i] += step + xx[ind] += step f1 = self.objective_function(xx) - xx[i] -= 2.*step + xx[ind] -= 2.*step f2 = self.objective_function(xx) - + print ind numerical_gradient = (f1 - f2) / (2 * step) - ratio = (f1 - f2) / (2 * step * gradient[i]) - difference = np.abs((f1 - f2) / 2 / step - gradient[i]) + ratio = (f1 - f2) / (2 * step * gradient[ind]) + difference = np.abs((f1 - f2) / 2 / step - gradient[ind]) if (np.abs(1. - ratio) < tolerance) or np.abs(difference) < tolerance: formatted_name = "\033[92m {0} \033[0m".format(names[ind]) @@ -466,7 +472,7 @@ class Model(Parameterized): r = '%.6f' % float(ratio) d = '%.6f' % float(difference) - g = '%.6f' % gradient[i] + g = '%.6f' % gradient[ind] ng = '%.6f' % float(numerical_gradient) grad_string = "{0:<{c0}}|{1:^{c1}}|{2:^{c2}}|{3:^{c3}}|{4:^{c4}}".format(formatted_name, r, d, g, ng, c0=cols[0] + 9, c1=cols[1], c2=cols[2], c3=cols[3], c4=cols[4]) print grad_string From 9af4c34f90ebdefeb402603496f34e8584e7f318 Mon Sep 17 00:00:00 2001 From: Max Zwiessele Date: Thu, 13 Feb 2014 21:14:08 +0000 Subject: [PATCH 5/7] gradcheck fixes are not easy --- GPy/core/model.py | 53 +++++++++++++-------- GPy/core/parameterization/param.py | 2 +- GPy/core/parameterization/parameter_core.py | 8 ++-- GPy/core/parameterization/parameterized.py | 8 +--- 4 files changed, 42 insertions(+), 29 deletions(-) diff --git a/GPy/core/model.py b/GPy/core/model.py index 1252198a..e7c6c7ec 100644 --- a/GPy/core/model.py +++ b/GPy/core/model.py @@ -411,12 +411,21 @@ class Model(Parameterized): f2 = self.objective_function(x - dx) gradient = self.objective_function_gradients(x) - param_list = self._raveled_index_for(target_param) - if self._has_fixes(): - param_list = np.intersect1d(np.r_[:self.size][self._fixes_], param_list, True) - - + if target_param is None: + transformed_index = range(len(x)) + else: + transformed_index = self._raveled_index_for(target_param) + if self._has_fixes(): + indices = np.r_[:self.size] + which = (transformed_index[:,None]==indices[self._fixes_]).nonzero()[0] + indices -= (~self._fixes_).cumsum() + transformed_index = indices[which] + + if transformed_index.size == 0: + print "No free parameters to check" + return + gradient = gradient[transformed_index] numerical_gradient = (f1 - f2) / (2 * dx) global_ratio = (f1 - f2) / (2 * np.dot(dx, np.where(gradient == 0, 1e-32, gradient))) @@ -439,40 +448,46 @@ class Model(Parameterized): separator = '-' * len(header_string[0]) print '\n'.join([header_string[0], separator]) if target_param is None: - param_list = range(len(x)) + param_index = range(len(x)) + transformed_index = param_index else: - param_list = self._raveled_index_for(target_param) + param_index = self._raveled_index_for(target_param) if self._has_fixes(): - param_list = np.intersect1d(np.r_[:self.size][self._fixes_], param_list, True) - - if param_list.size == 0: + indices = np.r_[:self.size] + which = (param_index[:,None]==indices[self._fixes_][None,:]).nonzero() + transformed_index = (indices-(~self._fixes_).cumsum())[which[1]] + param_index = indices[which[0]] + print param_index, transformed_index + else: + transformed_index = param_index + + if param_index.size == 0: print "No free parameters to check" return gradient = self.objective_function_gradients(x) np.where(gradient == 0, 1e-312, gradient) ret = True - for i, ind in enumerate(param_list): + for nind, xind in itertools.izip(param_index, transformed_index): xx = x.copy() - xx[ind] += step + xx[xind] += step f1 = self.objective_function(xx) - xx[ind] -= 2.*step + xx[xind] -= 2.*step f2 = self.objective_function(xx) - print ind numerical_gradient = (f1 - f2) / (2 * step) - ratio = (f1 - f2) / (2 * step * gradient[ind]) - difference = np.abs((f1 - f2) / 2 / step - gradient[ind]) + ratio = (f1 - f2) / (2 * step * gradient[xind]) + difference = np.abs((f1 - f2) / 2 / step - gradient[xind]) if (np.abs(1. - ratio) < tolerance) or np.abs(difference) < tolerance: - formatted_name = "\033[92m {0} \033[0m".format(names[ind]) + formatted_name = "\033[92m {0} \033[0m".format(names[nind]) ret &= True else: - formatted_name = "\033[91m {0} \033[0m".format(names[ind]) + formatted_name = "\033[91m {0} \033[0m".format(names[nind]) ret &= False r = '%.6f' % float(ratio) d = '%.6f' % float(difference) - g = '%.6f' % gradient[ind] + g = '%.6f' % gradient[xind] ng = '%.6f' % float(numerical_gradient) grad_string = "{0:<{c0}}|{1:^{c1}}|{2:^{c2}}|{3:^{c3}}|{4:^{c4}}".format(formatted_name, r, d, g, ng, c0=cols[0] + 9, c1=cols[1], c2=cols[2], c3=cols[3], c4=cols[4]) print grad_string diff --git a/GPy/core/parameterization/param.py b/GPy/core/parameterization/param.py index 56b8aab6..940cd1a0 100644 --- a/GPy/core/parameterization/param.py +++ b/GPy/core/parameterization/param.py @@ -348,7 +348,7 @@ class Param(ObservableArray, Constrainable, Gradcheckable, Indexable, Parameteri def _description_str(self): if self.size <= 1: return ["%f" % self] else: return [str(self.shape)] - def _parameter_names(self, add_name): + def parameter_names(self, add_name=False): return [self.name] @property def flattened_parameters(self): diff --git a/GPy/core/parameterization/parameter_core.py b/GPy/core/parameterization/parameter_core.py index 65504652..7ad228f7 100644 --- a/GPy/core/parameterization/parameter_core.py +++ b/GPy/core/parameterization/parameter_core.py @@ -25,8 +25,10 @@ class Parameterizable(object): from GPy.core.parameterization.array_core import ParamList _parameters_ = ParamList() - def parameter_names(self): - return [p.name for p in self._parameters_] + def parameter_names(self, add_name=False): + if add_name: + return [adjust_name_for_printing(self.name) + "." + xi for x in self._parameters_ for xi in x.parameter_names(add_name=True)] + return [xi for x in self._parameters_ for xi in x.parameter_names(add_name=True)] def parameters_changed(self): """ @@ -209,7 +211,7 @@ class Constrainable(Nameable, Indexable, Parameterizable): reconstrained = self.unconstrain() self.constraints.add(transform, self._raveled_index()) if warning and reconstrained.size > 0: - print "WARNING: reconstraining parameters {}".format(self.parameter_names) + print "WARNING: reconstraining parameters {}".format(self.parameter_names() or self.name) if update: self._highest_parent_.parameters_changed() diff --git a/GPy/core/parameterization/parameterized.py b/GPy/core/parameterization/parameterized.py index a976eb93..593c37e9 100644 --- a/GPy/core/parameterization/parameterized.py +++ b/GPy/core/parameterization/parameterized.py @@ -434,11 +434,7 @@ class Parameterized(Constrainable, Pickleable, Observable, Gradcheckable): return self._direct_parent_.hirarchy_name() + adjust_name_for_printing(self.name) else: return adjust_name_for_printing(self.name) - def _parameter_names(self, add_name=False): - if add_name: - return [adjust_name_for_printing(self.name) + "." + xi for x in self._parameters_ for xi in x._parameter_names(add_name=True)] - return [xi for x in self._parameters_ for xi in x._parameter_names(add_name=True)] - parameter_names = property(_parameter_names, doc="Names for all parameters handled by this parameterization object -- will add hirarchy name entries for printing") + #parameter_names = property(parameter_names, doc="Names for all parameters handled by this parameterization object -- will add hirarchy name entries for printing") def _collect_gradient(self, target): [p._collect_gradient(target[s]) for p, s in itertools.izip(self._parameters_, self._param_slices_)] @property @@ -468,7 +464,7 @@ class Parameterized(Constrainable, Pickleable, Observable, Gradcheckable): name = adjust_name_for_printing(self.name) + "." constrs = self._constraints_str; ts = self._ties_str - desc = self._description_str; names = self.parameter_names + desc = self._description_str; names = self.parameter_names() nl = max([len(str(x)) for x in names + [name]]) sl = max([len(str(x)) for x in desc + ["Value"]]) cl = max([len(str(x)) if x else 0 for x in constrs + ["Constraint"]]) From ffd09c7820f82d6bc8f3b46ab4b59ee34952b7e6 Mon Sep 17 00:00:00 2001 From: Max Zwiessele Date: Thu, 13 Feb 2014 21:59:08 +0000 Subject: [PATCH 6/7] =?UTF-8?q?checkgrad=20=20=20=20(=E2=95=AF=C2=B0?= =?UTF-8?q?=E2=96=A1=C2=B0=EF=BC=89=E2=95=AF=EF=B8=B5=20=E2=94=BB=E2=94=81?= =?UTF-8?q?=E2=94=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- GPy/core/model.py | 4 ++-- GPy/core/parameterization/array_core.py | 2 +- GPy/core/parameterization/param.py | 7 ++++++- GPy/examples/regression.py | 1 - 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/GPy/core/model.py b/GPy/core/model.py index e7c6c7ec..6bc3edb9 100644 --- a/GPy/core/model.py +++ b/GPy/core/model.py @@ -455,8 +455,8 @@ class Model(Parameterized): if self._has_fixes(): indices = np.r_[:self.size] which = (param_index[:,None]==indices[self._fixes_][None,:]).nonzero() - transformed_index = (indices-(~self._fixes_).cumsum())[which[1]] - param_index = indices[which[0]] + param_index = param_index[which[0]] + transformed_index = (indices-(~self._fixes_).cumsum())[param_index] print param_index, transformed_index else: transformed_index = param_index diff --git a/GPy/core/parameterization/array_core.py b/GPy/core/parameterization/array_core.py index 975b78f4..78b9e8f2 100644 --- a/GPy/core/parameterization/array_core.py +++ b/GPy/core/parameterization/array_core.py @@ -28,8 +28,8 @@ class ObservableArray(np.ndarray, Observable): """ __array_priority__ = -1 # Never give back ObservableArray def __new__(cls, input_array): - cls.__name__ = "ObservableArray\n " obj = np.atleast_1d(input_array).view(cls) + cls.__name__ = "ObservableArray\n " obj._observers_ = {} return obj def __array_finalize__(self, obj): diff --git a/GPy/core/parameterization/param.py b/GPy/core/parameterization/param.py index 940cd1a0..97fcfd32 100644 --- a/GPy/core/parameterization/param.py +++ b/GPy/core/parameterization/param.py @@ -43,6 +43,7 @@ class Param(ObservableArray, Constrainable, Gradcheckable, Indexable, Parameteri _parameters_ = [] def __new__(cls, name, input_array, default_constraint=None): obj = numpy.atleast_1d(super(Param, cls).__new__(cls, input_array=input_array)) + cls.__name__ = "Param" obj._current_slice_ = (slice(obj.shape[0]),) obj._realshape_ = obj.shape obj._realsize_ = obj.size @@ -57,7 +58,7 @@ class Param(ObservableArray, Constrainable, Gradcheckable, Indexable, Parameteri def __init__(self, name, input_array, default_constraint=None): super(Param, self).__init__(name=name, default_constraint=default_constraint) - + def __array_finalize__(self, obj): # see InfoArray.__array_finalize__ for comments if obj is None: return @@ -75,6 +76,7 @@ class Param(ObservableArray, Constrainable, Gradcheckable, Indexable, Parameteri self._original_ = getattr(obj, '_original_', None) self._name = getattr(obj, 'name', None) self.gradient = getattr(obj, 'gradient', None) + self.constraints = getattr(obj, 'constraints', None) def __array_wrap__(self, out_arr, context=None): return out_arr.view(numpy.ndarray) @@ -391,6 +393,9 @@ class Param(ObservableArray, Constrainable, Gradcheckable, Indexable, Parameteri slice_index = self._current_slice_ if isinstance(slice_index, (tuple, list)): clean_curr_slice = [s for s in slice_index if numpy.any(s != Ellipsis)] + for i in range(self._realndim_-len(clean_curr_slice)): + i+=len(clean_curr_slice) + clean_curr_slice += range(self._realshape_[i]) if (all(isinstance(n, (numpy.ndarray, list, tuple)) for n in clean_curr_slice) and len(set(map(len, clean_curr_slice))) <= 1): return numpy.fromiter(itertools.izip(*clean_curr_slice), diff --git a/GPy/examples/regression.py b/GPy/examples/regression.py index f8d3d5a9..c073369f 100644 --- a/GPy/examples/regression.py +++ b/GPy/examples/regression.py @@ -473,7 +473,6 @@ def uncertain_inputs_sparse_regression(max_iters=200, optimize=True, plot=True): Z = np.random.uniform(-3., 3., (7, 1)) k = GPy.kern.rbf(1) - import ipdb;ipdb.set_trace() # create simple GP Model - no input uncertainty on this one m = GPy.models.SparseGPRegression(X, Y, kernel=GPy.kern.rbf(1), Z=Z) From 8d7273b8ff0bc2f91ebebc7ca886251226d10c3f Mon Sep 17 00:00:00 2001 From: Max Zwiessele Date: Fri, 14 Feb 2014 09:57:18 +0000 Subject: [PATCH 7/7] non verbose checkgrad adjusted to new system --- GPy/core/model.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/GPy/core/model.py b/GPy/core/model.py index 6bc3edb9..c944f943 100644 --- a/GPy/core/model.py +++ b/GPy/core/model.py @@ -403,32 +403,34 @@ class Model(Parameterized): x = self._get_params_transformed().copy() if not verbose: - # just check the global ratio - dx = step * np.sign(np.random.uniform(-1, 1, x.size)) - - # evaulate around the point x - f1 = self.objective_function(x + dx) - f2 = self.objective_function(x - dx) - gradient = self.objective_function_gradients(x) - + # make sure only to test the selected parameters if target_param is None: transformed_index = range(len(x)) else: transformed_index = self._raveled_index_for(target_param) if self._has_fixes(): indices = np.r_[:self.size] - which = (transformed_index[:,None]==indices[self._fixes_]).nonzero()[0] - indices -= (~self._fixes_).cumsum() - transformed_index = indices[which] + which = (transformed_index[:,None]==indices[self._fixes_][None,:]).nonzero() + transformed_index = (indices-(~self._fixes_).cumsum())[transformed_index[which[0]]] if transformed_index.size == 0: print "No free parameters to check" return + # just check the global ratio + dx = np.zeros_like(x) + dx[transformed_index] = step * np.sign(np.random.uniform(-1, 1, transformed_index.size)) + + # evaulate around the point x + f1 = self.objective_function(x + dx) + f2 = self.objective_function(x - dx) + gradient = self.objective_function_gradients(x) + + dx = dx[transformed_index] gradient = gradient[transformed_index] + numerical_gradient = (f1 - f2) / (2 * dx) global_ratio = (f1 - f2) / (2 * np.dot(dx, np.where(gradient == 0, 1e-32, gradient))) - return (np.abs(1. - global_ratio) < tolerance) or (np.abs(gradient - numerical_gradient).mean() < tolerance) else: # check the gradient of each parameter individually, and do some pretty printing @@ -457,7 +459,7 @@ class Model(Parameterized): which = (param_index[:,None]==indices[self._fixes_][None,:]).nonzero() param_index = param_index[which[0]] transformed_index = (indices-(~self._fixes_).cumsum())[param_index] - print param_index, transformed_index + #print param_index, transformed_index else: transformed_index = param_index