merge current devel in

GPy/core/gp.py

@@ -10,8 +10,13 @@ from model import Model
 from parameterization import ObsAr
 from .. import likelihoods
 from ..likelihoods.gaussian import Gaussian
-from ..inference.latent_function_inference import exact_gaussian_inference, expectation_propagation
+from ..inference.latent_function_inference import exact_gaussian_inference, expectation_propagation, LatentFunctionInference
 from parameterization.variational import VariationalPosterior
+from scipy.sparse.base import issparse
+
+import logging
+from GPy.util.normalizer import MeanNorm
+logger = logging.getLogger("GP")
 
 class GP(Model):
     """
@@ -21,24 +26,46 @@ class GP(Model):
     :param Y: output observations
     :param kernel: a GPy kernel, defaults to rbf+white
     :param likelihood: a GPy likelihood
+    :param :class:`~GPy.inference.latent_function_inference.LatentFunctionInference` inference_method: The inference method to use for this GP
     :rtype: model object
+    :param Norm normalizer:
+        normalize the outputs Y.
+        Prediction will be un-normalized using this normalizer.
+        If normalizer is None, we will normalize using MeanNorm.
+        If normalizer is False, no normalization will be done.
 
     .. Note:: Multiple independent outputs are allowed using columns of Y
 
 
     """
-    def __init__(self, X, Y, kernel, likelihood, inference_method=None, name='gp', Y_metadata=None):
+    def __init__(self, X, Y, kernel, likelihood, inference_method=None, name='gp', Y_metadata=None, normalizer=False):
         super(GP, self).__init__(name)
 
         assert X.ndim == 2
         if isinstance(X, (ObsAr, VariationalPosterior)):
-            self.X = X
+            self.X = X.copy()
         else: self.X = ObsAr(X)
 
         self.num_data, self.input_dim = self.X.shape
 
         assert Y.ndim == 2
+        logger.info("initializing Y")
+
+        if normalizer is None:
+            self.normalizer = MeanNorm()
+        elif normalizer is False:
+            self.normalizer = None
+        else:
+            self.normalizer = normalizer
+
+        if self.normalizer is not None:
+            self.normalizer.scale_by(Y)
+            self.Y_normalized = ObsAr(self.normalizer.normalize(Y))
+            self.Y = Y
+        else:
             self.Y = ObsAr(Y)
+            self.Y_normalized = self.Y
+
         assert Y.shape[0] == self.num_data
         _, self.output_dim = self.Y.shape
 
@@ -53,6 +80,7 @@ class GP(Model):
         self.likelihood = likelihood
 
         #find a sensible inference method
+        logger.info("initializing inference method")
         if inference_method is None:
             if isinstance(likelihood, likelihoods.Gaussian) or isinstance(likelihood, likelihoods.MixedNoise):
                 inference_method = exact_gaussian_inference.ExactGaussianInference()
@@ -61,11 +89,12 @@ class GP(Model):
                 print "defaulting to ", inference_method, "for latent function inference"
         self.inference_method = inference_method
 
+        logger.info("adding kernel and likelihood as parameters")
         self.add_parameter(self.kern)
         self.add_parameter(self.likelihood)
 
     def parameters_changed(self):
-        self.posterior, self._log_marginal_likelihood, self.grad_dict = self.inference_method.inference(self.kern, self.X, self.likelihood, self.Y, self.Y_metadata)
+        self.posterior, self._log_marginal_likelihood, self.grad_dict = self.inference_method.inference(self.kern, self.X, self.likelihood, self.Y_normalized, self.Y_metadata)
         self.likelihood.update_gradients(self.grad_dict['dL_dthetaL'])
         self.kern.update_gradients_full(self.grad_dict['dL_dK'], self.X)
 
@@ -130,6 +159,8 @@ class GP(Model):
         """
         #predict the latent function values
         mu, var = self._raw_predict(Xnew, full_cov=full_cov, kern=kern)
+        if self.normalizer is not None:
+            mu, var = self.normalizer.inverse_mean(mu), self.normalizer.inverse_variance(var)
 
         # now push through likelihood
         mean, var = self.likelihood.predictive_values(mu, var, full_cov, Y_metadata)
@@ -137,8 +168,32 @@ class GP(Model):
 
     def predict_quantiles(self, X, quantiles=(2.5, 97.5), Y_metadata=None):
         m, v = self._raw_predict(X,  full_cov=False)
+        if self.normalizer is not None:
+            m, v = self.normalizer.inverse_mean(m), self.normalizer.inverse_variance(v)
         return self.likelihood.predictive_quantiles(m, v, quantiles, Y_metadata)
 
+    def predictive_gradients(self, Xnew):
+        """
+        Compute the derivatives of the latent function with respect to X*
+
+        Given a set of points at which to predict X* (size [N*,Q]), compute the
+        derivatives of the mean and variance. Resulting arrays are sized:
+         dmu_dX* -- [N*, Q ,D], where D is the number of output in this GP (usually one).
+         dv_dX*  -- [N*, Q],    (since all outputs have the same variance)
+
+        """
+        dmu_dX = np.empty((Xnew.shape[0],Xnew.shape[1],self.output_dim))
+        for i in range(self.output_dim):
+            dmu_dX[:,:,i] = self.kern.gradients_X(self.posterior.woodbury_vector[:,i:i+1].T, Xnew, self.X)
+
+        # gradients wrt the diagonal part k_{xx}
+        dv_dX = self.kern.gradients_X(np.eye(Xnew.shape[0]), Xnew)
+        #grads wrt 'Schur' part K_{xf}K_{ff}^{-1}K_{fx}
+        alpha = -2.*np.dot(self.kern.K(Xnew, self.X),self.posterior.woodbury_inv)
+        dv_dX += self.kern.gradients_X(alpha, Xnew, self.X)
+        return dmu_dX, dv_dX
+
+
     def posterior_samples_f(self,X,size=10, full_cov=True):
         """
         Samples the posterior GP at the points X.
@@ -152,6 +207,8 @@ class GP(Model):
         :returns: Ysim: set of simulations, a Numpy array (N x samples).
         """
         m, v = self._raw_predict(X,  full_cov=full_cov)
+        if self.normalizer is not None:
+            m, v = self.normalizer.inverse_mean(m), self.normalizer.inverse_variance(v)
         v = v.reshape(m.size,-1) if len(v.shape)==3 else v
         if not full_cov:
             Ysim = np.random.multivariate_normal(m.flatten(), np.diag(v.flatten()), size).T
@@ -179,44 +236,105 @@ class GP(Model):
 
         return Ysim
 
-    def plot_f(self, *args, **kwargs):
+    def plot_f(self, plot_limits=None, which_data_rows='all',
+        which_data_ycols='all', fixed_inputs=[],
+        levels=20, samples=0, fignum=None, ax=None, resolution=None,
+        plot_raw=True,
+        linecol=None,fillcol=None, Y_metadata=None, data_symbol='kx'):
         """
-
+        Plot the GP's view of the world, where the data is normalized and before applying a likelihood.
-        Plot the GP's view of the world, where the data is normalized and
+        This is a call to plot with plot_raw=True.
-        before applying a likelihood.
+        Data will not be plotted in this, as the GP's view of the world
-
+        may live in another space, or units then the data.
-        This is a convenience function: arguments are passed to
-        GPy.plotting.matplot_dep.models_plots.plot_f_fit
-
         """
         assert "matplotlib" in sys.modules, "matplotlib package has not been imported."
         from ..plotting.matplot_dep import models_plots
-        return models_plots.plot_fit_f(self,*args,**kwargs)
+        kw = {}
+        if linecol is not None:
+            kw['linecol'] = linecol
+        if fillcol is not None:
+            kw['fillcol'] = fillcol
+        return models_plots.plot_fit(self, plot_limits, which_data_rows,
+                                     which_data_ycols, fixed_inputs,
+                                     levels, samples, fignum, ax, resolution,
+                                     plot_raw=plot_raw, Y_metadata=Y_metadata,
+                                     data_symbol=data_symbol, **kw)
 
-    def plot(self, *args, **kwargs):
+    def plot(self, plot_limits=None, which_data_rows='all',
+        which_data_ycols='all', fixed_inputs=[],
+        levels=20, samples=0, fignum=None, ax=None, resolution=None,
+        plot_raw=False,
+        linecol=None,fillcol=None, Y_metadata=None, data_symbol='kx'):
         """
         Plot the posterior of the GP.
-          - In one dimension, the function is plotted with a shaded region
+          - In one dimension, the function is plotted with a shaded region identifying two standard deviations.
-            identifying two standard deviations.
+          - In two dimsensions, a contour-plot shows the mean predicted function
-          - In two dimsensions, a contour-plot shows the mean predicted
+          - In higher dimensions, use fixed_inputs to plot the GP  with some of the inputs fixed.
-            function
-          - In higher dimensions, use fixed_inputs to plot the GP  with some of
-            the inputs fixed.
 
         Can plot only part of the data and part of the posterior functions
-        using which_data_rows which_data_ycols and which_parts
+        using which_data_rowsm which_data_ycols.
-
-        This is a convenience function: arguments are passed to
-        GPy.plotting.matplot_dep.models_plots.plot_fit
 
+        :param plot_limits: The limits of the plot. If 1D [xmin,xmax], if 2D [[xmin,ymin],[xmax,ymax]]. Defaluts to data limits
+        :type plot_limits: np.array
+        :param which_data_rows: which of the training data to plot (default all)
+        :type which_data_rows: 'all' or a slice object to slice model.X, model.Y
+        :param which_data_ycols: when the data has several columns (independant outputs), only plot these
+        :type which_data_rows: 'all' or a list of integers
+        :param fixed_inputs: a list of tuple [(i,v), (i,v)...], specifying that input index i should be set to value v.
+        :type fixed_inputs: a list of tuples
+        :param resolution: the number of intervals to sample the GP on. Defaults to 200 in 1D and 50 (a 50x50 grid) in 2D
+        :type resolution: int
+        :param levels: number of levels to plot in a contour plot.
+        :type levels: int
+        :param samples: the number of a posteriori samples to plot
+        :type samples: int
+        :param fignum: figure to plot on.
+        :type fignum: figure number
+        :param ax: axes to plot on.
+        :type ax: axes handle
+        :type output: integer (first output is 0)
+        :param linecol: color of line to plot [Tango.colorsHex['darkBlue']]
+        :type linecol:
+        :param fillcol: color of fill [Tango.colorsHex['lightBlue']]
+        :param levels: for 2D plotting, the number of contour levels to use is ax is None, create a new figure
         """
         assert "matplotlib" in sys.modules, "matplotlib package has not been imported."
         from ..plotting.matplot_dep import models_plots
-        return models_plots.plot_fit(self,*args,**kwargs)
+        kw = {}
+        if linecol is not None:
+            kw['linecol'] = linecol
+        if fillcol is not None:
+            kw['fillcol'] = fillcol
+        return models_plots.plot_fit(self, plot_limits, which_data_rows,
+                                     which_data_ycols, fixed_inputs,
+                                     levels, samples, fignum, ax, resolution,
+                                     plot_raw=plot_raw, Y_metadata=Y_metadata,
+                                     data_symbol=data_symbol, **kw)
 
-    def input_sensitivity(self):
+    def input_sensitivity(self, summarize=True):
         """
         Returns the sensitivity for each dimension of this model
         """
-        return self.kern.input_sensitivity()
+        return self.kern.input_sensitivity(summarize=summarize)
 
+    def optimize(self, optimizer=None, start=None, **kwargs):
+        """
+        Optimize the model using self.log_likelihood and self.log_likelihood_gradient, as well as self.priors.
+        kwargs are passed to the optimizer. They can be:
+
+        :param max_f_eval: maximum number of function evaluations
+        :type max_f_eval: int
+        :messages: whether to display during optimisation
+        :type messages: bool
+        :param optimizer: which optimizer to use (defaults to self.preferred optimizer)
+        :type optimizer: string
+
+        TODO: valid args
+        """
+        self.inference_method.on_optimization_start()
+        try:
+            super(GP, self).optimize(optimizer, start, **kwargs)
+        except KeyboardInterrupt:
+            print "KeyboardInterrupt caught, calling on_optimization_end() to round things up"
+            self.inference_method.on_optimization_end()
+            raise

GPy/core/model.py

@@ -20,7 +20,11 @@ class Model(Parameterized):
         super(Model, self).__init__(name)  # Parameterized.__init__(self)
         self.optimization_runs = []
         self.sampling_runs = []
-        self.preferred_optimizer = 'scg'
+        self.preferred_optimizer = 'bfgs'
+        from .parameterization.ties_and_remappings import Tie
+        self.tie = Tie()
+        self.add_parameter(self.tie, -1)
+        self.add_observer(self.tie, self.tie._parameters_changed_notification, priority=-500)
 
     def log_likelihood(self):
         raise NotImplementedError, "this needs to be implemented to use the model class"
@@ -61,7 +65,7 @@ class Model(Parameterized):
         on the current machine.
 
         """
-        initial_parameters = self._get_params_transformed()
+        initial_parameters = self.optimizer_array.copy()
 
         if parallel:
             try:
@@ -97,9 +101,9 @@ class Model(Parameterized):
 
         if len(self.optimization_runs):
             i = np.argmin([o.f_opt for o in self.optimization_runs])
-            self._set_params_transformed(self.optimization_runs[i].x_opt)
+            self.optimizer_array = self.optimization_runs[i].x_opt
         else:
-            self._set_params_transformed(initial_parameters)
+            self.optimizer_array = initial_parameters
 
     def ensure_default_constraints(self, warning=True):
         """
@@ -124,13 +128,15 @@ class Model(Parameterized):
 
         For probabilistic models this is the negative log_likelihood
         (including the MAP prior), so we return it here. If your model is not
-        probabilistic, just return your objective here!
+        probabilistic, just return your objective to minimize here!
         """
         return -float(self.log_likelihood()) - self.log_prior()
 
     def objective_function_gradients(self):
         """
         The gradients for the objective function for the given algorithm.
+        The gradients are w.r.t. the *negative* objective function, as
+        this framework works with *negative* log-likelihoods as a default.
 
         You can find the gradient for the parameters in self.gradient at all times.
         This is the place, where gradients get stored for parameters.
@@ -141,7 +147,7 @@ class Model(Parameterized):
 
         For probabilistic models this is the gradient of the negative log_likelihood
         (including the MAP prior), so we return it here. If your model is not
-        probabilistic, just return your gradient here!
+        probabilistic, just return your *negative* gradient here!
         """
         return -(self._log_likelihood_gradients() + self._log_prior_gradients())
 
@@ -157,7 +163,8 @@ class Model(Parameterized):
         :type x: np.array
         """
         try:
-            self._set_params_transformed(x)
+            # self._set_params_transformed(x)
+            self.optimizer_array = x
             obj_grads = self._transform_gradients(self.objective_function_gradients())
             self._fail_count = 0
         except (LinAlgError, ZeroDivisionError, ValueError):
@@ -180,7 +187,7 @@ class Model(Parameterized):
         :parameter type: np.array
         """
         try:
-            self._set_params_transformed(x)
+            self.optimizer_array = x
             obj = self.objective_function()
             self._fail_count = 0
         except (LinAlgError, ZeroDivisionError, ValueError):
@@ -192,7 +199,7 @@ class Model(Parameterized):
 
     def _objective_grads(self, x):
         try:
-            self._set_params_transformed(x)
+            self.optimizer_array = x
             obj_f, obj_grads = self.objective_function(), self._transform_gradients(self.objective_function_gradients())
             self._fail_count = 0
         except (LinAlgError, ZeroDivisionError, ValueError):
@@ -220,14 +227,18 @@ class Model(Parameterized):
         if self.is_fixed:
             raise RuntimeError, "Cannot optimize, when everything is fixed"
         if self.size == 0:
-            raise RuntimeError, "Model without parameters cannot be minimized"
+            raise RuntimeError, "Model without parameters cannot be optimized"
+
+        if start == None:
+            start = self.optimizer_array
 
         if optimizer is None:
             optimizer = self.preferred_optimizer
 
-        if start == None:
+        if isinstance(optimizer, optimization.Optimizer):
-            start = self._get_params_transformed()
+            opt = optimizer
-
+            opt.model = self
+        else:
             optimizer = optimization.get_optimizer(optimizer)
             opt = optimizer(start, model=self, **kwargs)
 
@@ -235,7 +246,7 @@ class Model(Parameterized):
 
         self.optimization_runs.append(opt)
 
-        self._set_params_transformed(opt.x_opt)
+        self.optimizer_array = opt.x_opt
 
     def optimize_SGD(self, momentum=0.1, learning_rate=0.01, iterations=20, **kwargs):
         # assert self.Y.shape[1] > 1, "SGD only works with D > 1"
@@ -246,7 +257,7 @@ class Model(Parameterized):
     def _checkgrad(self, target_param=None, verbose=False, step=1e-6, tolerance=1e-3):
         """
         Check the gradient of the ,odel by comparing to a numerical
-        estimate.  If the verbose flag is passed, invividual
+        estimate.  If the verbose flag is passed, individual
         components are tested (and printed)
 
         :param verbose: If True, print a "full" checking of each parameter
@@ -260,7 +271,7 @@ class Model(Parameterized):
            The gradient is considered correct if the ratio of the analytical
            and numerical gradients is within <tolerance> of unity.
         """
-        x = self._get_params_transformed().copy()
+        x = self.optimizer_array.copy()
 
         if not verbose:
             # make sure only to test the selected parameters
@@ -270,8 +281,8 @@ class Model(Parameterized):
                 transformed_index = self._raveled_index_for(target_param)
                 if self._has_fixes():
                     indices = np.r_[:self.size]
-                    which = (transformed_index[:,None]==indices[self._fixes_][None,:]).nonzero()
+                    which = (transformed_index[:, None] == indices[self._fixes_][None, :]).nonzero()
-                    transformed_index = (indices-(~self._fixes_).cumsum())[transformed_index[which[0]]]
+                    transformed_index = (indices - (~self._fixes_).cumsum())[transformed_index[which[0]]]
 
                 if transformed_index.size == 0:
                     print "No free parameters to check"
@@ -290,7 +301,7 @@ class Model(Parameterized):
             gradient = gradient[transformed_index]
 
             denominator = (2 * np.dot(dx, gradient))
-            global_ratio = (f1 - f2) / np.where(denominator==0., 1e-32, denominator)
+            global_ratio = (f1 - f2) / np.where(denominator == 0., 1e-32, denominator)
             global_diff = np.abs(f1 - f2) < tolerance and np.allclose(gradient, 0, atol=tolerance)
             if global_ratio is np.nan:
                 global_ratio = 0
@@ -319,10 +330,10 @@ class Model(Parameterized):
                 param_index = self._raveled_index_for(target_param)
                 if self._has_fixes():
                     indices = np.r_[:self.size]
-                    which = (param_index[:,None]==indices[self._fixes_][None,:]).nonzero()
+                    which = (param_index[:, None] == indices[self._fixes_][None, :]).nonzero()
                     param_index = param_index[which[0]]
-                    transformed_index = (indices-(~self._fixes_).cumsum())[param_index]
+                    transformed_index = (indices - (~self._fixes_).cumsum())[param_index]
-                    #print param_index, transformed_index
+                    # print param_index, transformed_index
                 else:
                     transformed_index = param_index
 
@@ -340,9 +351,9 @@ class Model(Parameterized):
                 xx[xind] -= 2.*step
                 f2 = self._objective(xx)
                 numerical_gradient = (f1 - f2) / (2 * step)
-                if np.all(gradient[xind]==0): ratio = (f1-f2) == gradient[xind]
+                if np.all(gradient[xind] == 0): ratio = (f1 - f2) == gradient[xind]
                 else: ratio = (f1 - f2) / (2 * step * gradient[xind])
-                difference = np.abs((f1 - f2) / 2 / step - gradient[xind])
+                difference = np.abs(numerical_gradient - gradient[xind])
 
                 if (np.abs(1. - ratio) < tolerance) or np.abs(difference) < tolerance:
                     formatted_name = "\033[92m {0} \033[0m".format(names[nind])
@@ -358,7 +369,7 @@ class Model(Parameterized):
                 grad_string = "{0:<{c0}}|{1:^{c1}}|{2:^{c2}}|{3:^{c3}}|{4:^{c4}}".format(formatted_name, r, d, g, ng, c0=cols[0] + 9, c1=cols[1], c2=cols[2], c3=cols[3], c4=cols[4])
                 print grad_string
 
-            self._set_params_transformed(x)
+            self.optimizer_array = x
             return ret
 
 

GPy/core/parameterization/index_operations.py

@@ -1,9 +1,26 @@
-# Copyright (c) 2014, Max Zwiessele
+'''
-# Licensed under the BSD 3-clause license (see LICENSE.txt)
+Created on Oct 2, 2013
+
+@author: maxzwiessele
+'''
 import numpy
 from numpy.lib.function_base import vectorize
 from lists_and_dicts import IntArrayDict
 
+def extract_properties_to_index(index, props):
+    prop_index = dict()
+    for i, cl in enumerate(props):
+        for c in cl:
+            ind = prop_index.get(c, list())
+            ind.append(index[i])
+            prop_index[c] = ind
+
+    for c, i in prop_index.items():
+        prop_index[c] = numpy.array(i, dtype=int)
+
+    return prop_index
+
+
 class ParameterIndexOperations(object):
     """
     This object wraps a dictionary, whos keys are _operations_ that we'd like
@@ -92,8 +109,34 @@ class ParameterIndexOperations(object):
         return self._properties.values()
 
     def properties_for(self, index):
+        """
+        Returns a list of properties, such that each entry in the list corresponds
+        to the element of the index given.
+
+        Example:
+        let properties: 'one':[1,2,3,4], 'two':[3,5,6]
+
+        >>> properties_for([2,3,5])
+        [['one'], ['one', 'two'], ['two']]
+        """
         return vectorize(lambda i: [prop for prop in self.iterproperties() if i in self[prop]], otypes=[list])(index)
 
+    def properties_to_index_dict(self, index):
+        """
+        Return a dictionary, containing properties as keys and indices as index
+        Thus, the indices for each constraint, which is contained will be collected as
+        one dictionary
+
+        Example:
+        let properties: 'one':[1,2,3,4], 'two':[3,5,6]
+
+        >>> properties_to_index_dict([2,3,5])
+        {'one':[2,3], 'two':[3,5]}
+        """
+        props = self.properties_for(index)
+        prop_index = extract_properties_to_index(index, props)
+        return prop_index
+
     def add(self, prop, indices):
         self._properties[prop] = combine_indices(self._properties[prop], indices)
 
@@ -200,8 +243,32 @@ class ParameterIndexOperationsView(object):
 
 
     def properties_for(self, index):
+        """
+        Returns a list of properties, such that each entry in the list corresponds
+        to the element of the index given.
+
+        Example:
+        let properties: 'one':[1,2,3,4], 'two':[3,5,6]
+
+        >>> properties_for([2,3,5])
+        [['one'], ['one', 'two'], ['two']]
+        """
         return vectorize(lambda i: [prop for prop in self.iterproperties() if i in self[prop]], otypes=[list])(index)
 
+    def properties_to_index_dict(self, index):
+        """
+        Return a dictionary, containing properties as keys and indices as index
+        Thus, the indices for each constraint, which is contained will be collected as
+        one dictionary
+
+        Example:
+        let properties: 'one':[1,2,3,4], 'two':[3,5,6]
+
+        >>> properties_to_index_dict([2,3,5])
+        {'one':[2,3], 'two':[3,5]}
+        """
+        return extract_properties_to_index(index, self.properties_for(index))
+
 
     def add(self, prop, indices):
         self._param_index_ops.add(prop, indices+self._offset)

GPy/core/parameterization/lists_and_dicts.py

@@ -38,7 +38,12 @@ class ArrayList(list):
         raise ValueError, "{} is not in list".format(item)
     pass
 
-class ObservablesList(object):
+class ObserverList(object):
+    """
+    A list which containts the observables.
+    It only holds weak references to observers, such that unbound
+    observers dont dangle in memory.
+    """
     def __init__(self):
         self._poc = []
 
@@ -46,29 +51,44 @@ class ObservablesList(object):
         p,o,c = self._poc[ind]
         return p, o(), c
 
-    def remove(self, priority, observable, callble):
+    def remove(self, priority, observer, callble):
         """
+        Remove one observer, which had priority and callble.
         """
         self.flush()
         for i in range(len(self) - 1, -1, -1):
             p,o,c = self[i]
-            if priority==p and observable==o and callble==c:
+            if priority==p and observer==o and callble==c:
                 del self._poc[i]
 
     def __repr__(self):
         return self._poc.__repr__()
 
-    def add(self, priority, observable, callble):
+    def add(self, priority, observer, callble):
+        """
+        Add an observer with priority and callble
+        """
+        if observer is not None:
             ins = 0
             for pr, _, _ in self:
                 if priority > pr:
                     break
                 ins += 1
-        self._poc.insert(ins, (priority, weakref.ref(observable), callble))
+            self._poc.insert(ins, (priority, weakref.ref(observer), callble))
 
     def __str__(self):
+        from . import ObsAr, Param
+        from parameter_core import Parameterizable
         ret = []
         curr_p = None
+        
+        def frmt(o):
+            if isinstance(o, ObsAr):
+                return 'ObsArr <{}>'.format(hex(id(o)))
+            elif isinstance(o, (Param,Parameterizable)):
+                return '{}'.format(o.hierarchy_name())
+            else:
+                return repr(o)                
         for p, o, c in self:
             curr = ''
             if curr_p != p:
@@ -77,16 +97,19 @@ class ObservablesList(object):
             else: curr_pre = " "*len(pre)
             curr_p = p
             curr += curr_pre
-            ret.append(curr + ", ".join(map(repr, [o,c])))
+            
+            ret.append(curr + ", ".join([frmt(o), str(c)]))
             return '\n'.join(ret)
 
     def flush(self):
+        """
+        Make sure all weak references, which point to nothing are flushed (deleted)
+        """
         self._poc = [(p,o,c) for p,o,c in self._poc if o() is not None]
 
     def __iter__(self):
         self.flush()
         for p, o, c in self._poc:
-            if o() is not None:
             yield p, o(), c 
 
     def __len__(self):
@@ -94,10 +117,11 @@ class ObservablesList(object):
         return self._poc.__len__()
 
     def __deepcopy__(self, memo):
-        self.flush()
+        s = ObserverList()
-        s = ObservablesList()
+        for p,o,c in self:
             import copy
-        s._poc = copy.deepcopy(self._poc, memo)
+            s.add(p, copy.deepcopy(o, memo), copy.deepcopy(c, memo))
+        s.flush()
         return s
 
     def __getstate__(self):

GPy/core/parameterization/observable_array.py

@@ -1,7 +1,7 @@
 # Copyright (c) 2012, GPy authors (see AUTHORS.txt).
 # Licensed under the BSD 3-clause license (see LICENSE.txt)
 
-__updated__ = '2014-04-15'
+__updated__ = '2014-05-12'
 
 import numpy as np
 from parameter_core import Observable, Pickleable
@@ -15,10 +15,10 @@ class ObsAr(np.ndarray, Pickleable, Observable):
     """
     __array_priority__ = -1 # Never give back ObsAr
     def __new__(cls, input_array, *a, **kw):
+        # allways make a copy of input paramters, as we need it to be in C order:
         if not isinstance(input_array, ObsAr):
-            obj = np.atleast_1d(np.require(input_array, dtype=np.float64, requirements=['W', 'C'])).view(cls)
+            obj = np.atleast_1d(np.require(np.copy(input_array), dtype=np.float64, requirements=['W', 'C'])).view(cls)
         else: obj = input_array
-        #cls.__name__ = "ObsAr" # because of fixed printing of `array` in np printing
         super(ObsAr, obj).__init__(*a, **kw)
         return obj
 
@@ -30,16 +30,22 @@ class ObsAr(np.ndarray, Pickleable, Observable):
     def __array_wrap__(self, out_arr, context=None):
         return out_arr.view(np.ndarray)
 
+    def _setup_observers(self):
+        # do not setup anything, as observable arrays do not have default observers
+        pass
+
     def copy(self):
+        from lists_and_dicts import ObserverList
         memo = {}
         memo[id(self)] = self
+        memo[id(self.observers)] = ObserverList()
         return self.__deepcopy__(memo)
 
     def __deepcopy__(self, memo):
         s = self.__new__(self.__class__, input_array=self.view(np.ndarray).copy())
         memo[id(self)] = s
         import copy
-        s.__dict__.update(copy.deepcopy(self.__dict__, memo))
+        Pickleable.__setstate__(s, copy.deepcopy(self.__getstate__(), memo))
         return s
 
     def __reduce__(self):

GPy/core/parameterization/param.py

@@ -4,7 +4,7 @@
 import itertools
 import numpy
 np = numpy
-from parameter_core import OptimizationHandlable, adjust_name_for_printing
+from parameter_core import Parameterizable, adjust_name_for_printing, Pickleable
 from observable_array import ObsAr
 
 ###### printing
@@ -16,7 +16,7 @@ __precision__ = numpy.get_printoptions()['precision'] # numpy printing precision
 __print_threshold__ = 5
 ######
 
-class Param(OptimizationHandlable, ObsAr):
+class Param(Parameterizable, ObsAr):
     """
     Parameter object for GPy models.
 
@@ -42,10 +42,9 @@ class Param(OptimizationHandlable, ObsAr):
     """
     __array_priority__ = -1 # Never give back Param
     _fixes_ = None
-    _parameters_ = []
+    parameters = []
     def __new__(cls, name, input_array, default_constraint=None):
         obj = numpy.atleast_1d(super(Param, cls).__new__(cls, input_array=input_array))
-        cls.__name__ = "Param"
         obj._current_slice_ = (slice(obj.shape[0]),)
         obj._realshape_ = obj.shape
         obj._realsize_ = obj.size
@@ -58,9 +57,9 @@ class Param(OptimizationHandlable, ObsAr):
 
     def build_pydot(self,G):
         import pydot
-        node = pydot.Node(id(self), shape='record', label=self.name)
+        node = pydot.Node(id(self), shape='trapezium', label=self.name)#, fontcolor='white', color='white')
         G.add_node(node)
-        for o in self.observers.keys():
+        for _, o, _ in self.observers:
             label = o.name if hasattr(o, 'name') else str(o)
             observed_node = pydot.Node(id(o), label=label)
             G.add_node(observed_node)
@@ -88,8 +87,18 @@ class Param(OptimizationHandlable, ObsAr):
 
     @property
     def param_array(self):
+        """
+        As we are a leaf, this just returns self
+        """
         return self
 
+    @property
+    def values(self):
+        """
+        Return self as numpy array view
+        """
+        return self.view(np.ndarray)
+
     @property
     def gradient(self):
         """
@@ -100,11 +109,11 @@ class Param(OptimizationHandlable, ObsAr):
         """
         if getattr(self, '_gradient_array_', None) is None:
             self._gradient_array_ = numpy.empty(self._realshape_, dtype=numpy.float64)
-        return self._gradient_array_[self._current_slice_]
+        return self._gradient_array_#[self._current_slice_]
 
     @gradient.setter
     def gradient(self, val):
-        self._gradient_array_[self._current_slice_] = val
+        self._gradient_array_[:] = val
 
     #===========================================================================
     # Array operations -> done
@@ -112,10 +121,13 @@ class Param(OptimizationHandlable, ObsAr):
     def __getitem__(self, s, *args, **kwargs):
         if not isinstance(s, tuple):
             s = (s,)
-        if not reduce(lambda a, b: a or numpy.any(b is Ellipsis), s, False) and len(s) <= self.ndim:
+        #if not reduce(lambda a, b: a or numpy.any(b is Ellipsis), s, False) and len(s) <= self.ndim:
-            s += (Ellipsis,)
+        #    s += (Ellipsis,)
         new_arr = super(Param, self).__getitem__(s, *args, **kwargs)
-        try: new_arr._current_slice_ = s; new_arr._original_ = self.base is new_arr.base
+        try: 
+            new_arr._current_slice_ = s
+            new_arr._gradient_array_ = self.gradient[s]
+            new_arr._original_ = self.base is new_arr.base
         except AttributeError: pass  # returning 0d array or float, double etc
         return new_arr
 
@@ -130,6 +142,9 @@ class Param(OptimizationHandlable, ObsAr):
     def _raveled_index_for(self, obj):
         return self._raveled_index()
 
+    #===========================================================================
+    # Index recreation
+    #===========================================================================
     def _expand_index(self, slice_index=None):
         # this calculates the full indexing arrays from the slicing objects given by get_item for _real..._ attributes
         # it basically translates slices to their respective index arrays and turns negative indices around
@@ -138,6 +153,8 @@ class Param(OptimizationHandlable, ObsAr):
             slice_index = self._current_slice_
         def f(a):
             a, b = a
+            if isinstance(a, numpy.ndarray) and a.dtype == bool:
+                raise ValueError, "Boolean indexing not implemented, use Param[np.where(index)] to index by boolean arrays!"
             if a not in (slice(None), Ellipsis):
                 if isinstance(a, slice):
                     start, stop, step = a.indices(b)
@@ -170,13 +187,23 @@ class Param(OptimizationHandlable, ObsAr):
     #===========================================================================
     # Pickling and copying
     #===========================================================================
+    def copy(self):
+        return Parameterizable.copy(self, which=self)
+    
     def __deepcopy__(self, memo):
         s = self.__new__(self.__class__, name=self.name, input_array=self.view(numpy.ndarray).copy())
         memo[id(self)] = s        
         import copy
-        s.__dict__.update(copy.deepcopy(self.__dict__, memo))
+        Pickleable.__setstate__(s, copy.deepcopy(self.__getstate__(), memo))
         return s
+    def _setup_observers(self):
+        """
+        Setup the default observers
         
+        1: pass through to parent, if present
+        """
+        if self.has_parent():
+            self.add_observer(self._parent_, self._parent_._pass_through_notify_observers, -np.inf)
     
     #===========================================================================
     # Printing -> done
@@ -228,9 +255,16 @@ class Param(OptimizationHandlable, ObsAr):
                 and len(set(map(len, clean_curr_slice))) <= 1):
                 return numpy.fromiter(itertools.izip(*clean_curr_slice),
                     dtype=[('', int)] * self._realndim_, count=len(clean_curr_slice[0])).view((int, self._realndim_))
+        try:
             expanded_index = list(self._expand_index(slice_index))
-        return numpy.fromiter(itertools.product(*expanded_index),
+            indices = numpy.fromiter(itertools.product(*expanded_index),
                  dtype=[('', int)] * self._realndim_, count=reduce(lambda a, b: a * b.size, expanded_index, 1)).view((int, self._realndim_))
+        except:
+            print "Warning: extended indexing was used"
+            indices = np.indices(self._realshape_, dtype=int)
+            indices = indices[(slice(None),)+slice_index]
+            indices = np.rollaxis(indices, 0, indices.ndim)
+        return indices
     def _max_len_names(self, gen, header):
         gen = map(lambda x: " ".join(map(str, x)), gen)
         return reduce(lambda a, b:max(a, len(b)), gen, len(header))
@@ -272,7 +306,7 @@ class Param(OptimizationHandlable, ObsAr):
 class ParamConcatenation(object):
     def __init__(self, params):
         """
-        Parameter concatenation for convienience of printing regular expression matched arrays
+        Parameter concatenation for convenience of printing regular expression matched arrays
         you can index this concatenation as if it was the flattened concatenation
         of all the parameters it contains, same for setting parameters (Broadcasting enabled).
 
@@ -316,8 +350,8 @@ class ParamConcatenation(object):
             val = val.values()
         ind = numpy.zeros(sum(self._param_sizes), dtype=bool); ind[s] = True;
         vals = self.values(); vals[s] = val
-        [numpy.copyto(p, vals[ps], where=ind[ps])
+        for p, ps in zip(self.params, self._param_slices_):
-         for p, ps in zip(self.params, self._param_slices_)]
+            p.flat[ind[ps]] = vals[ps]
         if update:
             self.update_all_params()
     def values(self):

GPy/core/parameterization/parameterized.py

@@ -2,19 +2,28 @@
 # Licensed under the BSD 3-clause license (see LICENSE.txt)
 
 import numpy; np = numpy
-import cPickle
 import itertools
 from re import compile, _pattern_type
 from param import ParamConcatenation
-from parameter_core import Pickleable, Parameterizable, adjust_name_for_printing
+from parameter_core import HierarchyError, Parameterizable, adjust_name_for_printing
-from transformations import __fixed__
+
-from lists_and_dicts import ArrayList
+import logging
+logger = logging.getLogger("parameters changed meta")
 
 class ParametersChangedMeta(type):
     def __call__(self, *args, **kw):
-        instance = super(ParametersChangedMeta, self).__call__(*args, **kw)
+        self._in_init_ = True
-        instance.parameters_changed()
+        #import ipdb;ipdb.set_trace()
-        return instance
+        self = super(ParametersChangedMeta, self).__call__(*args, **kw)
+        logger.debug("finished init")
+        self._in_init_ = False
+        logger.debug("connecting parameters")
+        self._highest_parent_._connect_parameters()
+        self._highest_parent_._notify_parent_change()
+        self._highest_parent_._connect_fixes()
+        logger.debug("calling parameters changed")
+        self.parameters_changed()
+        return self
 
 class Parameterized(Parameterizable):
     """
@@ -66,30 +75,27 @@ class Parameterized(Parameterizable):
     #===========================================================================
     def __init__(self, name=None, parameters=[], *a, **kw):
         super(Parameterized, self).__init__(name=name, *a, **kw)
-        self._in_init_ = True
+        self.size = sum(p.size for p in self.parameters)
-        self._parameters_ = ArrayList()
-        self.size = sum(p.size for p in self._parameters_)
         self.add_observer(self, self._parameters_changed_notification, -100)
         if not self._has_fixes():
             self._fixes_ = None
         self._param_slices_ = []
-        self._connect_parameters()
+        #self._connect_parameters()
-        del self._in_init_
         self.add_parameters(*parameters)
 
     def build_pydot(self, G=None):
         import pydot  # @UnresolvedImport
         iamroot = False
         if G is None:
-            G = pydot.Dot(graph_type='digraph')
+            G = pydot.Dot(graph_type='digraph', bgcolor=None)
             iamroot=True
-        node = pydot.Node(id(self), shape='record', label=self.name)
+        node = pydot.Node(id(self), shape='box', label=self.name)#, color='white')
         G.add_node(node)
-        for child in self._parameters_:
+        for child in self.parameters:
             child_node = child.build_pydot(G)
-            G.add_edge(pydot.Edge(node, child_node))
+            G.add_edge(pydot.Edge(node, child_node))#, color='white'))
 
-        for o in self.observers.keys():
+        for _, o, _ in self.observers:
             label = o.name if hasattr(o, 'name') else str(o)
             observed_node = pydot.Node(id(o), label=label)
             G.add_node(observed_node)
@@ -101,58 +107,143 @@ class Parameterized(Parameterizable):
         return node
 
     #===========================================================================
-    # Gradient control
+    # Add remove parameters:
     #===========================================================================
-    def _transform_gradients(self, g):
+    def add_parameter(self, param, index=None, _ignore_added_names=False):
-        if self.has_parent():
+        """
-            return g
+        :param parameters:  the parameters to add
-        [numpy.put(g, i, g[i] * c.gradfactor(self.param_array[i])) for c, i in self.constraints.iteritems() if c != __fixed__]
+        :type parameters:   list of or one :py:class:`GPy.core.param.Param`
-        if self._has_fixes(): return g[self._fixes_]
+        :param [index]:     index of where to put parameters
-        return g
 
+        :param bool _ignore_added_names: whether the name of the parameter overrides a possibly existing field
 
-    #===========================================================================
+        Add all parameters to this param class, you can insert parameters
-    # Indexable
+        at any given index using the :func:`list.insert` syntax
-    #===========================================================================
+        """
-    def _offset_for(self, param):
+        if param in self.parameters and index is not None:
-        # get the offset in the parameterized index array for param
+            self.remove_parameter(param)
+            self.add_parameter(param, index)
+        # elif param.has_parent():
+        #    raise HierarchyError, "parameter {} already in another model ({}), create new object (or copy) for adding".format(param._short(), param._highest_parent_._short())
+        elif param not in self.parameters:
             if param.has_parent():
-            if param._parent_._get_original(param) in self._parameters_:
+                def visit(parent, self):
-                return self._param_slices_[param._parent_._get_original(param)._parent_index_].start
+                    if parent is self:
-            return self._offset_for(param._parent_) + param._parent_._offset_for(param)
+                        raise HierarchyError, "You cannot add a parameter twice into the hierarchy"
-        return 0
+                param.traverse_parents(visit, self)
+                param._parent_.remove_parameter(param)
+            # make sure the size is set
+            if index is None:
+                start = sum(p.size for p in self.parameters)
+                self.constraints.shift_right(start, param.size)
+                self.priors.shift_right(start, param.size)
+                self.constraints.update(param.constraints, self.size)
+                self.priors.update(param.priors, self.size)
+                self.parameters.append(param)
+            else:
+                start = sum(p.size for p in self.parameters[:index])
+                self.constraints.shift_right(start, param.size)
+                self.priors.shift_right(start, param.size)
+                self.constraints.update(param.constraints, start)
+                self.priors.update(param.priors, start)
+                self.parameters.insert(index, param)
 
-    def _raveled_index_for(self, param):
+            param.add_observer(self, self._pass_through_notify_observers, -np.inf)
-        """
-        get the raveled index for a param
-        that is an int array, containing the indexes for the flattened
-        param inside this parameterized logic.
-        """
-        if isinstance(param, ParamConcatenation):
-            return numpy.hstack((self._raveled_index_for(p) for p in param.params))
-        return param._raveled_index() + self._offset_for(param)
 
-    def _raveled_index(self):
+            parent = self
-        """
+            while parent is not None:
-        get the raveled index for this object,
+                parent.size += param.size
-        this is not in the global view of things!
+                parent = parent._parent_
-        """
-        return numpy.r_[:self.size]
 
-    #===========================================================================
+            if not self._in_init_:
-    # Convenience for fixed, tied checking of param:
+                self._connect_parameters()
-    #===========================================================================
+                self._notify_parent_change()
-    @property
-    def is_fixed(self):
-        for p in self._parameters_:
-            if not p.is_fixed: return False
-        return True
 
-    def _get_original(self, param):
+                self._highest_parent_._connect_parameters(ignore_added_names=_ignore_added_names)
-        # if advanced indexing is activated it happens that the array is a copy
+                self._highest_parent_._notify_parent_change()
-        # you can retrieve the original param through this method, by passing
+                self._highest_parent_._connect_fixes()
-        # the copy here
+
-        return self._parameters_[param._parent_index_]
+        else:
+            raise HierarchyError, """Parameter exists already, try making a copy"""
+
+
+    def add_parameters(self, *parameters):
+        """
+        convenience method for adding several
+        parameters without gradient specification
+        """
+        [self.add_parameter(p) for p in parameters]
+
+    def remove_parameter(self, param):
+        """
+        :param param: param object to remove from being a parameter of this parameterized object.
+        """
+        if not param in self.parameters:
+            try:
+                raise RuntimeError, "{} does not belong to this object {}, remove parameters directly from their respective parents".format(param._short(), self.name)
+            except AttributeError:
+                raise RuntimeError, "{} does not seem to be a parameter, remove parameters directly from their respective parents".format(str(param))
+
+        start = sum([p.size for p in self.parameters[:param._parent_index_]])
+        self._remove_parameter_name(param)
+        self.size -= param.size
+        del self.parameters[param._parent_index_]
+
+        param._disconnect_parent()
+        param.remove_observer(self, self._pass_through_notify_observers)
+        self.constraints.shift_left(start, param.size)
+
+        self._connect_parameters()
+        self._notify_parent_change()
+
+        parent = self._parent_
+        while parent is not None:
+            parent.size -= param.size
+            parent = parent._parent_
+
+        self._highest_parent_._connect_parameters()
+        self._highest_parent_._connect_fixes()
+        self._highest_parent_._notify_parent_change()
+
+    def _connect_parameters(self, ignore_added_names=False):
+        # connect parameterlist to this parameterized object
+        # This just sets up the right connection for the params objects
+        # to be used as parameters
+        # it also sets the constraints for each parameter to the constraints
+        # of their respective parents
+        if not hasattr(self, "parameters") or len(self.parameters) < 1:
+            # no parameters for this class
+            return
+        if self.param_array.size != self.size:
+            self._param_array_ = np.empty(self.size, dtype=np.float64)
+        if self.gradient.size != self.size:
+            self._gradient_array_ = np.empty(self.size, dtype=np.float64)
+
+        old_size = 0
+        self._param_slices_ = []
+        for i, p in enumerate(self.parameters):
+            if not p.param_array.flags['C_CONTIGUOUS']:
+                raise ValueError, "This should not happen! Please write an email to the developers with the code, which reproduces this error. All parameter arrays must be C_CONTIGUOUS"
+
+            p._parent_ = self
+            p._parent_index_ = i
+
+            pslice = slice(old_size, old_size + p.size)
+
+            # first connect all children
+            p._propagate_param_grad(self.param_array[pslice], self.gradient_full[pslice])
+
+            # then connect children to self
+            self.param_array[pslice] = p.param_array.flat  # , requirements=['C', 'W']).ravel(order='C')
+            self.gradient_full[pslice] = p.gradient_full.flat  # , requirements=['C', 'W']).ravel(order='C')
+
+            p.param_array.data = self.param_array[pslice].data
+            p.gradient_full.data = self.gradient_full[pslice].data
+
+            self._param_slices_.append(pslice)
+
+            self._add_parameter_name(p, ignore_added_names=ignore_added_names)
+            old_size += p.size
 
     #===========================================================================
     # Get/set parameters:
@@ -199,10 +290,38 @@ class Parameterized(Parameterizable):
 
     def __setattr__(self, name, val):
         # override the default behaviour, if setting a param, so broadcasting can by used
-        if hasattr(self, "_parameters_"):
+        if hasattr(self, "parameters"):
+            try:
                 pnames = self.parameter_names(False, adjust_for_printing=True, recursive=False)
-            if name in pnames: self._parameters_[pnames.index(name)][:] = val; return
+                if name in pnames: self.parameters[pnames.index(name)][:] = val; return
+            except AttributeError:
+                pass
         object.__setattr__(self, name, val);
+
+    #===========================================================================
+    # Pickling
+    #===========================================================================
+    def __setstate__(self, state):
+        super(Parameterized, self).__setstate__(state)
+        try:
+            self._connect_parameters()
+            self._connect_fixes()
+            self._notify_parent_change()
+            self.parameters_changed()
+        except Exception as e:
+            print "WARNING: caught exception {!s}, trying to continue".format(e)
+
+    def copy(self, memo=None):
+        if memo is None:
+            memo = {}
+        memo[id(self.optimizer_array)] = None # and param_array
+        memo[id(self.param_array)] = None # and param_array
+        copy = super(Parameterized, self).copy(memo)
+        copy._connect_parameters()
+        copy._connect_fixes()
+        copy._notify_parent_change()
+        return copy
+
     #===========================================================================
     # Printing:
     #===========================================================================
@@ -210,22 +329,22 @@ class Parameterized(Parameterizable):
         return self.hierarchy_name()
     @property
     def flattened_parameters(self):
-        return [xi for x in self._parameters_ for xi in x.flattened_parameters]
+        return [xi for x in self.parameters for xi in x.flattened_parameters]
     @property
     def _parameter_sizes_(self):
-        return [x.size for x in self._parameters_]
+        return [x.size for x in self.parameters]
     @property
     def parameter_shapes(self):
-        return [xi for x in self._parameters_ for xi in x.parameter_shapes]
+        return [xi for x in self.parameters for xi in x.parameter_shapes]
     @property
     def _constraints_str(self):
-        return [cs for p in self._parameters_ for cs in p._constraints_str]
+        return [cs for p in self.parameters for cs in p._constraints_str]
     @property
     def _priors_str(self):
-        return [cs for p in self._parameters_ for cs in p._priors_str]
+        return [cs for p in self.parameters for cs in p._priors_str]
     @property
     def _description_str(self):
-        return [xi for x in self._parameters_ for xi in x._description_str]
+        return [xi for x in self.parameters for xi in x._description_str]
     @property
     def _ties_str(self):
         return [','.join(x._ties_str) for x in self.flattened_parameters]
@@ -245,7 +364,7 @@ class Parameterized(Parameterizable):
         to_print = []
         for n, d, c, t, p in itertools.izip(names, desc, constrs, ts, prirs):
             to_print.append(format_spec.format(name=n, desc=d, const=c, t=t, pri=p))
-        # to_print = [format_spec.format(p=p, const=c, t=t) if isinstance(p, Param) else p.__str__(header=False) for p, c, t in itertools.izip(self._parameters_, constrs, ts)]
+        # to_print = [format_spec.format(p=p, const=c, t=t) if isinstance(p, Param) else p.__str__(header=False) for p, c, t in itertools.izip(self.parameters, constrs, ts)]
         sep = '-' * (nl + sl + cl + + pl + tl + 8 * 2 + 3)
         if header:
             header = "  {{0:<{0}s}}  |  {{1:^{1}s}}  |  {{2:^{2}s}}  |  {{3:^{3}s}}  |  {{4:^{4}s}}".format(nl, sl, cl, pl, tl).format(name, "Value", "Constraint", "Prior", "Tied to")

GPy/core/parameterization/ties_and_remappings.py

@@ -31,48 +31,193 @@ class Fix(Remapping):
 
 
 
-class Tie(Remapping):
+class Tie(Parameterized):
-    def __init__(self, value, name):
+    """
+    The new parameter tie framework. (under development)
+    
+    All the parameters tied together get a new parameter inside the *Tie* object. 
+    Its value should always be equal to all the tied parameters, and its gradient
+    is the sum of all the tied parameters.
+    
+    =====Implementation Details=====
+    The *Tie* object should only exist on the top of param tree (the highest parent).
+    
+    self.label_buf:
+    It uses a label buffer that has the same length as all the parameters (self._highest_parent_.param_array).
+    The buffer keeps track of all the tied parameters. All the tied parameters have a label (an interger) higher 
+    than 0, and the parameters that have the same label are tied together.
+    
+    self.buf_index:
+    An auxiliary index list for the global index of the tie parameter inside the *Tie* object.
+    
+    ================================
+    
+    TODO:
+    * EVERYTHING
+    
+    """
+    def __init__(self, name='tie'):
         super(Tie, self).__init__(name)
-        self.tied_parameters = []
+        self.tied_param = None
-        self.value = Param('val', value)
+        # The buffer keeps track of tie status
-        self.add_parameter(self.value)
+        self.label_buf = None
+        # The global indices of the 'tied' param
+        self.buf_idx = None
+        # A boolean array indicating non-tied parameters
+        self._tie_ = None
         
-    def add_tied_parameter(self, p):
+    def getTieFlag(self, p=None):
-        self.tied_parameters.append(p)
+        if self.tied_param is None:
-        p.add_observer(self, self.callback)
+            if self._tie_ is None or self._tie_.size != self._highest_parent_.param_array.size:
-        self.parameters_changed()
+                self._tie_ = np.ones((self._highest_parent_.param_array.size,),dtype=np.bool)
+        if p is not None:
+            return self._tie_[p._highest_parent_._raveled_index_for(p)]
+        return self._tie_
     
-    def callback(self, param=None, which=None):
+    def _init_labelBuf(self):
+        if self.label_buf is None:
+            self.label_buf = np.zeros(self._highest_parent_.param_array.shape, dtype=np.int)
+        if self._tie_ is None or self._tie_.size != self._highest_parent_.param_array.size:
+            self._tie_ = np.ones((self._highest_parent_.param_array.size,),dtype=np.bool)
+            
+    def _updateTieFlag(self):
+        if self._tie_.size != self.label_buf.size:
+            self._tie_ = np.ones((self._highest_parent_.param_array.size,),dtype=np.bool)
+        self._tie_[self.label_buf>0] = False
+        self._tie_[self.buf_idx] = True
+
+    def add_tied_parameter(self, p, p2=None):
         """
-        This gets called whenever any of the tied parameters changes. we spend
+        Tie the list of parameters p together (p2==None) or 
-        considerable effort working out whhat has changed ant to what value.
+        Tie the list of parameters p with the list of parameters p2 (p2!=None) 
-        Then we store that value in self.value, and broadcast it everywhere
-        with parameters_changed.
         """
-        if which is self:return
+        self._init_labelBuf()
-        index = self._highest_parent_.constraints[self]
+        if p2 is None:
-        if len(index)==0:
+            idx = self._highest_parent_._raveled_index_for(p)
-            return # nothing to tie together, this tie exists without any tied parameters
+            val = self._sync_val_group(idx)            
-        vals = self._highest_parent_.param_array[index]
+            if np.all(self.label_buf[idx]==0):
-        uvals = np.unique(vals)
+                # None of p has been tied before.
-        if len(uvals)==1:
+                tie_idx = self._expandTieParam(1)
-            #all of the tied things are at the same value
+                print tie_idx
-            self.value[...] = uvals[0]
+                tie_id = self.label_buf.max()+1
-        elif len(uvals)==2:
+                self.label_buf[tie_idx] = tie_id
-            #only *one* of the tied things has changed. it must be different to self.value
-            newval = uvals[uvals != self.value*1]
-            self.value[...] = newval
             else:
-            #more than one of the tied things changed. panic.
+                b = self.label_buf[idx]
-            raise ValueError, "something is wrong with the tieing"
+                ids = np.unique(b[b>0])
+                tie_id, tie_idx = self._merge_tie_param(ids)
+            self._highest_parent_.param_array[tie_idx] = val
+            idx = self._highest_parent_._raveled_index_for(p)
+            self.label_buf[idx] = tie_id
+        else:
+            pass
+        self._updateTieFlag()
         
-    def mapping(self):
+    def _merge_tie_param(self, ids):
-        return self.value
+        """Merge the tie parameters with ids in the list."""
+        if len(ids)==1:
+            id_final_idx = self.buf_idx[self.label_buf[self.buf_idx]==ids[0]][0]
+            return ids[0],id_final_idx
+        id_final = ids[0]
+        ids_rm = ids[1:]
+        label_buf_param = self.label_buf[self.buf_idx]
+        idx_param = [np.where(label_buf_param==i)[0][0] for i in ids_rm]
+        self._removeTieParam(idx_param)
+        [np.put(self.label_buf, np.where(self.label_buf==i), id_final) for i in ids_rm]
+        id_final_idx = self.buf_idx[self.label_buf[self.buf_idx]==id_final][0]
+        return id_final, id_final_idx
+        
+    def _sync_val_group(self, idx):
+        self._highest_parent_.param_array[idx] = self._highest_parent_.param_array[idx].mean()
+        return self._highest_parent_.param_array[idx][0]
+        
+    def _expandTieParam(self, num):
+        """Expand the tie param with the number of *num* parameters"""
+        if self.tied_param is None:
+            new_buf = np.empty((num,))
+        else:
+            new_buf = np.empty((self.tied_param.size+num,))
+            new_buf[:self.tied_param.size] = self.tied_param.param_array.copy()
+            self.remove_parameter(self.tied_param)
+        self.tied_param = Param('tied',new_buf)
+        self.add_parameter(self.tied_param)
+        buf_idx_new = self._highest_parent_._raveled_index_for(self.tied_param)
+        self._expand_label_buf(self.buf_idx, buf_idx_new)
+        self.buf_idx = buf_idx_new
+        return self.buf_idx[-num:]
+
+    def _removeTieParam(self, idx):
+        """idx within tied_param"""
+        new_buf = np.empty((self.tied_param.size-len(idx),))
+        bool_list = np.ones((self.tied_param.size,),dtype=np.bool)
+        bool_list[idx] = False
+        new_buf[:] = self.tied_param.param_array[bool_list]
+        self.remove_parameter(self.tied_param)
+        self.tied_param = Param('tied',new_buf)
+        self.add_parameter(self.tied_param)
+        buf_idx_new = self._highest_parent_._raveled_index_for(self.tied_param)
+        self._shrink_label_buf(self.buf_idx, buf_idx_new, bool_list)
+        self.buf_idx = buf_idx_new
+        
+    def _expand_label_buf(self, idx_old, idx_new):
+        """Expand label buffer accordingly"""
+        if idx_old is None:
+            self.label_buf = np.zeros(self._highest_parent_.param_array.shape, dtype=np.int)
+        else:
+            bool_old = np.zeros((self.label_buf.size,),dtype=np.bool)
+            bool_old[idx_old] = True
+            bool_new = np.zeros((self._highest_parent_.param_array.size,),dtype=np.bool)
+            bool_new[idx_new] = True
+            label_buf_new = np.zeros(self._highest_parent_.param_array.shape, dtype=np.int)
+            label_buf_new[np.logical_not(bool_new)] = self.label_buf[np.logical_not(bool_old)]
+            label_buf_new[idx_new[:len(idx_old)]] = self.label_buf[idx_old]
+            self.label_buf = label_buf_new
+
+    def _shrink_label_buf(self, idx_old, idx_new, bool_list):
+        bool_old = np.zeros((self.label_buf.size,),dtype=np.bool)
+        bool_old[idx_old] = True
+        bool_new = np.zeros((self._highest_parent_.param_array.size,),dtype=np.bool)
+        bool_new[idx_new] = True
+        label_buf_new = np.empty(self._highest_parent_.param_array.shape, dtype=np.int)
+        label_buf_new[np.logical_not(bool_new)] = self.label_buf[np.logical_not(bool_old)]
+        label_buf_new[idx_new] = self.label_buf[idx_old[bool_list]]
+        self.label_buf = label_buf_new
+
+    def _check_change(self):
+        changed = False
+        if self.tied_param is not None:
+            for i in xrange(self.tied_param.size):
+                b0 = self.label_buf==self.label_buf[self.buf_idx[i]]
+                b = self._highest_parent_.param_array[b0]!=self.tied_param[i]
+                if b.sum()==0:
+                    print 'XXX'
+                    continue
+                elif b.sum()==1:
+                    print '!!!'
+                    val = self._highest_parent_.param_array[b0][b][0]
+                    self._highest_parent_.param_array[b0] = val
+                else:
+                    print '@@@'
+                    self._highest_parent_.param_array[b0] = self.tied_param[i]
+                changed = True
+        return changed
+
+    def parameters_changed(self):
+        #ensure all out parameters have the correct value, as specified by our mapping
+        changed = self._check_change()
+        if changed:
+            self._highest_parent_._trigger_params_changed()
+        self.collate_gradient()
 
     def collate_gradient(self):
-        index = self._highest_parent_.constraints[self]
+        if self.tied_param is not None:
-        self.value.gradient = np.sum(self._highest_parent_.gradient[index])
+            self.tied_param.gradient = 0.
+            [np.put(self.tied_param.gradient, i, self._highest_parent_.gradient[self.label_buf==self.label_buf[self.buf_idx[i]]].sum()) 
+                for i in xrange(self.tied_param.size)]
+    
+    def propagate_val(self):
+        if self.tied_param is not None:
+            for i in xrange(self.tied_param.size):
+                self._highest_parent_.param_array[self.label_buf==self.label_buf[self.buf_idx[i]]] = self.tied_param[i]
 
 
 

GPy/core/parameterization/transformations.py

@@ -54,7 +54,7 @@ class Transformation(object):
 class Logexp(Transformation):
     domain = _POSITIVE
     def f(self, x):
-        return np.where(x>_lim_val, x, np.log(1. + np.exp(np.clip(x, -_lim_val, _lim_val)))) + epsilon
+        return np.where(x>_lim_val, x, np.log1p(np.exp(np.clip(x, -_lim_val, _lim_val)))) + epsilon
         #raises overflow warning: return np.where(x>_lim_val, x, np.log(1. + np.exp(x)))
     def finv(self, f):
         return np.where(f>_lim_val, f, np.log(np.exp(f+1e-20) - 1.))
@@ -195,6 +195,9 @@ class Logistic(Transformation):
         self.lower, self.upper = float(lower), float(upper)
         self.difference = self.upper - self.lower
     def f(self, x):
+        if (x<-300.).any():
+            x = x.copy()
+            x[x<-300.] = -300.
         return self.lower + self.difference / (1. + np.exp(-x))
     def finv(self, f):
         return np.log(np.clip(f - self.lower, 1e-10, np.inf) / np.clip(self.upper - f, 1e-10, np.inf))

GPy/core/parameterization/variational.py

@@ -34,36 +34,49 @@ class NormalPrior(VariationalPrior):
         variational_posterior.variance.gradient -= (1. - (1. / (variational_posterior.variance))) * 0.5
 
 class SpikeAndSlabPrior(VariationalPrior):
-    def __init__(self, pi, variance = 1.0, name='SpikeAndSlabPrior', **kw):
+    def __init__(self, pi=None, learnPi=False, variance = 1.0, name='SpikeAndSlabPrior', **kw):
         super(VariationalPrior, self).__init__(name=name, **kw)
-        assert variance==1.0, "Not Implemented!"
         self.pi = Param('pi', pi, Logistic(1e-10,1.-1e-10))
         self.variance = Param('variance',variance)
+        self.learnPi = learnPi
+        if learnPi:
             self.add_parameters(self.pi)
-        self.group_spike_prob = False
 
     def KL_divergence(self, variational_posterior):
         mu = variational_posterior.mean
         S = variational_posterior.variance
         gamma = variational_posterior.binary_prob
-        var_mean = np.square(mu)
+        if len(self.pi.shape)==2:
-        var_S = (S - np.log(S))
+            idx = np.unique(gamma._raveled_index()/gamma.shape[-1])
-        var_gamma = (gamma*np.log(gamma/self.pi)).sum()+((1-gamma)*np.log((1-gamma)/(1-self.pi))).sum()
+            pi = self.pi[idx]
-        return var_gamma+ 0.5 * (gamma* (var_mean + var_S -1)).sum()
+        else:
+            pi = self.pi
+            
+        var_mean = np.square(mu)/self.variance
+        var_S = (S/self.variance - np.log(S))
+        var_gamma = (gamma*np.log(gamma/pi)).sum()+((1-gamma)*np.log((1-gamma)/(1-pi))).sum()
+        return var_gamma+ (gamma* (np.log(self.variance)-1. +var_mean + var_S)).sum()/2.
 
     def update_gradients_KL(self, variational_posterior):
         mu = variational_posterior.mean
         S = variational_posterior.variance
         gamma = variational_posterior.binary_prob
-
+        if len(self.pi.shape)==2:
-        if self.group_spike_prob:
+            idx = np.unique(gamma._raveled_index()/gamma.shape[-1])
-            gamma_grad = np.log((1-self.pi)/self.pi*gamma/(1.-gamma))+(np.square(mu)+S-np.log(S)-1.)/2.
+            pi = self.pi[idx]
-            gamma.gradient -= gamma_grad.mean(axis=0)
         else:
-            gamma.gradient -= np.log((1-self.pi)/self.pi*gamma/(1.-gamma))+(np.square(mu)+S-np.log(S)-1.)/2.
+            pi = self.pi
-        mu.gradient -= gamma*mu
+
-        S.gradient -= (1. - (1. / (S))) * gamma /2.
+        gamma.gradient -= np.log((1-pi)/pi*gamma/(1.-gamma))+((np.square(mu)+S)/self.variance-np.log(S)+np.log(self.variance)-1.)/2.
+        mu.gradient -= gamma*mu/self.variance
+        S.gradient -= (1./self.variance - 1./S) * gamma /2.
+        if self.learnPi:
+            if len(self.pi)==1:
+                self.pi.gradient = (gamma/self.pi - (1.-gamma)/(1.-self.pi)).sum()
+            elif len(self.pi.shape)==1:
                 self.pi.gradient = (gamma/self.pi - (1.-gamma)/(1.-self.pi)).sum(axis=0)
+            else:
+                self.pi[idx].gradient = (gamma/self.pi[idx] - (1.-gamma)/(1.-self.pi[idx]))
 
 class VariationalPosterior(Parameterized):
     def __init__(self, means=None, variances=None, name='latent space', *a, **kw):
@@ -81,7 +94,7 @@ class VariationalPosterior(Parameterized):
     def _raveled_index(self):
         index = np.empty(dtype=int, shape=0)
         size = 0
-        for p in self._parameters_:
+        for p in self.parameters:
             index = np.hstack((index, p._raveled_index()+size))
             size += p._realsize_ if hasattr(p, '_realsize_') else p.size
         return index
@@ -96,17 +109,20 @@ class VariationalPosterior(Parameterized):
             dc = self.__dict__.copy()
             dc['mean'] = self.mean[s]
             dc['variance'] = self.variance[s]
-            dc['_parameters_'] = copy.copy(self._parameters_)
+            dc['parameters'] = copy.copy(self.parameters)
             n.__dict__.update(dc)
-            n._parameters_[dc['mean']._parent_index_] = dc['mean']
+            n.parameters[dc['mean']._parent_index_] = dc['mean']
-            n._parameters_[dc['variance']._parent_index_] = dc['variance']
+            n.parameters[dc['variance']._parent_index_] = dc['variance']
+            n._gradient_array_ = None
+            oversize = self.size - self.mean.size - self.variance.size
+            n.size = n.mean.size + n.variance.size + oversize
             n.ndim = n.mean.ndim
             n.shape = n.mean.shape
             n.num_data = n.mean.shape[0]
             n.input_dim = n.mean.shape[1] if n.ndim != 1 else 1
             return n
         else:
-            return super(VariationalPrior, self).__getitem__(s)
+            return super(VariationalPosterior, self).__getitem__(s)
 
 class NormalPosterior(VariationalPosterior):
     '''
@@ -147,11 +163,14 @@ class SpikeAndSlabPosterior(VariationalPosterior):
             dc['mean'] = self.mean[s]
             dc['variance'] = self.variance[s]
             dc['binary_prob'] = self.binary_prob[s]
-            dc['_parameters_'] = copy.copy(self._parameters_)
+            dc['parameters'] = copy.copy(self.parameters)
             n.__dict__.update(dc)
-            n._parameters_[dc['mean']._parent_index_] = dc['mean']
+            n.parameters[dc['mean']._parent_index_] = dc['mean']
-            n._parameters_[dc['variance']._parent_index_] = dc['variance']
+            n.parameters[dc['variance']._parent_index_] = dc['variance']
-            n._parameters_[dc['binary_prob']._parent_index_] = dc['binary_prob']
+            n.parameters[dc['binary_prob']._parent_index_] = dc['binary_prob']
+            n._gradient_array_ = None
+            oversize = self.size - self.mean.size - self.variance.size
+            n.size = n.mean.size + n.variance.size + oversize
             n.ndim = n.mean.ndim
             n.shape = n.mean.shape
             n.num_data = n.mean.shape[0]
@@ -160,7 +179,7 @@ class SpikeAndSlabPosterior(VariationalPosterior):
         else:
             return super(VariationalPrior, self).__getitem__(s)
 
-    def plot(self, *args):
+    def plot(self, *args, **kwargs):
         """
         Plot latent space X in 1D:
 
@@ -169,4 +188,4 @@ class SpikeAndSlabPosterior(VariationalPosterior):
         import sys
         assert "matplotlib" in sys.modules, "matplotlib package has not been imported."
         from ...plotting.matplot_dep import variational_plots
-        return variational_plots.plot_SpikeSlab(self,*args)
+        return variational_plots.plot_SpikeSlab(self,*args, **kwargs)

GPy/core/sparse_gp.py

@@ -8,6 +8,9 @@ from ..inference.latent_function_inference import var_dtc
 from .. import likelihoods
 from parameterization.variational import VariationalPosterior
 
+import logging
+logger = logging.getLogger("sparse gp")
+
 class SparseGP(GP):
     """
     A general purpose Sparse GP model
@@ -31,7 +34,7 @@ class SparseGP(GP):
 
     """
 
-    def __init__(self, X, Y, Z, kernel, likelihood, inference_method=None, name='sparse gp', Y_metadata=None):
+    def __init__(self, X, Y, Z, kernel, likelihood, inference_method=None, name='sparse gp', Y_metadata=None, normalizer=False):
 
         #pick a sensible inference method
         if inference_method is None:
@@ -45,28 +48,36 @@ class SparseGP(GP):
         self.Z = Param('inducing inputs', Z)
         self.num_inducing = Z.shape[0]
 
-        GP.__init__(self, X, Y, kernel, likelihood, inference_method=inference_method, name=name, Y_metadata=Y_metadata)
+        GP.__init__(self, X, Y, kernel, likelihood, inference_method=inference_method, name=name, Y_metadata=Y_metadata, normalizer=normalizer)
-
+        logger.info("Adding Z as parameter")
         self.add_parameter(self.Z, index=0)
 
     def has_uncertain_inputs(self):
         return isinstance(self.X, VariationalPosterior)
 
     def parameters_changed(self):
-        self.posterior, self._log_marginal_likelihood, self.grad_dict = self.inference_method.inference(self.kern, self.X, self.Z, self.likelihood, self.Y, self.Y_metadata)
+        self.posterior, self._log_marginal_likelihood, self.grad_dict = self.inference_method.inference(self.kern, self.X, self.Z, self.likelihood, self.Y_normalized, self.Y_metadata)
         self.likelihood.update_gradients(self.grad_dict['dL_dthetaL'])
         if isinstance(self.X, VariationalPosterior):
             #gradients wrt kernel
-            dL_dKmm = self.grad_dict.pop('dL_dKmm')
+            dL_dKmm = self.grad_dict['dL_dKmm']
             self.kern.update_gradients_full(dL_dKmm, self.Z, None)
             target = self.kern.gradient.copy()
-            self.kern.update_gradients_expectations(variational_posterior=self.X, Z=self.Z, dL_dpsi0=self.grad_dict['dL_dpsi0'], dL_dpsi1=self.grad_dict['dL_dpsi1'], dL_dpsi2=self.grad_dict['dL_dpsi2'])
+            self.kern.update_gradients_expectations(variational_posterior=self.X,
+                                                    Z=self.Z,
+                                                    dL_dpsi0=self.grad_dict['dL_dpsi0'],
+                                                    dL_dpsi1=self.grad_dict['dL_dpsi1'],
+                                                    dL_dpsi2=self.grad_dict['dL_dpsi2'])
             self.kern.gradient += target
 
             #gradients wrt Z
             self.Z.gradient = self.kern.gradients_X(dL_dKmm, self.Z)
             self.Z.gradient += self.kern.gradients_Z_expectations(
-                               self.grad_dict['dL_dpsi1'], self.grad_dict['dL_dpsi2'], Z=self.Z, variational_posterior=self.X)
+                               self.grad_dict['dL_dpsi0'],
+                               self.grad_dict['dL_dpsi1'],
+                               self.grad_dict['dL_dpsi2'],
+                               Z=self.Z,
+                               variational_posterior=self.X)
         else:
             #gradients wrt kernel
             self.kern.update_gradients_diag(self.grad_dict['dL_dKdiag'], self.X)
@@ -107,5 +118,3 @@ class SparseGP(GP):
                 psi2 = kern.psi2(self.Z, Xnew)
                 var = Kxx - np.sum(np.sum(psi2 * Kmmi_LmiBLmi[None, :, :], 1), 1)
         return mu, var
-
-

GPy/core/sparse_gp_mpi.py

@@ -0,0 +1,115 @@
+# Copyright (c) 2012, GPy authors (see AUTHORS.txt).
+# Licensed under the BSD 3-clause license (see LICENSE.txt)
+
+import numpy as np
+from sparse_gp import SparseGP
+from ..inference.latent_function_inference.var_dtc_parallel import update_gradients, VarDTC_minibatch
+
+import logging
+logger = logging.getLogger("sparse gp mpi")
+
+class SparseGP_MPI(SparseGP):
+    """
+    A general purpose Sparse GP model with MPI parallelization support
+
+    This model allows (approximate) inference using variational DTC or FITC
+    (Gaussian likelihoods) as well as non-conjugate sparse methods based on
+    these.
+
+    :param X: inputs
+    :type X: np.ndarray (num_data x input_dim)
+    :param likelihood: a likelihood instance, containing the observed data
+    :type likelihood: GPy.likelihood.(Gaussian | EP | Laplace)
+    :param kernel: the kernel (covariance function). See link kernels
+    :type kernel: a GPy.kern.kern instance
+    :param X_variance: The uncertainty in the measurements of X (Gaussian variance)
+    :type X_variance: np.ndarray (num_data x input_dim) | None
+    :param Z: inducing inputs
+    :type Z: np.ndarray (num_inducing x input_dim)
+    :param num_inducing: Number of inducing points (optional, default 10. Ignored if Z is not None)
+    :type num_inducing: int
+    :param mpi_comm: The communication group of MPI, e.g. mpi4py.MPI.COMM_WORLD
+    :type mpi_comm: mpi4py.MPI.Intracomm
+
+    """
+
+    def __init__(self, X, Y, Z, kernel, likelihood, variational_prior=None, inference_method=None, name='sparse gp mpi', Y_metadata=None, mpi_comm=None, normalizer=False):
+        self._IN_OPTIMIZATION_ = False
+        if mpi_comm != None:
+            if inference_method is None:
+                inference_method = VarDTC_minibatch(mpi_comm=mpi_comm)
+            else:
+                assert isinstance(inference_method, VarDTC_minibatch), 'inference_method has to support MPI!'
+                        
+        super(SparseGP_MPI, self).__init__(X, Y, Z, kernel, likelihood, inference_method=inference_method, name=name, Y_metadata=Y_metadata, normalizer=normalizer)
+        self.updates = False
+        self.add_parameter(self.X, index=0)
+        if variational_prior is not None:
+            self.add_parameter(variational_prior)
+        self.X.fix()
+
+        self.mpi_comm = mpi_comm
+        # Manage the data (Y) division
+        if mpi_comm != None:
+            from ..util.mpi import divide_data
+            N_start, N_end, N_list = divide_data(Y.shape[0], mpi_comm)
+            self.N_range = (N_start, N_end)
+            self.N_list = np.array(N_list)
+            self.Y_local = self.Y[N_start:N_end]
+            print 'MPI RANK '+str(self.mpi_comm.rank)+' with the data range '+str(self.N_range)
+            mpi_comm.Bcast(self.param_array, root=0)
+        self.updates = True
+
+    def __getstate__(self):
+        dc = super(SparseGP_MPI, self).__getstate__()
+        dc['mpi_comm'] = None
+        if self.mpi_comm != None:
+            del dc['N_range']
+            del dc['N_list']
+            del dc['Y_local']
+        return dc
+
+    #=====================================================
+    # The MPI parallelization 
+    #     - can move to model at some point
+    #=====================================================
+    
+    @SparseGP.optimizer_array.setter
+    def optimizer_array(self, p):
+        if self.mpi_comm != None:
+            if self._IN_OPTIMIZATION_ and self.mpi_comm.rank==0:
+                self.mpi_comm.Bcast(np.int32(1),root=0)
+            self.mpi_comm.Bcast(p, root=0)
+            
+        from ..util.debug import checkFinite
+        checkFinite(p, 'optimizer_array')
+        
+        SparseGP.optimizer_array.fset(self,p)
+        
+    def optimize(self, optimizer=None, start=None, **kwargs):
+        self._IN_OPTIMIZATION_ = True
+        if self.mpi_comm==None:
+            super(SparseGP_MPI, self).optimize(optimizer,start,**kwargs)
+        elif self.mpi_comm.rank==0:
+            super(SparseGP_MPI, self).optimize(optimizer,start,**kwargs)
+            self.mpi_comm.Bcast(np.int32(-1),root=0)
+        elif self.mpi_comm.rank>0:
+            x = self.optimizer_array.copy()
+            flag = np.empty(1,dtype=np.int32)
+            while True:
+                self.mpi_comm.Bcast(flag,root=0)
+                if flag==1:
+                    self.optimizer_array = x
+                elif flag==-1:
+                    break
+                else:
+                    self._IN_OPTIMIZATION_ = False
+                    raise Exception("Unrecognizable flag for synchronization!")
+        self._IN_OPTIMIZATION_ = False
+
+    def parameters_changed(self):
+        if isinstance(self.inference_method,VarDTC_minibatch):
+            update_gradients(self, mpi_comm=self.mpi_comm)
+        else:
+            super(SparseGP_MPI,self).parameters_changed()
+

GPy/defaults.cfg

@@ -1,11 +1,20 @@
-# This is the configuration file for GPy
+# This is the default configuration file for GPy
 
+# Do note edit this file.
+
+# For machine specific changes (i.e. those specific to a given installation) edit GPy/installation.cfg
+
+# For user specific changes edit $HOME/.gpy_user.cfg
 [parallel]
 # Enable openmp support. This speeds up some computations, depending on the number
 # of cores available. Setting up a compiler with openmp support can be difficult on
-# some platforms, hence this option.
+# some platforms, hence by default it is off.
 openmp=False
 
+[datasets]
+# location for the local data cache
+dir=$HOME/tmp/GPy-datasets/
+
 [anaconda]
 # if you have an anaconda python installation please specify it here.
 installed = False

GPy/examples/classification.py

@@ -96,15 +96,11 @@ def toy_linear_1d_classification_laplace(seed=default_seed, optimize=True, plot=
 
     # Optimize
     if optimize:
-        #m.update_likelihood_approximation()
-        # Parameters optimization:
         try:
             m.optimize('scg', messages=1)
         except Exception as e:
             return m
 
-        #m.pseudo_EM()
-
     # Plot
     if plot:
         fig, axes = pb.subplots(2, 1)
@@ -133,10 +129,7 @@ def sparse_toy_linear_1d_classification(num_inducing=10, seed=default_seed, opti
 
     # Optimize
     if optimize:
-        #m.update_likelihood_approximation()
+        m.optimize()
-        # Parameters optimization:
-        #m.optimize()
-        m.pseudo_EM()
 
     # Plot
     if plot:

GPy/examples/dimensionality_reduction.py

@@ -99,7 +99,7 @@ def sparse_gplvm_oil(optimize=True, verbose=0, plot=True, N=100, Q=6, num_induci
         m.kern.plot_ARD()
     return m
 
-def swiss_roll(optimize=True, verbose=1, plot=True, N=1000, num_inducing=15, Q=4, sigma=.2):
+def swiss_roll(optimize=True, verbose=1, plot=True, N=1000, num_inducing=25, Q=4, sigma=.2):
     import GPy
     from GPy.util.datasets import swiss_roll_generated
     from GPy.models import BayesianGPLVM
@@ -144,16 +144,15 @@ def swiss_roll(optimize=True, verbose=1, plot=True, N=1000, num_inducing=15, Q=4
     m = BayesianGPLVM(Y, Q, X=X, X_variance=S, num_inducing=num_inducing, Z=Z, kernel=kernel)
     m.data_colors = c
     m.data_t = t
-    m['noise_variance'] = Y.var() / 100.
 
     if optimize:
-        m.optimize('scg', messages=verbose, max_iters=2e3)
+        m.optimize('bfgs', messages=verbose, max_iters=2e3)
 
     if plot:
         fig = plt.figure('fitted')
         ax = fig.add_subplot(111)
         s = m.input_sensitivity().argsort()[::-1][:2]
-        ax.scatter(*m.X.T[s], c=c)
+        ax.scatter(*m.X.mean.T[s], c=c)
 
     return m
 
@@ -161,6 +160,7 @@ def bgplvm_oil(optimize=True, verbose=1, plot=True, N=200, Q=7, num_inducing=40,
     import GPy
     from matplotlib import pyplot as plt
     from ..util.misc import param_to_array
+    import numpy as np
 
     _np.random.seed(0)
     data = GPy.util.datasets.oil()
@@ -171,15 +171,41 @@ def bgplvm_oil(optimize=True, verbose=1, plot=True, N=200, Q=7, num_inducing=40,
     m.data_labels = data['Y'][:N].argmax(axis=1)
 
     if optimize:
-        m.optimize('scg', messages=verbose, max_iters=max_iters, gtol=.05)
+        m.optimize('bfgs', messages=verbose, max_iters=max_iters, gtol=.05)
 
     if plot:
-        y = m.Y
         fig, (latent_axes, sense_axes) = plt.subplots(1, 2)
         m.plot_latent(ax=latent_axes, labels=m.data_labels)
-        data_show = GPy.plotting.matplot_dep.visualize.vector_show(y)
+        data_show = GPy.plotting.matplot_dep.visualize.vector_show((m.Y[0,:]))
-        lvm_visualizer = GPy.plotting.matplot_dep.visualize.lvm_dimselect(param_to_array(m.X.mean), # @UnusedVariable
+        lvm_visualizer = GPy.plotting.matplot_dep.visualize.lvm_dimselect(param_to_array(m.X.mean)[0:1,:], # @UnusedVariable
-            m, data_show, latent_axes=latent_axes, sense_axes=sense_axes)
+            m, data_show, latent_axes=latent_axes, sense_axes=sense_axes, labels=m.data_labels)
+        raw_input('Press enter to finish')
+        plt.close(fig)
+    return m
+
+def ssgplvm_oil(optimize=True, verbose=1, plot=True, N=200, Q=7, num_inducing=40, max_iters=1000, **k):
+    import GPy
+    from matplotlib import pyplot as plt
+    from ..util.misc import param_to_array
+    import numpy as np
+
+    _np.random.seed(0)
+    data = GPy.util.datasets.oil()
+
+    kernel = GPy.kern.RBF(Q, 1., 1./_np.random.uniform(0,1,(Q,)), ARD=True)# + GPy.kern.Bias(Q, _np.exp(-2))
+    Y = data['X'][:N]
+    m = GPy.models.SSGPLVM(Y, Q, kernel=kernel, num_inducing=num_inducing, **k)
+    m.data_labels = data['Y'][:N].argmax(axis=1)
+
+    if optimize:
+        m.optimize('bfgs', messages=verbose, max_iters=max_iters, gtol=.05)
+
+    if plot:
+        fig, (latent_axes, sense_axes) = plt.subplots(1, 2)
+        m.plot_latent(ax=latent_axes, labels=m.data_labels)
+        data_show = GPy.plotting.matplot_dep.visualize.vector_show((m.Y[0,:]))
+        lvm_visualizer = GPy.plotting.matplot_dep.visualize.lvm_dimselect(param_to_array(m.X.mean)[0:1,:], # @UnusedVariable
+            m, data_show, latent_axes=latent_axes, sense_axes=sense_axes, labels=m.data_labels)
         raw_input('Press enter to finish')
         plt.close(fig)
     return m
@@ -289,6 +315,31 @@ def bgplvm_simulation(optimize=True, verbose=1,
         m.kern.plot_ARD('BGPLVM Simulation ARD Parameters')
     return m
 
+def ssgplvm_simulation(optimize=True, verbose=1,
+                      plot=True, plot_sim=False,
+                      max_iters=2e4, useGPU=False
+                      ):
+    from GPy import kern
+    from GPy.models import SSGPLVM
+
+    D1, D2, D3, N, num_inducing, Q = 13, 5, 8, 45, 3, 9
+    _, _, Ylist = _simulate_sincos(D1, D2, D3, N, num_inducing, Q, plot_sim)
+    Y = Ylist[0]
+    k = kern.Linear(Q, ARD=True, useGPU=useGPU)# + kern.white(Q, _np.exp(-2)) # + kern.bias(Q)
+    #k = kern.RBF(Q, ARD=True, lengthscale=10.)
+    m = SSGPLVM(Y, Q, init="pca", num_inducing=num_inducing, kernel=k)
+    m.X.variance[:] = _np.random.uniform(0,.01,m.X.shape)
+    m.likelihood.variance = .1
+
+    if optimize:
+        print "Optimizing model:"
+        m.optimize('scg', messages=verbose, max_iters=max_iters,
+                   gtol=.05)
+    if plot:
+        m.X.plot("SSGPLVM Latent Space 1D")
+        m.kern.plot_ARD('SSGPLVM Simulation ARD Parameters')
+    return m
+
 def bgplvm_simulation_missing_data(optimize=True, verbose=1,
                       plot=True, plot_sim=False,
                       max_iters=2e4,
@@ -297,15 +348,18 @@ def bgplvm_simulation_missing_data(optimize=True, verbose=1,
     from GPy.models import BayesianGPLVM
     from GPy.inference.latent_function_inference.var_dtc import VarDTCMissingData
 
-    D1, D2, D3, N, num_inducing, Q = 13, 5, 8, 45, 7, 9
+    D1, D2, D3, N, num_inducing, Q = 13, 5, 8, 400, 3, 4
     _, _, Ylist = _simulate_sincos(D1, D2, D3, N, num_inducing, Q, plot_sim)
     Y = Ylist[0]
     k = kern.Linear(Q, ARD=True)# + kern.white(Q, _np.exp(-2)) # + kern.bias(Q)
 
-    inan = _np.random.binomial(1, .6, size=Y.shape).astype(bool)
+    inan = _np.random.binomial(1, .8, size=Y.shape).astype(bool) # 80% missing data
-    m = BayesianGPLVM(Y.copy(), Q, init="random", num_inducing=num_inducing, kernel=k)
+    Ymissing = Y.copy()
-    m.inference_method = VarDTCMissingData()
+    Ymissing[inan] = _np.nan
-    m.Y[inan] = _np.nan
+
+    m = BayesianGPLVM(Ymissing, Q, init="random", num_inducing=num_inducing,
+                      inference_method=VarDTCMissingData(inan=inan), kernel=k)
+
     m.X.variance[:] = _np.random.uniform(0,.01,m.X.shape)
     m.likelihood.variance = .01
     m.parameters_changed()
@@ -338,7 +392,40 @@ def mrd_simulation(optimize=True, verbose=True, plot=True, plot_sim=True, **kw):
         print "Optimizing Model:"
         m.optimize(messages=verbose, max_iters=8e3, gtol=.1)
     if plot:
-        m.plot_X_1d("MRD Latent Space 1D")
+        m.X.plot("MRD Latent Space 1D")
+        m.plot_scales("MRD Scales")
+    return m
+
+def mrd_simulation_missing_data(optimize=True, verbose=True, plot=True, plot_sim=True, **kw):
+    from GPy import kern
+    from GPy.models import MRD
+    from GPy.inference.latent_function_inference.var_dtc import VarDTCMissingData
+
+    D1, D2, D3, N, num_inducing, Q = 60, 20, 36, 60, 6, 5
+    _, _, Ylist = _simulate_sincos(D1, D2, D3, N, num_inducing, Q, plot_sim)
+
+    #Ylist = [Ylist[0]]
+    k = kern.Linear(Q, ARD=True)
+    inanlist = []
+
+    for Y in Ylist:
+        inan = _np.random.binomial(1, .6, size=Y.shape).astype(bool)
+        inanlist.append(inan)
+        Y[inan] = _np.nan
+
+    imlist = []
+    for inan in inanlist:
+        imlist.append(VarDTCMissingData(limit=1, inan=inan))
+
+    m = MRD(Ylist, input_dim=Q, num_inducing=num_inducing,
+            kernel=k, inference_method=imlist,
+            initx="random", initz='permute', **kw)
+
+    if optimize:
+        print "Optimizing Model:"
+        m.optimize('bfgs', messages=verbose, max_iters=8e3, gtol=.1)
+    if plot:
+        m.X.plot("MRD Latent Space 1D")
         m.plot_scales("MRD Scales")
     return m
 
@@ -351,18 +438,17 @@ def brendan_faces(optimize=True, verbose=True, plot=True):
     Yn = Y - Y.mean()
     Yn /= Yn.std()
 
-    m = GPy.models.GPLVM(Yn, Q)
+    m = GPy.models.BayesianGPLVM(Yn, Q, num_inducing=20)
 
     # optimize
-    m.constrain('rbf|noise|white', GPy.transformations.LogexpClipped())
 
-    if optimize: m.optimize('scg', messages=verbose, max_iters=1000)
+    if optimize: m.optimize('bfgs', messages=verbose, max_iters=1000)
 
     if plot:
         ax = m.plot_latent(which_indices=(0, 1))
-        y = m.likelihood.Y[0, :]
+        y = m.Y[0, :]
         data_show = GPy.plotting.matplot_dep.visualize.image_show(y[None, :], dimensions=(20, 28), transpose=True, order='F', invert=False, scale=False)
-        GPy.plotting.matplot_dep.visualize.lvm(m.X[0, :].copy(), m, data_show, ax)
+        lvm = GPy.plotting.matplot_dep.visualize.lvm(m.X.mean[0, :].copy(), m, data_show, ax)
         raw_input('Press enter to finish')
 
     return m
@@ -376,13 +462,14 @@ def olivetti_faces(optimize=True, verbose=True, plot=True):
     Yn = Y - Y.mean()
     Yn /= Yn.std()
 
-    m = GPy.models.GPLVM(Yn, Q)
+    m = GPy.models.BayesianGPLVM(Yn, Q, num_inducing=20)
-    if optimize: m.optimize('scg', messages=verbose, max_iters=1000)
+
+    if optimize: m.optimize('bfgs', messages=verbose, max_iters=1000)
     if plot:
         ax = m.plot_latent(which_indices=(0, 1))
-        y = m.likelihood.Y[0, :]
+        y = m.Y[0, :]
         data_show = GPy.plotting.matplot_dep.visualize.image_show(y[None, :], dimensions=(112, 92), transpose=False, invert=False, scale=False)
-        GPy.plotting.matplot_dep.visualize.lvm(m.X[0, :].copy(), m, data_show, ax)
+        lvm = GPy.plotting.matplot_dep.visualize.lvm(m.X.mean[0, :].copy(), m, data_show, ax)
         raw_input('Press enter to finish')
 
     return m
@@ -408,15 +495,16 @@ def stick(kernel=None, optimize=True, verbose=True, plot=True):
     data = GPy.util.datasets.osu_run1()
     # optimize
     m = GPy.models.GPLVM(data['Y'], 2, kernel=kernel)
-    if optimize: m.optimize(messages=verbose, max_f_eval=10000)
+    if optimize: m.optimize('bfgs', messages=verbose, max_f_eval=10000)
     if plot:
         plt.clf
         ax = m.plot_latent()
         y = m.Y[0, :]
         data_show = GPy.plotting.matplot_dep.visualize.stick_show(y[None, :], connect=data['connect'])
-        vis = GPy.plotting.matplot_dep.visualize.lvm(m.X[0, :].copy(), m, data_show, latent_axes=ax)
+        lvm_visualizer = GPy.plotting.matplot_dep.visualize.lvm(m.X[:1, :].copy(), m, data_show, latent_axes=ax)
         raw_input('Press enter to finish')
-
+        lvm_visualizer.close()
+        data_show.close()
     return m
 
 def bcgplvm_linear_stick(kernel=None, optimize=True, verbose=True, plot=True):
@@ -464,9 +552,8 @@ def robot_wireless(optimize=True, verbose=True, plot=True):
 
     data = GPy.util.datasets.robot_wireless()
     # optimize
-    m = GPy.models.GPLVM(data['Y'], 2)
+    m = GPy.models.BayesianGPLVM(data['Y'], 4, num_inducing=25)
     if optimize: m.optimize(messages=verbose, max_f_eval=10000)
-    m._set_params(m._get_params())
     if plot:
         m.plot_latent()
 
@@ -475,23 +562,32 @@ def robot_wireless(optimize=True, verbose=True, plot=True):
 def stick_bgplvm(model=None, optimize=True, verbose=True, plot=True):
     from GPy.models import BayesianGPLVM
     from matplotlib import pyplot as plt
+    import numpy as np
     import GPy
 
     data = GPy.util.datasets.osu_run1()
     Q = 6
-    kernel = GPy.kern.RBF(Q, ARD=True) + GPy.kern.Bias(Q, _np.exp(-2)) + GPy.kern.White(Q, _np.exp(-2))
+    kernel = GPy.kern.RBF(Q, lengthscale=np.repeat(.5, Q), ARD=True)
     m = BayesianGPLVM(data['Y'], Q, init="PCA", num_inducing=20, kernel=kernel)
+
+    m.data = data
+    m.likelihood.variance = 0.001
+
     # optimize
-    m.ensure_default_constraints()
+    try:
-    if optimize: m.optimize('scg', messages=verbose, max_iters=200, xtol=1e-300, ftol=1e-300)
+        if optimize: m.optimize('bfgs', messages=verbose, max_iters=5e3, bfgs_factor=10)
-    m._set_params(m._get_params())
+    except KeyboardInterrupt:
+        print "Keyboard interrupt, continuing to plot and return"
+
     if plot:
-        plt.clf, (latent_axes, sense_axes) = plt.subplots(1, 2)
+        fig, (latent_axes, sense_axes) = plt.subplots(1, 2)
         plt.sca(latent_axes)
-        m.plot_latent()
+        m.plot_latent(ax=latent_axes)
-        y = m.likelihood.Y[0, :].copy()
+        y = m.Y[:1, :].copy()
-        data_show = GPy.plotting.matplot_dep.visualize.stick_show(y[None, :], connect=data['connect'])
+        data_show = GPy.plotting.matplot_dep.visualize.stick_show(y, connect=data['connect'])
-        GPy.plotting.matplot_dep.visualize.lvm_dimselect(m.X[0, :].copy(), m, data_show, latent_axes=latent_axes, sense_axes=sense_axes)
+        dim_select = GPy.plotting.matplot_dep.visualize.lvm_dimselect(m.X.mean[:1, :].copy(), m, data_show, latent_axes=latent_axes, sense_axes=sense_axes)
+        fig.canvas.draw()
+        fig.canvas.show()
         raw_input('Press enter to finish')
 
     return m
@@ -509,11 +605,12 @@ def cmu_mocap(subject='35', motion=['01'], in_place=True, optimize=True, verbose
     if optimize: m.optimize(messages=verbose, max_f_eval=10000)
     if plot:
         ax = m.plot_latent()
-        y = m.likelihood.Y[0, :]
+        y = m.Y[0, :]
         data_show = GPy.plotting.matplot_dep.visualize.skeleton_show(y[None, :], data['skel'])
-        lvm_visualizer = GPy.plotting.matplot_dep.visualize.lvm(m.X[0, :].copy(), m, data_show, ax)
+        lvm_visualizer = GPy.plotting.matplot_dep.visualize.lvm(m.X[0].copy(), m, data_show, latent_axes=ax)
         raw_input('Press enter to finish')
         lvm_visualizer.close()
+        data_show.close()
 
     return m
 

GPy/examples/regression.py

@@ -387,7 +387,7 @@ def silhouette(max_iters=100, optimize=True, plot=True):
     print m
     return m
 
-def sparse_GP_regression_1D(num_samples=400, num_inducing=5, max_iters=100, optimize=True, plot=True):
+def sparse_GP_regression_1D(num_samples=400, num_inducing=5, max_iters=100, optimize=True, plot=True, checkgrad=True):
     """Run a 1D example of a sparse GP regression."""
     # sample inputs and outputs
     X = np.random.uniform(-3., 3., (num_samples, 1))
@@ -396,6 +396,8 @@ def sparse_GP_regression_1D(num_samples=400, num_inducing=5, max_iters=100, opti
     rbf = GPy.kern.RBF(1)
     # create simple GP Model
     m = GPy.models.SparseGPRegression(X, Y, kernel=rbf, num_inducing=num_inducing)
+
+    if checkgrad:
         m.checkgrad(verbose=1)
 
     if optimize:

GPy/inference/latent_function_inference/__init__.py

@@ -25,6 +25,39 @@ etc.
 
 """
 
+class LatentFunctionInference(object):
+    def on_optimization_start(self):
+        """
+        This function gets called, just before the optimization loop to start.
+        """
+        pass
+
+    def on_optimization_end(self):
+        """
+        This function gets called, just after the optimization loop ended.
+        """
+        pass
+
+class InferenceMethodList(LatentFunctionInference, list):
+
+    def on_optimization_start(self):
+        for inf in self:
+            inf.on_optimization_start()
+
+    def on_optimization_end(self):
+        for inf in self:
+            inf.on_optimization_end()
+    
+    def __getstate__(self):
+        state = []
+        for inf in self:
+            state.append(inf)
+        return state
+    
+    def __setstate__(self, state):
+        for inf in state:
+            self.append(inf)
+
 from exact_gaussian_inference import ExactGaussianInference
 from laplace import Laplace
 from GPy.inference.latent_function_inference.var_dtc import VarDTC
@@ -38,11 +71,26 @@ from var_dtc_gpu import VarDTC_GPU
 # class FullLatentFunctionData(object):
 #
 #
-# class LatentFunctionInference(object):
+
-#     def inference(self, kern, X, likelihood, Y, Y_metadata=None):
+# class EMLikeLatentFunctionInference(LatentFunctionInference):
+#     def update_approximation(self):
+#         """
+#         This function gets called when the 
+#         """
+#     
+#     def inference(self, kern, X, Z, likelihood, Y, Y_metadata=None):
 #         """
 #         Do inference on the latent functions given a covariance function `kern`,
-#         inputs and outputs `X` and `Y`, and a likelihood `likelihood`.
+#         inputs and outputs `X` and `Y`, inducing_inputs `Z`, and a likelihood `likelihood`.
+#         Additional metadata for the outputs `Y` can be given in `Y_metadata`.
+#         """
+#         raise NotImplementedError, "Abstract base class for full inference"
+# 
+# class VariationalLatentFunctionInference(LatentFunctionInference):
+#     def inference(self, kern, X, Z, likelihood, Y, Y_metadata=None):
+#         """
+#         Do inference on the latent functions given a covariance function `kern`,
+#         inputs and outputs `X` and `Y`, inducing_inputs `Z`, and a likelihood `likelihood`.
 #         Additional metadata for the outputs `Y` can be given in `Y_metadata`.
 #         """
 #         raise NotImplementedError, "Abstract base class for full inference"

GPy/inference/latent_function_inference/dtc.py

@@ -4,9 +4,10 @@
 from posterior import Posterior
 from ...util.linalg import jitchol, tdot, dtrtrs, dpotri, pdinv
 import numpy as np
+from . import LatentFunctionInference
 log_2_pi = np.log(2*np.pi)
 
-class DTC(object):
+class DTC(LatentFunctionInference):
     """
     An object for inference when the likelihood is Gaussian, but we want to do sparse inference.
 

GPy/inference/latent_function_inference/exact_gaussian_inference.py

@@ -5,10 +5,11 @@ from posterior import Posterior
 from ...util.linalg import pdinv, dpotrs, tdot
 from ...util import diag
 import numpy as np
+from . import LatentFunctionInference
 log_2_pi = np.log(2*np.pi)
 
 
-class ExactGaussianInference(object):
+class ExactGaussianInference(LatentFunctionInference):
     """
     An object for inference when the likelihood is Gaussian.
 

GPy/inference/latent_function_inference/expectation_propagation.py

@@ -1,9 +1,10 @@
 import numpy as np
 from ...util.linalg import pdinv,jitchol,DSYR,tdot,dtrtrs, dpotrs
 from posterior import Posterior
+from . import LatentFunctionInference
 log_2_pi = np.log(2*np.pi)
 
-class EP(object):
+class EP(LatentFunctionInference):
     def __init__(self, epsilon=1e-6, eta=1., delta=1.):
         """
         The expectation-propagation algorithm.
@@ -21,14 +22,25 @@ class EP(object):
 
     def reset(self):
         self.old_mutilde, self.old_vtilde = None, None
+        self._ep_approximation = None
+
+    def on_optimization_start(self):
+        self._ep_approximation = None
+
+    def on_optimization_end(self):
+        # TODO: update approximation in the end as well? Maybe even with a switch?
+        pass
 
     def inference(self, kern, X, likelihood, Y, Y_metadata=None, Z=None):
-        num_data, output_dim = X.shape
+        num_data, output_dim = Y.shape
         assert output_dim ==1, "ep in 1D only (for now!)"
 
         K = kern.K(X)
 
-        mu, Sigma, mu_tilde, tau_tilde, Z_hat = self.expectation_propagation(K, Y, likelihood, Y_metadata)
+        if self._ep_approximation is None:
+            mu, Sigma, mu_tilde, tau_tilde, Z_hat = self._ep_approximation = self.expectation_propagation(K, Y, likelihood, Y_metadata)
+        else:
+            mu, Sigma, mu_tilde, tau_tilde, Z_hat = self._ep_approximation
 
         Wi, LW, LWi, W_logdet = pdinv(K + np.diag(1./tau_tilde))
 
@@ -42,8 +54,6 @@ class EP(object):
 
         return Posterior(woodbury_inv=Wi, woodbury_vector=alpha, K=K), log_marginal, {'dL_dK':dL_dK, 'dL_dthetaL':dL_dthetaL}
 
-
-
     def expectation_propagation(self, K, Y, likelihood, Y_metadata):
 
         num_data, data_dim = Y.shape
@@ -108,4 +118,3 @@ class EP(object):
 
         mu_tilde = v_tilde/tau_tilde
         return mu, Sigma, mu_tilde, tau_tilde, Z_hat
-

GPy/inference/latent_function_inference/expectation_propagation_dtc.py

@@ -1,39 +1,192 @@
 import numpy as np
-from ...util.linalg import pdinv,jitchol,DSYR,tdot,dtrtrs, dpotrs
+from ...util import diag
-from expectation_propagation import EP
+from ...util.linalg import mdot, jitchol, backsub_both_sides, tdot, dtrtrs, dtrtri, dpotri, dpotrs, symmetrify, DSYR
+from ...util.misc import param_to_array
+from ...core.parameterization.variational import VariationalPosterior
+from . import LatentFunctionInference
 from posterior import Posterior
 log_2_pi = np.log(2*np.pi)
 
-class EPDTC(EP):
+class EPDTC(LatentFunctionInference):
-    #def __init__(self, epsilon=1e-6, eta=1., delta=1.):
+    const_jitter = 1e-6
+    def __init__(self, epsilon=1e-6, eta=1., delta=1., limit=1):
+        from ...util.caching import Cacher
+        self.limit = limit
+        self.get_trYYT = Cacher(self._get_trYYT, limit)
+        self.get_YYTfactor = Cacher(self._get_YYTfactor, limit)
+
+        self.epsilon, self.eta, self.delta = epsilon, eta, delta
+        self.reset()
+
+    def set_limit(self, limit):
+        self.get_trYYT.limit = limit
+        self.get_YYTfactor.limit = limit
+
+    def _get_trYYT(self, Y):
+        return param_to_array(np.sum(np.square(Y)))
+
+    def __getstate__(self):
+        # has to be overridden, as Cacher objects cannot be pickled.
+        return self.limit
+
+    def __setstate__(self, state):
+        # has to be overridden, as Cacher objects cannot be pickled.
+        self.limit = state
+        from ...util.caching import Cacher
+        self.get_trYYT = Cacher(self._get_trYYT, self.limit)
+        self.get_YYTfactor = Cacher(self._get_YYTfactor, self.limit)
+
+    def _get_YYTfactor(self, Y):
+        """
+        find a matrix L which satisfies LLT = YYT.
+
+        Note that L may have fewer columns than Y.
+        """
+        N, D = Y.shape
+        if (N>=D):
+            return param_to_array(Y)
+        else:
+            return jitchol(tdot(Y))
+
+    def get_VVTfactor(self, Y, prec):
+        return Y * prec # TODO chache this, and make it effective
+
+    def reset(self):
+        self.old_mutilde, self.old_vtilde = None, None
+        self._ep_approximation = None
 
     def inference(self, kern, X, Z, likelihood, Y, Y_metadata=None):
-        num_data, output_dim = X.shape
+        num_data, output_dim = Y.shape
         assert output_dim ==1, "ep in 1D only (for now!)"
 
         Kmm = kern.K(Z)
         Kmn = kern.K(Z,X)
 
+        if self._ep_approximation is None:
+            mu, Sigma, mu_tilde, tau_tilde, Z_hat = self._ep_approximation = self.expectation_propagation(Kmm, Kmn, Y, likelihood, Y_metadata)
+        else:
+            mu, Sigma, mu_tilde, tau_tilde, Z_hat = self._ep_approximation
+
+
+        if isinstance(X, VariationalPosterior):
+            uncertain_inputs = True
+            psi0 = kern.psi0(Z, X)
+            psi1 = Kmn.T#kern.psi1(Z, X)
+            psi2 = kern.psi2(Z, X)
+        else:
+            uncertain_inputs = False
+            psi0 = kern.Kdiag(X)
+            psi1 = Kmn.T#kern.K(X, Z)
+            psi2 = None
+
+        #see whether we're using variational uncertain inputs
+
+        _, output_dim = Y.shape
+
+        #see whether we've got a different noise variance for each datum
+        #beta = 1./np.fmax(likelihood.gaussian_variance(Y_metadata), 1e-6)
+        beta = tau_tilde
+        VVT_factor = beta[:,None]*mu_tilde[:,None]
+        trYYT = self.get_trYYT(mu_tilde[:,None])
+
+        # do the inference:
+        het_noise = beta.size > 1
+        num_inducing = Z.shape[0]
+        num_data = Y.shape[0]
+        # kernel computations, using BGPLVM notation
+
+        Kmm = kern.K(Z).copy()
+        diag.add(Kmm, self.const_jitter)
         Lm = jitchol(Kmm)
-        Lmi = dtrtrs(Lm,np.eye(Lm.shape[0]))[0]
+
-        Kmmi = np.dot(Lmi.T,Lmi)
+        # The rather complex computations of A
-        KmmiKmn = np.dot(Kmmi,Kmn)
+        if uncertain_inputs:
-        K = np.dot(Kmn.T,KmmiKmn)
+            if het_noise:
+                psi2_beta = psi2 * (beta.flatten().reshape(num_data, 1, 1)).sum(0)
+            else:
+                psi2_beta = psi2.sum(0) * beta
+            LmInv = dtrtri(Lm)
+            A = LmInv.dot(psi2_beta.dot(LmInv.T))
+        else:
+            if het_noise:
+                tmp = psi1 * (np.sqrt(beta.reshape(num_data, 1)))
+            else:
+                tmp = psi1 * (np.sqrt(beta))
+            tmp, _ = dtrtrs(Lm, tmp.T, lower=1)
+            A = tdot(tmp) #print A.sum()
+
+        # factor B
+        B = np.eye(num_inducing) + A
+        LB = jitchol(B)
+        psi1Vf = np.dot(psi1.T, VVT_factor)
+        # back substutue C into psi1Vf
+        tmp, _ = dtrtrs(Lm, psi1Vf, lower=1, trans=0)
+        _LBi_Lmi_psi1Vf, _ = dtrtrs(LB, tmp, lower=1, trans=0)
+        tmp, _ = dtrtrs(LB, _LBi_Lmi_psi1Vf, lower=1, trans=1)
+        Cpsi1Vf, _ = dtrtrs(Lm, tmp, lower=1, trans=1)
+
+        # data fit and derivative of L w.r.t. Kmm
+        delit = tdot(_LBi_Lmi_psi1Vf)
+        data_fit = np.trace(delit)
+        DBi_plus_BiPBi = backsub_both_sides(LB, output_dim * np.eye(num_inducing) + delit)
+        delit = -0.5 * DBi_plus_BiPBi
+        delit += -0.5 * B * output_dim
+        delit += output_dim * np.eye(num_inducing)
+        # Compute dL_dKmm
+        dL_dKmm = backsub_both_sides(Lm, delit)
+
+        # derivatives of L w.r.t. psi
+        dL_dpsi0, dL_dpsi1, dL_dpsi2 = _compute_dL_dpsi(num_inducing, num_data, output_dim, beta, Lm,
+            VVT_factor, Cpsi1Vf, DBi_plus_BiPBi,
+            psi1, het_noise, uncertain_inputs)
+
+        # log marginal likelihood
+        log_marginal = _compute_log_marginal_likelihood(likelihood, num_data, output_dim, beta, het_noise,
+            psi0, A, LB, trYYT, data_fit, VVT_factor)
+
+        #put the gradients in the right places
+        dL_dR = _compute_dL_dR(likelihood,
+            het_noise, uncertain_inputs, LB,
+            _LBi_Lmi_psi1Vf, DBi_plus_BiPBi, Lm, A,
+            psi0, psi1, beta,
+            data_fit, num_data, output_dim, trYYT, mu_tilde[:,None])
+
+        dL_dthetaL = 0#likelihood.exact_inference_gradients(dL_dR,Y_metadata)
+
+        if uncertain_inputs:
+            grad_dict = {'dL_dKmm': dL_dKmm,
+                         'dL_dpsi0':dL_dpsi0,
+                         'dL_dpsi1':dL_dpsi1,
+                         'dL_dpsi2':dL_dpsi2,
+                         'dL_dthetaL':dL_dthetaL}
+        else:
+            grad_dict = {'dL_dKmm': dL_dKmm,
+                         'dL_dKdiag':dL_dpsi0,
+                         'dL_dKnm':dL_dpsi1,
+                         'dL_dthetaL':dL_dthetaL}
+
+        #get sufficient things for posterior prediction
+        #TODO: do we really want to do this in  the loop?
+        if VVT_factor.shape[1] == Y.shape[1]:
+            woodbury_vector = Cpsi1Vf # == Cpsi1V
+        else:
+            print 'foobar'
+            psi1V = np.dot(mu_tilde[:,None].T*beta, psi1).T
+            tmp, _ = dtrtrs(Lm, psi1V, lower=1, trans=0)
+            tmp, _ = dpotrs(LB, tmp, lower=1)
+            woodbury_vector, _ = dtrtrs(Lm, tmp, lower=1, trans=1)
+        Bi, _ = dpotri(LB, lower=1)
+        symmetrify(Bi)
+        Bi = -dpotri(LB, lower=1)[0]
+        diag.add(Bi, 1)
+
+        woodbury_inv = backsub_both_sides(Lm, Bi)
+
+        #construct a posterior object
+        post = Posterior(woodbury_inv=woodbury_inv, woodbury_vector=woodbury_vector, K=Kmm, mean=None, cov=None, K_chol=Lm)
+        return post, log_marginal, grad_dict
 
 
-        mu, Sigma, mu_tilde, tau_tilde, Z_hat = self.expectation_propagation(Kmm, Kmn, Y, likelihood, Y_metadata)
-
-        Wi, LW, LWi, W_logdet = pdinv(K + np.diag(1./tau_tilde))
-
-        alpha, _ = dpotrs(LW, mu_tilde, lower=1)
-
-        log_marginal =  0.5*(-num_data * log_2_pi - W_logdet - np.sum(alpha * mu_tilde)) # TODO: add log Z_hat??
-
-        dL_dK = 0.5 * (tdot(alpha[:,None]) - Wi)
-
-        dL_dthetaL = np.zeros(likelihood.size)#TODO: derivatives of the likelihood parameters
-
-        return Posterior(woodbury_inv=Wi, woodbury_vector=alpha, K=K), log_marginal, {'dL_dK':dL_dK, 'dL_dthetaL':dL_dthetaL}
 
 
 
@@ -121,3 +274,69 @@ class EPDTC(EP):
 
         mu_tilde = v_tilde/tau_tilde
         return mu, Sigma, mu_tilde, tau_tilde, Z_hat
+
+def _compute_dL_dpsi(num_inducing, num_data, output_dim, beta, Lm, VVT_factor, Cpsi1Vf, DBi_plus_BiPBi, psi1, het_noise, uncertain_inputs):
+    dL_dpsi0 = -0.5 * output_dim * (beta[:,None] * np.ones([num_data, 1])).flatten()
+    dL_dpsi1 = np.dot(VVT_factor, Cpsi1Vf.T)
+    dL_dpsi2_beta = 0.5 * backsub_both_sides(Lm, output_dim * np.eye(num_inducing) - DBi_plus_BiPBi)
+    if het_noise:
+        if uncertain_inputs:
+            dL_dpsi2 = beta[:, None, None] * dL_dpsi2_beta[None, :, :]
+        else:
+            dL_dpsi1 += 2.*np.dot(dL_dpsi2_beta, (psi1 * beta.reshape(num_data, 1)).T).T
+            dL_dpsi2 = None
+    else:
+        dL_dpsi2 = beta * dL_dpsi2_beta
+        if uncertain_inputs:
+            # repeat for each of the N psi_2 matrices
+            dL_dpsi2 = np.repeat(dL_dpsi2[None, :, :], num_data, axis=0)
+        else:
+            # subsume back into psi1 (==Kmn)
+            dL_dpsi1 += 2.*np.dot(psi1, dL_dpsi2)
+            dL_dpsi2 = None
+
+    return dL_dpsi0, dL_dpsi1, dL_dpsi2
+
+
+def _compute_dL_dR(likelihood, het_noise, uncertain_inputs, LB, _LBi_Lmi_psi1Vf, DBi_plus_BiPBi, Lm, A, psi0, psi1, beta, data_fit, num_data, output_dim, trYYT, Y):
+    # the partial derivative vector for the likelihood
+    if likelihood.size == 0:
+        # save computation here.
+        dL_dR = None
+    elif het_noise:
+        if uncertain_inputs:
+            raise NotImplementedError, "heteroscedatic derivates with uncertain inputs not implemented"
+        else:
+            #from ...util.linalg import chol_inv
+            #LBi = chol_inv(LB)
+            LBi, _ = dtrtrs(LB,np.eye(LB.shape[0]))
+
+            Lmi_psi1, nil = dtrtrs(Lm, psi1.T, lower=1, trans=0)
+            _LBi_Lmi_psi1, _ = dtrtrs(LB, Lmi_psi1, lower=1, trans=0)
+
+            dL_dR = -0.5 * beta + 0.5 * (beta*Y)**2
+            dL_dR += 0.5 * output_dim * (psi0 - np.sum(Lmi_psi1**2,0))[:,None] * beta**2
+
+            dL_dR += 0.5*np.sum(mdot(LBi.T,LBi,Lmi_psi1)*Lmi_psi1,0)[:,None]*beta**2
+
+            dL_dR += -np.dot(_LBi_Lmi_psi1Vf.T,_LBi_Lmi_psi1).T * Y * beta**2
+            dL_dR += 0.5*np.dot(_LBi_Lmi_psi1Vf.T,_LBi_Lmi_psi1).T**2 * beta**2
+    else:
+        # likelihood is not heteroscedatic
+        dL_dR = -0.5 * num_data * output_dim * beta + 0.5 * trYYT * beta ** 2
+        dL_dR += 0.5 * output_dim * (psi0.sum() * beta ** 2 - np.trace(A) * beta)
+        dL_dR += beta * (0.5 * np.sum(A * DBi_plus_BiPBi) - data_fit)
+    return dL_dR
+
+def _compute_log_marginal_likelihood(likelihood, num_data, output_dim, beta, het_noise, psi0, A, LB, trYYT, data_fit,Y):
+    #compute log marginal likelihood
+    if het_noise:
+        lik_1 = -0.5 * num_data * output_dim * np.log(2. * np.pi) + 0.5 * np.sum(np.log(beta)) - 0.5 * np.sum(beta * np.square(Y).sum(axis=-1))
+        lik_2 = -0.5 * output_dim * (np.sum(beta.flatten() * psi0) - np.trace(A))
+    else:
+        lik_1 = -0.5 * num_data * output_dim * (np.log(2. * np.pi) - np.log(beta)) - 0.5 * beta * trYYT
+        lik_2 = -0.5 * output_dim * (np.sum(beta * psi0) - np.trace(A))
+    lik_3 = -output_dim * (np.sum(np.log(np.diag(LB))))
+    lik_4 = 0.5 * data_fit
+    log_marginal = lik_1 + lik_2 + lik_3 + lik_4
+    return log_marginal

GPy/inference/latent_function_inference/fitc.py

@@ -5,9 +5,10 @@ from posterior import Posterior
 from ...util.linalg import jitchol, tdot, dtrtrs, dpotri, pdinv
 from ...util import diag
 import numpy as np
+from . import LatentFunctionInference
 log_2_pi = np.log(2*np.pi)
 
-class FITC(object):
+class FITC(LatentFunctionInference):
     """
     An object for inference when the likelihood is Gaussian, but we want to do sparse inference.
 

GPy/inference/latent_function_inference/laplace.py

@@ -16,8 +16,9 @@ from ...util.misc import param_to_array
 from posterior import Posterior
 import warnings
 from scipy import optimize
+from . import LatentFunctionInference
 
-class Laplace(object):
+class Laplace(LatentFunctionInference):
 
     def __init__(self):
         """

GPy/inference/latent_function_inference/posterior.py

@@ -95,7 +95,7 @@ class Posterior(object):
         """
         if self._covariance is None:
             #LiK, _ = dtrtrs(self.woodbury_chol, self._K, lower=1)
-            self._covariance = self._K - (np.tensordot(np.dot(np.atleast_3d(self.woodbury_inv).T, self._K), self._K, [1,0]).T).squeeze()
+            self._covariance = (np.atleast_3d(self._K) - np.tensordot(np.dot(np.atleast_3d(self.woodbury_inv).T, self._K), self._K, [1,0]).T).squeeze()
             #self._covariance = self._K - self._K.dot(self.woodbury_inv).dot(self._K)
         return self._covariance
 
@@ -153,9 +153,14 @@ class Posterior(object):
         $$
         """
         if self._woodbury_inv is None:
-            self._woodbury_inv, _ = dpotri(self.woodbury_chol, lower=1)
+            if self._woodbury_chol is not None:
+                self._woodbury_inv, _ = dpotri(self._woodbury_chol, lower=1)
                 #self._woodbury_inv, _ = dpotrs(self.woodbury_chol, np.eye(self.woodbury_chol.shape[0]), lower=1)
                 symmetrify(self._woodbury_inv)
+            elif self._covariance is not None:
+                B = self._K - self._covariance
+                tmp, _ = dpotrs(self.K_chol, B)
+                self._woodbury_inv, _ = dpotrs(self.K_chol, tmp.T)                
         return self._woodbury_inv
 
     @property

GPy/inference/latent_function_inference/var_dtc.py

@@ -7,9 +7,12 @@ from ...util import diag
 from ...core.parameterization.variational import VariationalPosterior
 import numpy as np
 from ...util.misc import param_to_array
+from . import LatentFunctionInference
 log_2_pi = np.log(2*np.pi)
+import logging, itertools
+logger = logging.getLogger('vardtc')
 
-class VarDTC(object):
+class VarDTC(LatentFunctionInference):
     """
     An object for inference when the likelihood is Gaussian, but we want to do sparse inference.
 
@@ -61,20 +64,9 @@ class VarDTC(object):
         return Y * prec # TODO chache this, and make it effective
 
     def inference(self, kern, X, Z, likelihood, Y, Y_metadata=None):
-        if isinstance(X, VariationalPosterior):
-            uncertain_inputs = True
-            psi0 = kern.psi0(Z, X)
-            psi1 = kern.psi1(Z, X)
-            psi2 = kern.psi2(Z, X)
-        else:
-            uncertain_inputs = False
-            psi0 = kern.Kdiag(X)
-            psi1 = kern.K(X, Z)
-            psi2 = None
-
-        #see whether we're using variational uncertain inputs
 
         _, output_dim = Y.shape
+        uncertain_inputs = isinstance(X, VariationalPosterior)
 
         #see whether we've got a different noise variance for each datum
         beta = 1./np.fmax(likelihood.gaussian_variance(Y_metadata), 1e-6)
@@ -95,23 +87,21 @@ class VarDTC(object):
         diag.add(Kmm, self.const_jitter)
         Lm = jitchol(Kmm)
 
-        # The rather complex computations of A
+
+        # The rather complex computations of A, and the psi stats
         if uncertain_inputs:
+            psi0 = kern.psi0(Z, X)
+            psi1 = kern.psi1(Z, X)
             if het_noise:
-                psi2_beta = psi2 * (beta.flatten().reshape(num_data, 1, 1)).sum(0)
+                psi2_beta = np.sum([kern.psi2(Z,X[i:i+1,:]) * beta_i for i,beta_i in enumerate(beta)],0)
             else:
-                psi2_beta = psi2.sum(0) * beta
+                psi2_beta = kern.psi2(Z,X) * beta
-            #if 0:
-            #    evals, evecs = linalg.eigh(psi2_beta)
-            #    clipped_evals = np.clip(evals, 0., 1e6) # TODO: make clipping configurable
-            #    if not np.array_equal(evals, clipped_evals):
-            #        pass # print evals
-            #    tmp = evecs * np.sqrt(clipped_evals)
-            #    tmp = tmp.T
-            # no backsubstitution because of bound explosion on tr(A) if not...
             LmInv = dtrtri(Lm)
             A = LmInv.dot(psi2_beta.dot(LmInv.T))
         else:
+            psi0 = kern.Kdiag(X)
+            psi1 = kern.K(X, Z)
+            psi2 = None
             if het_noise:
                 tmp = psi1 * (np.sqrt(beta.reshape(num_data, 1)))
             else:
@@ -148,7 +138,7 @@ class VarDTC(object):
         log_marginal = _compute_log_marginal_likelihood(likelihood, num_data, output_dim, beta, het_noise,
             psi0, A, LB, trYYT, data_fit, VVT_factor)
 
-        #put the gradients in the right places
+        #noise derivatives
         dL_dR = _compute_dL_dR(likelihood,
             het_noise, uncertain_inputs, LB,
             _LBi_Lmi_psi1Vf, DBi_plus_BiPBi, Lm, A,
@@ -157,6 +147,7 @@ class VarDTC(object):
 
         dL_dthetaL = likelihood.exact_inference_gradients(dL_dR,Y_metadata)
 
+        #put the gradients in the right places
         if uncertain_inputs:
             grad_dict = {'dL_dKmm': dL_dKmm,
                          'dL_dpsi0':dL_dpsi0,
@@ -190,36 +181,62 @@ class VarDTC(object):
         post = Posterior(woodbury_inv=woodbury_inv, woodbury_vector=woodbury_vector, K=Kmm, mean=None, cov=None, K_chol=Lm)
         return post, log_marginal, grad_dict
 
-class VarDTCMissingData(object):
+class VarDTCMissingData(LatentFunctionInference):
-    const_jitter = 1e-6
+    const_jitter = 1e-10
     def __init__(self, limit=1, inan=None):
         from ...util.caching import Cacher
         self._Y = Cacher(self._subarray_computations, limit)
-        self._inan = inan
+        if inan is not None: self._inan = ~inan
+        else: self._inan = None
         pass
 
     def set_limit(self, limit):
         self._Y.limit = limit
 
+    def __getstate__(self):
+        # has to be overridden, as Cacher objects cannot be pickled.
+        return self._Y.limit, self._inan
+
+    def __setstate__(self, state):
+        # has to be overridden, as Cacher objects cannot be pickled.
+        from ...util.caching import Cacher
+        self.limit = state[0]
+        self._inan = state[1]
+        self._Y = Cacher(self._subarray_computations, self.limit)
+
     def _subarray_computations(self, Y):
         if self._inan is None:
             inan = np.isnan(Y)
             has_none = inan.any()
+            self._inan = ~inan
         else:
             inan = self._inan
             has_none = True
         if has_none:
-            from ...util.subarray_and_sorting import common_subarrays
+            #print "caching missing data slices, this can take several minutes depending on the number of unique dimensions of the data..."
-            self._subarray_indices = []
+            #csa = common_subarrays(inan, 1)
-            for v,ind in common_subarrays(inan, 1).iteritems():
+            size = Y.shape[1]
-                if not np.all(v):
+            #logger.info('preparing subarrays {:3.3%}'.format((i+1.)/size))
-                    v = ~np.array(v, dtype=bool)
+            Ys = []
-                    ind = np.array(ind, dtype=int)
+            next_ten = [0.]
-                    if ind.size == Y.shape[1]:
+            count = itertools.count()
-                        ind = slice(None)
+            for v, y in itertools.izip(inan.T, Y.T[:,:,None]):
-                    self._subarray_indices.append([v,ind])
+                i = count.next()
-            Ys = [Y[v, :][:, ind] for v, ind in self._subarray_indices]
+                if ((i+1.)/size) >= next_ten[0]:
-            traces = [(y**2).sum() for y in Ys]
+                    logger.info('preparing subarrays {:>6.1%}'.format((i+1.)/size))
+                    next_ten[0] += .1
+                Ys.append(y[v,:])
+
+            next_ten = [0.]
+            count = itertools.count()
+            def trace(y):
+                i = count.next()
+                if ((i+1.)/size) >= next_ten[0]:
+                    logger.info('preparing traces {:>6.1%}'.format((i+1.)/size))
+                    next_ten[0] += .1
+                y = y[inan[:,i],i:i+1]
+                return np.einsum('ij,ij->', y,y)
+            traces = [trace(Y) for _ in xrange(size)]
             return Ys, traces
         else:
             self._subarray_indices = [[slice(None),slice(None)]]
@@ -241,7 +258,6 @@ class VarDTCMissingData(object):
         beta_all = 1./np.fmax(likelihood.gaussian_variance(Y_metadata), 1e-6)
         het_noise = beta_all.size != 1
 
-        import itertools
         num_inducing = Z.shape[0]
 
         dL_dpsi0_all = np.zeros(Y.shape[0])
@@ -261,18 +277,17 @@ class VarDTCMissingData(object):
         Lm = jitchol(Kmm)
         if uncertain_inputs: LmInv = dtrtri(Lm)
 
-        VVT_factor_all = np.empty(Y.shape)
+        size = Y.shape[1]
-        full_VVT_factor = VVT_factor_all.shape[1] == Y.shape[1]
+        next_ten = 0
-        if not full_VVT_factor:
+        for i, [y, v, trYYT] in enumerate(itertools.izip(Ys, self._inan.T, traces)):
-            psi1V = np.dot(Y.T*beta_all, psi1_all).T
+            if ((i+1.)/size) >= next_ten:
-
+                logger.info('inference {:> 6.1%}'.format((i+1.)/size))
-        for y, trYYT, [v, ind] in itertools.izip(Ys, traces, self._subarray_indices):
+                next_ten += .1
-            if het_noise: beta = beta_all[ind]
+            if het_noise: beta = beta_all[i]
             else: beta = beta_all
 
-            VVT_factor = (beta*y)
+            VVT_factor = (y*beta)
-            VVT_factor_all[v, ind].flat = VVT_factor.flat
+            output_dim = 1#len(ind)
-            output_dim = y.shape[1]
 
             psi0 = psi0_all[v]
             psi1 = psi1_all[v, :]
@@ -314,7 +329,6 @@ class VarDTCMissingData(object):
                 VVT_factor, Cpsi1Vf, DBi_plus_BiPBi,
                 psi1, het_noise, uncertain_inputs)
 
-            #import ipdb;ipdb.set_trace()
             dL_dpsi0_all[v] += dL_dpsi0
             dL_dpsi1_all[v, :] += dL_dpsi1
             if uncertain_inputs:
@@ -331,19 +345,20 @@ class VarDTCMissingData(object):
                 psi0, psi1, beta,
                 data_fit, num_data, output_dim, trYYT, Y)
 
-            if full_VVT_factor: woodbury_vector[:, ind] = Cpsi1Vf
+            #if full_VVT_factor:
-            else:
+            woodbury_vector[:, i:i+1] = Cpsi1Vf
-                print 'foobar'
+            #else:
-                tmp, _ = dtrtrs(Lm, psi1V, lower=1, trans=0)
+            #    print 'foobar'
-                tmp, _ = dpotrs(LB, tmp, lower=1)
+            #    tmp, _ = dtrtrs(Lm, psi1V, lower=1, trans=0)
-                woodbury_vector[:, ind] = dtrtrs(Lm, tmp, lower=1, trans=1)[0]
+            #    tmp, _ = dpotrs(LB, tmp, lower=1)
+            #    woodbury_vector[:, ind] = dtrtrs(Lm, tmp, lower=1, trans=1)[0]
 
             #import ipdb;ipdb.set_trace()
             Bi, _ = dpotri(LB, lower=1)
             symmetrify(Bi)
             Bi = -dpotri(LB, lower=1)[0]
             diag.add(Bi, 1)
-            woodbury_inv_all[:, :, ind] = backsub_both_sides(Lm, Bi)[:,:,None]
+            woodbury_inv_all[:, :, i:i+1] = backsub_both_sides(Lm, Bi)[:,:,None]
 
         dL_dthetaL = likelihood.exact_inference_gradients(dL_dR)
 
@@ -360,23 +375,6 @@ class VarDTCMissingData(object):
                          'dL_dKnm':dL_dpsi1_all,
                          'dL_dthetaL':dL_dthetaL}
 
-        #get sufficient things for posterior prediction
-        #TODO: do we really want to do this in  the loop?
-        #if not full_VVT_factor:
-        #    print 'foobar'
-        #    psi1V = np.dot(Y.T*beta_all, psi1_all).T
-        #    tmp, _ = dtrtrs(Lm, psi1V, lower=1, trans=0)
-        #    tmp, _ = dpotrs(LB_all, tmp, lower=1)
-        #    woodbury_vector, _ = dtrtrs(Lm, tmp, lower=1, trans=1)
-        #import ipdb;ipdb.set_trace()
-        #Bi, _ = dpotri(LB_all, lower=1)
-        #symmetrify(Bi)
-        #Bi = -dpotri(LB_all, lower=1)[0]
-        #from ...util import diag
-        #diag.add(Bi, 1)
-
-        #woodbury_inv = backsub_both_sides(Lm, Bi)
-
         post = Posterior(woodbury_inv=woodbury_inv_all, woodbury_vector=woodbury_vector, K=Kmm, mean=None, cov=None, K_chol=Lm)
 
         return post, log_marginal, grad_dict
@@ -393,10 +391,7 @@ def _compute_dL_dpsi(num_inducing, num_data, output_dim, beta, Lm, VVT_factor, C
             dL_dpsi2 = None
     else:
         dL_dpsi2 = beta * dL_dpsi2_beta
-        if uncertain_inputs:
+        if not uncertain_inputs:
-            # repeat for each of the N psi_2 matrices
-            dL_dpsi2 = np.repeat(dL_dpsi2[None, :, :], num_data, axis=0)
-        else:
             # subsume back into psi1 (==Kmn)
             dL_dpsi1 += 2.*np.dot(psi1, dL_dpsi2)
             dL_dpsi2 = None

GPy/inference/latent_function_inference/var_dtc_gpu.py

@@ -7,6 +7,7 @@ from ...util import diag
 from ...core.parameterization.variational import VariationalPosterior
 import numpy as np
 from ...util.misc import param_to_array
+from . import LatentFunctionInference
 log_2_pi = np.log(2*np.pi)
 
 from ...util import gpu_init
@@ -15,11 +16,11 @@ try:
     import scikits.cuda.linalg as culinalg
     import pycuda.gpuarray as gpuarray
     from scikits.cuda import cublas
-    from ...util.linalg_gpu import logDiagSum, strideSum, mul_bcast, sum_axis, outer_prod, mul_bcast_first, join_prod
+    from ...util.linalg_gpu import logDiagSum, strideSum, mul_bcast, sum_axis, outer_prod, mul_bcast_first, join_prod, traceDot
 except:
     pass
 
-class VarDTC_GPU(object):
+class VarDTC_GPU(LatentFunctionInference):
     """
     An object for inference when the likelihood is Gaussian, but we want to do sparse inference.
 
@@ -65,18 +66,13 @@ class VarDTC_GPU(object):
                              'beta_gpu'             :gpuarray.empty((ndata,),np.float64,order='F'),
                              'YT_gpu'               :gpuarray.to_gpu(np.asfortranarray(Y.T)), # DxN
                              'betaYT_gpu'           :gpuarray.empty(Y.T.shape,np.float64,order='F'), # DxN
-                             'psi2_t_gpu'           :gpuarray.empty((num_inducing*num_inducing*self.batchsize),np.float64,order='F'),
                              # inference_minibatch
                              'dL_dpsi0_gpu'         :gpuarray.empty((self.batchsize,),np.float64,order='F'),
                              'dL_dpsi1_gpu'         :gpuarray.empty((self.batchsize,num_inducing),np.float64,order='F'),
-                             'dL_dpsi2_gpu'         :gpuarray.empty((self.batchsize,num_inducing,num_inducing),np.float64,order='F'),
+                             'dL_dpsi2_gpu'         :gpuarray.empty((num_inducing,num_inducing),np.float64,order='F'),
-                             'dL_dthetaL_gpu'       :gpuarray.empty((self.batchsize,),np.float64,order='F'),
-                             'betapsi1_gpu'         :gpuarray.empty((self.batchsize,num_inducing),np.float64,order='F'),
-                             'thetaL_t_gpu'         :gpuarray.empty((self.batchsize,),np.float64,order='F'),
-                             'betaYT2_gpu'          :gpuarray.empty((output_dim,self.batchsize),np.float64,order='F'),
                              'psi0p_gpu'            :gpuarray.empty((self.batchsize,),np.float64,order='F'),
                              'psi1p_gpu'            :gpuarray.empty((self.batchsize,num_inducing),np.float64,order='F'),
-                             'psi2p_gpu'            :gpuarray.empty((self.batchsize,num_inducing,num_inducing),np.float64,order='F'),
+                             'psi2p_gpu'            :gpuarray.empty((num_inducing,num_inducing),np.float64,order='F'),
                              }
             self.gpuCache['ones_gpu'].fill(1.0)
             
@@ -125,41 +121,16 @@ class VarDTC_GPU(object):
         else:
             return jitchol(tdot(Y))
         
-    def inference_likelihood(self, kern, X, Z, likelihood, Y):
+    def gatherPsiStat(self, kern, X, Z, Y, beta, uncertain_inputs, het_noise):
-        """
-        The first phase of inference:
-        Compute: log-likelihood, dL_dKmm
-        
-        Cached intermediate results: Kmm, KmmInv,
-        """
-        
         num_inducing, input_dim = Z.shape[0], Z.shape[1]
         num_data, output_dim = Y.shape
-        
-        self._initGPUCache(kern, num_inducing, input_dim, output_dim, Y)
-
-        if isinstance(X, VariationalPosterior):
-            uncertain_inputs = True
-        else:
-            uncertain_inputs = False
-        
-        #see whether we've got a different noise variance for each datum
-        beta = 1./np.fmax(likelihood.variance, 1e-6)
-        het_noise = beta.size > 1
         trYYT = self._trYYT
-        
         psi1Y_gpu = self.gpuCache['psi1Y_gpu']
         psi2_gpu = self.gpuCache['psi2_gpu']
         beta_gpu = self.gpuCache['beta_gpu']
         YT_gpu = self.gpuCache['YT_gpu']
         betaYT_gpu = self.gpuCache['betaYT_gpu']
-        psi2_t_gpu = self.gpuCache['psi2_t_gpu']
         
-        if het_noise:
-            beta_gpu.set(np.asfortranarray(beta))
-            mul_bcast(betaYT_gpu,beta_gpu,YT_gpu,beta_gpu.size)
-            YRY_full = cublas.cublasDdot(self.cublas_handle, YT_gpu.size, betaYT_gpu.gpudata, 1, YT_gpu.gpudata, 1)
-        else:
         beta_gpu.fill(beta)
         betaYT_gpu.fill(0.)
         cublas.cublasDaxpy(self.cublas_handle, betaYT_gpu.size, beta, YT_gpu.gpudata, 1, betaYT_gpu.gpudata, 1)
@@ -174,93 +145,94 @@ class VarDTC_GPU(object):
                 n_end = min(self.batchsize+n_start, num_data)
                 ndata = n_end - n_start
                 X_slice = X[n_start:n_end]
-                beta_gpu_slice = beta_gpu[n_start:n_end]
                 betaYT_gpu_slice = betaYT_gpu[:,n_start:n_end]
-                if ndata==self.batchsize:
+
-                    psi2_t_gpu_slice = psi2_t_gpu
-                else:
-                    psi2_t_gpu_slice = psi2_t_gpu[:num_inducing*num_inducing*ndata]
                 if uncertain_inputs:
-                    psi0p_gpu = kern.psi0(Z, X_slice)
+                    psi0 = kern.psi0(Z, X_slice)
                     psi1p_gpu = kern.psi1(Z, X_slice)
                     psi2p_gpu = kern.psi2(Z, X_slice)
                 else:
-                    psi0p_gpu = kern.Kdiag(X_slice)
+                    psi0 = kern.Kdiag(X_slice)
                     psi1p_gpu = kern.K(X_slice, Z)
 
                 cublas.cublasDgemm(self.cublas_handle, 'T', 'T', num_inducing, output_dim, ndata, 1.0, psi1p_gpu.gpudata, ndata, betaYT_gpu_slice.gpudata, output_dim, 1.0, psi1Y_gpu.gpudata, num_inducing)
                 
-                if het_noise:
+                psi0_full += psi0.sum()
-                    psi0_full += cublas.cublasDdot(self.cublas_handle, psi0p_gpu.size, beta_gpu_slice.gpudata, 1, psi0p_gpu.gpudata, 1)
-                else:
-                    psi0_full += gpuarray.sum(psi0p_gpu).get()
                                     
                 if uncertain_inputs:
-                    if het_noise:
+                    sum_axis(psi2_gpu,psi2p_gpu,1,1)
-                        mul_bcast(psi2_t_gpu_slice,beta_gpu_slice,psi2p_gpu,beta_gpu_slice.size)
-                        sum_axis(psi2_gpu,psi2_t_gpu_slice,1,ndata)
-                    else:
-                        sum_axis(psi2_gpu,psi2p_gpu,1,ndata)
-                else:
-                    if het_noise:
-                        psi1_t_gpu = psi2_t_gpu_slice[:,num_inducing*ndata]
-                        mul_bcast(psi1_t_gpu,beta_gpu_slice,psi1p_gpu,beta_gpu_slice.size)
-                        cublas.cublasDgemm(self.cublas_handle, 'T', 'N', num_inducing, num_inducing, ndata, 1.0, psi1p_gpu.gpudata, ndata, psi1_t_gpu.gpudata, ndata, 1.0, psi2_gpu.gpudata, num_inducing)
                 else:
                     cublas.cublasDgemm(self.cublas_handle, 'T', 'N', num_inducing, num_inducing, ndata, beta, psi1p_gpu.gpudata, ndata, psi1p_gpu.gpudata, ndata, 1.0, psi2_gpu.gpudata, num_inducing)
                     
-            if not het_noise:
             psi0_full *= beta
             if uncertain_inputs:
                 cublas.cublasDscal(self.cublas_handle, psi2_gpu.size, beta, psi2_gpu.gpudata, 1)
             
         else:    
-            psi2_full = np.zeros((num_inducing,num_inducing),order='F')
+            psi2_full = np.zeros((num_inducing,num_inducing))
-            psi1Y_full = np.zeros((num_inducing,output_dim),order='F') # MxD
+            psi1Y_full = np.zeros((output_dim,num_inducing)) # DxM
-            psi0_full = 0
+            psi0_full = 0.
-#             YRY_full = 0
+            YRY_full = 0.
             
             for n_start in xrange(0,num_data,self.batchsize):            
                 n_end = min(self.batchsize+n_start, num_data)
                 Y_slice = Y[n_start:n_end]
                 X_slice = X[n_start:n_end]
+                
+                if het_noise:
+                    b = beta[n_start]
+                    YRY_full += np.inner(Y_slice, Y_slice)*b
+                else:
+                    b = beta
+                
                 if uncertain_inputs:
                     psi0 = kern.psi0(Z, X_slice)
                     psi1 = kern.psi1(Z, X_slice)
-                    psi2 = kern.psi2(Z, X_slice)
+                    psi2_full += kern.psi2(Z, X_slice)*b
                 else:
                     psi0 = kern.Kdiag(X_slice)
                     psi1 = kern.K(X_slice, Z)
+                    psi2_full += np.dot(psi1.T,psi1)*b
                     
-                if het_noise:
+                psi0_full += psi0.sum()*b
-                    beta_slice = beta[n_start:n_end]
+                psi1Y_full += np.dot(Y_slice.T,psi1)*b # DxM                
-                    psi0_full += (beta_slice*psi0).sum()
-                    psi1Y_full += np.dot(psi1.T,beta_slice[:,None]*Y_slice) # MxD
-#                     YRY_full += (beta_slice*np.square(Y_slice).sum(axis=-1)).sum()
-                else:
-                    psi0_full += psi0.sum()
-                    psi1Y_full += np.dot(psi1.T,Y_slice) # MxD
-                                    
-                if uncertain_inputs:
-                    if het_noise:
-                        psi2_full += np.einsum('n,nmo->mo',beta_slice,psi2)
-                    else:
-                        psi2_full += psi2.sum(axis=0)
-                else:
-                    if het_noise:
-                        psi2_full += np.einsum('n,nm,no->mo',beta_slice,psi1,psi1)
-                    else:
-                        psi2_full += tdot(psi1.T)
     
             if not het_noise:
-                psi0_full *= beta
+                YRY_full = trYYT*beta
-                psi1Y_full *= beta
-                psi2_full *= beta
-#                 YRY_full = trYYT*beta
-            
             psi1Y_gpu.set(psi1Y_full)
             psi2_gpu.set(psi2_full)
     
+        return psi0_full, YRY_full
+        
+    def inference_likelihood(self, kern, X, Z, likelihood, Y):
+        """
+        The first phase of inference:
+        Compute: log-likelihood, dL_dKmm
+        
+        Cached intermediate results: Kmm, KmmInv,
+        """
+        
+        num_inducing, input_dim = Z.shape[0], Z.shape[1]
+        num_data, output_dim = Y.shape
+        
+        #see whether we've got a different noise variance for each datum
+        beta = 1./np.fmax(likelihood.variance, 1e-6)
+        het_noise = beta.size > 1
+        if het_noise:
+            self.batchsize=0
+        
+        self._initGPUCache(kern, num_inducing, input_dim, output_dim, Y)
+
+        if isinstance(X, VariationalPosterior):
+            uncertain_inputs = True
+        else:
+            uncertain_inputs = False
+        
+        psi1Y_gpu = self.gpuCache['psi1Y_gpu']
+        psi2_gpu = self.gpuCache['psi2_gpu']
+        
+        psi0_full, YRY_full = self.gatherPsiStat(kern, X, Z, Y, beta, uncertain_inputs, het_noise)
+        
         #======================================================================
         # Compute Common Components
         #======================================================================
@@ -372,6 +344,16 @@ class VarDTC_GPU(object):
                 
         post = Posterior(woodbury_inv=KmmInvPsi2P_gpu.get(), woodbury_vector=v_gpu.get(), K=Kmm_gpu.get(), mean=None, cov=None, K_chol=Lm_gpu.get())
 
+        #======================================================================
+        # Compute dL_dthetaL for uncertian input and non-heter noise
+        #======================================================================        
+        
+        if not het_noise:
+            dL_dthetaL = (YRY_full + output_dim*psi0_full - num_data*output_dim)/-2.
+            dL_dthetaL += cublas.cublasDdot(self.cublas_handle,dL_dpsi2R_gpu.size, dL_dpsi2R_gpu.gpudata,1,psi2_gpu.gpudata,1)
+            dL_dthetaL += cublas.cublasDdot(self.cublas_handle,v_gpu.size, v_gpu.gpudata,1,psi1Y_gpu.gpudata,1)
+            self.midRes['dL_dthetaL'] = -beta*dL_dthetaL
+            
         return logL, dL_dKmm_gpu.get(), post
 
     def inference_minibatch(self, kern, X, Z, likelihood, Y):
@@ -403,26 +385,26 @@ class VarDTC_GPU(object):
         
         nSlice = n_end-n_start
         X_slice = X[n_start:n_end]
+        if het_noise:
+            beta = beta[n_start] # nSlice==1
         
         if kern.useGPU:
-            if uncertain_inputs:
+            if not uncertain_inputs:
-                psi0p_gpu = kern.psi0(Z, X_slice)
-                psi1p_gpu = kern.psi1(Z, X_slice)
-                psi2p_gpu = kern.psi2(Z, X_slice)
-            else:
                 psi0p_gpu = kern.Kdiag(X_slice)
                 psi1p_gpu = kern.K(X_slice, Z)
                 psi2p_gpu = self.gpuCache['psi2p_gpu']
-                if psi2p_gpu.shape[0] > nSlice:
+            elif het_noise:
-                    psi2p_gpu = psi2p_gpu.ravel()[:nSlice*num_inducing*num_inducing].reshape(nSlice,num_inducing,num_inducing)
+                psi0p_gpu = kern.psi0(Z, X_slice)
-        else:
+                psi1p_gpu = kern.psi1(Z, X_slice)
-            if uncertain_inputs:
+                psi2p_gpu = kern.psi2(Z, X_slice)
+        elif not uncertain_inputs or het_noise:
+            if not uncertain_inputs:
+                psi0 = kern.Kdiag(X_slice)
+                psi1 = kern.K(X_slice, Z)
+            elif het_noise:
                 psi0 = kern.psi0(Z, X_slice)
                 psi1 = kern.psi1(Z, X_slice)
                 psi2 = kern.psi2(Z, X_slice)
-            else:
-                psi0 = kern.Kdiag(X_slice)
-                psi1 = kern.K(X_slice, Z)
 
             psi0p_gpu = self.gpuCache['psi0p_gpu']
             psi1p_gpu = self.gpuCache['psi1p_gpu']
@@ -430,91 +412,46 @@ class VarDTC_GPU(object):
             if psi0p_gpu.shape[0] > nSlice:
                 psi0p_gpu = psi0p_gpu[:nSlice]
                 psi1p_gpu = psi1p_gpu.ravel()[:nSlice*num_inducing].reshape(nSlice,num_inducing)
-                psi2p_gpu = psi2p_gpu.ravel()[:nSlice*num_inducing*num_inducing].reshape(nSlice,num_inducing,num_inducing)
             psi0p_gpu.set(np.asfortranarray(psi0))
             psi1p_gpu.set(np.asfortranarray(psi1))
             if uncertain_inputs:
                 psi2p_gpu.set(np.asfortranarray(psi2))
                             
         #======================================================================
-        # Prepare gpu memory
+        # Compute dL_dpsi
         #======================================================================
 
         dL_dpsi2R_gpu = self.gpuCache['dL_dpsi2R_gpu']
         v_gpu = self.gpuCache['v_gpu']        
-        betaYT_gpu = self.gpuCache['betaYT_gpu']
-        beta_gpu = self.gpuCache['beta_gpu']
         dL_dpsi0_gpu = self.gpuCache['dL_dpsi0_gpu']
         dL_dpsi1_gpu = self.gpuCache['dL_dpsi1_gpu']
         dL_dpsi2_gpu = self.gpuCache['dL_dpsi2_gpu']
-        dL_dthetaL_gpu = self.gpuCache['dL_dthetaL_gpu']
+        betaYT_gpu = self.gpuCache['betaYT_gpu']
-        psi2R_gpu = self.gpuCache['psi2_t_gpu'][:nSlice*num_inducing*num_inducing].reshape(nSlice,num_inducing,num_inducing)
-        betapsi1_gpu = self.gpuCache['betapsi1_gpu']
-        thetaL_t_gpu = self.gpuCache['thetaL_t_gpu']
-        betaYT2_gpu = self.gpuCache['betaYT2_gpu']
-        
         betaYT_gpu_slice = betaYT_gpu[:,n_start:n_end]
-        beta_gpu_slice = beta_gpu[n_start:n_end]
         
         # Adjust to the batch size
         if dL_dpsi0_gpu.shape[0] > nSlice:
-            betaYT2_gpu = betaYT2_gpu[:,:nSlice]
             dL_dpsi0_gpu = dL_dpsi0_gpu.ravel()[:nSlice]
             dL_dpsi1_gpu = dL_dpsi1_gpu.ravel()[:nSlice*num_inducing].reshape(nSlice,num_inducing)
-            dL_dpsi2_gpu = dL_dpsi2_gpu.ravel()[:nSlice*num_inducing*num_inducing].reshape(nSlice,num_inducing,num_inducing)
-            dL_dthetaL_gpu = dL_dthetaL_gpu.ravel()[:nSlice]
-            psi2R_gpu = psi2R_gpu.ravel()[:nSlice*num_inducing*num_inducing].reshape(nSlice,num_inducing,num_inducing)
-            thetaL_t_gpu = thetaL_t_gpu.ravel()[:nSlice]
-            betapsi1_gpu = betapsi1_gpu.ravel()[:nSlice*num_inducing].reshape(nSlice,num_inducing)
         
-        mul_bcast(betapsi1_gpu,beta_gpu_slice,psi1p_gpu,beta_gpu_slice.size)
+        dL_dpsi0_gpu.fill(-output_dim *beta/2.)
-
-        #======================================================================
-        # Compute dL_dpsi
-        #======================================================================
-        
-        dL_dpsi0_gpu.fill(0.)
-        cublas.cublasDaxpy(self.cublas_handle, dL_dpsi0_gpu.size, output_dim/(-2.), beta_gpu_slice.gpudata, 1, dL_dpsi0_gpu.gpudata, 1)
         
         cublas.cublasDgemm(self.cublas_handle, 'T', 'T', nSlice, num_inducing, output_dim, 1.0, betaYT_gpu_slice.gpudata, output_dim, v_gpu.gpudata, num_inducing, 0., dL_dpsi1_gpu.gpudata, nSlice)
         
         if uncertain_inputs:
-            outer_prod(dL_dpsi2_gpu,beta_gpu_slice,dL_dpsi2R_gpu,beta_gpu_slice.size)
+            cublas.cublasDcopy(self.cublas_handle, dL_dpsi2R_gpu.size, dL_dpsi2R_gpu.gpudata, 1, dL_dpsi2_gpu.gpudata, 1)
+            cublas.cublasDscal(self.cublas_handle, dL_dpsi2_gpu.size, beta, dL_dpsi2_gpu.gpudata, 1)
         else:
-            cublas.cublasDgemm(self.cublas_handle, 'N', 'N', nSlice, num_inducing, output_dim, 1.0, betapsi1_gpu.gpudata, nSlice, dL_dpsi2R_gpu.gpudata, num_inducing, 1.0, dL_dpsi1_gpu.gpudata, nSlice)
+            cublas.cublasDgemm(self.cublas_handle, 'N', 'N', nSlice, num_inducing, output_dim, beta, psi1p_gpu.gpudata, nSlice, dL_dpsi2R_gpu.gpudata, num_inducing, 1.0, dL_dpsi1_gpu.gpudata, nSlice)
 
         #======================================================================
         # Compute dL_dthetaL
         #======================================================================
-        
+        if het_noise:
-        if not uncertain_inputs:
+            betaY = betaYT_gpu_slice.get()
-            join_prod(psi2p_gpu,psi1p_gpu,psi1p_gpu,nSlice,num_inducing)
+            dL_dthetaL = ((np.square(betaY)).sum(axis=-1) + np.square(beta)*(output_dim*psi0p_gpu.get())-output_dim*beta)/2.
-
+            dL_dthetaL += -beta*beta*cublas.cublasDdot(self.cublas_handle,dL_dpsi2R_gpu.size, dL_dpsi2R_gpu.gpudata,1,psi2p_gpu.gpudata,1)
-        mul_bcast_first(psi2R_gpu,dL_dpsi2R_gpu,psi2p_gpu,nSlice)
+            dL_dthetaL += -beta*(betaY*np.dot(psi1p_gpu.get(),v_gpu.get())).sum(axis=-1)
-        
-
-        dL_dthetaL_gpu.fill(0.)
-        
-        cublas.cublasDcopy(self.cublas_handle, betaYT_gpu_slice.size, betaYT_gpu_slice.gpudata, 1, betaYT2_gpu.gpudata, 1)
-        mul_bcast(betaYT2_gpu,betaYT2_gpu,betaYT2_gpu,betaYT2_gpu.size)
-        cublas.cublasDscal(self.cublas_handle, betaYT2_gpu.size, 0.5, betaYT2_gpu.gpudata, 1)
-        sum_axis(dL_dthetaL_gpu, betaYT2_gpu, 1, output_dim)
-        
-        cublas.cublasDaxpy(self.cublas_handle, dL_dthetaL_gpu.size, output_dim/(-2.0), beta_gpu_slice.gpudata, 1, dL_dthetaL_gpu.gpudata, 1)
-        cublas.cublasDcopy(self.cublas_handle, beta_gpu_slice.size, beta_gpu_slice.gpudata, 1, thetaL_t_gpu.gpudata, 1)
-        mul_bcast(thetaL_t_gpu,thetaL_t_gpu,thetaL_t_gpu,thetaL_t_gpu.size)
-        mul_bcast(thetaL_t_gpu,thetaL_t_gpu,psi0p_gpu,thetaL_t_gpu.size)
-        cublas.cublasDaxpy(self.cublas_handle, dL_dthetaL_gpu.size, output_dim/2.0, thetaL_t_gpu.gpudata, 1, dL_dthetaL_gpu.gpudata, 1)
-        
-        thetaL_t_gpu.fill(0.)
-        sum_axis(thetaL_t_gpu, psi2R_gpu, nSlice, num_inducing*num_inducing)
-        mul_bcast(thetaL_t_gpu,thetaL_t_gpu,beta_gpu_slice,thetaL_t_gpu.size)
-        mul_bcast(thetaL_t_gpu,thetaL_t_gpu,beta_gpu_slice,thetaL_t_gpu.size)
-        cublas.cublasDaxpy(self.cublas_handle, dL_dthetaL_gpu.size, -1.0, thetaL_t_gpu.gpudata, 1, dL_dthetaL_gpu.gpudata, 1)
-        
-        cublas.cublasDgemm(self.cublas_handle, 'T', 'T', output_dim, nSlice, num_inducing, -1.0, v_gpu.gpudata, num_inducing, betapsi1_gpu.gpudata, nSlice, 0.0, betaYT2_gpu.gpudata, output_dim)
-        mul_bcast(betaYT2_gpu,betaYT2_gpu,betaYT_gpu_slice,betaYT2_gpu.size)
-        sum_axis(dL_dthetaL_gpu, betaYT2_gpu, 1, output_dim)
 
         if kern.useGPU:
             dL_dpsi0 = dL_dpsi0_gpu
@@ -527,10 +464,11 @@ class VarDTC_GPU(object):
                 dL_dpsi2 = dL_dpsi2_gpu
             else:
                 dL_dpsi2 = dL_dpsi2_gpu.get()
-        if het_noise:
+        if not het_noise:
-            dL_dthetaL = dL_dthetaL_gpu.get()
+            if isEnd:
+                dL_dthetaL = self.midRes['dL_dthetaL']
             else:
-            dL_dthetaL = gpuarray.sum(dL_dthetaL_gpu).get()
+                dL_dthetaL = 0.
         if uncertain_inputs:
             grad_dict = {'dL_dpsi0':dL_dpsi0,
                          'dL_dpsi1':dL_dpsi1,

GPy/inference/latent_function_inference/var_dtc_parallel.py

@@ -7,9 +7,15 @@ from ...util import diag
 from ...core.parameterization.variational import VariationalPosterior
 import numpy as np
 from ...util.misc import param_to_array
+from . import LatentFunctionInference
 log_2_pi = np.log(2*np.pi)
 
-class VarDTC_minibatch(object):
+try:
+    from mpi4py import MPI
+except:
+    pass
+
+class VarDTC_minibatch(LatentFunctionInference):
     """
     An object for inference when the likelihood is Gaussian, but we want to do sparse inference.
 
@@ -20,9 +26,11 @@ class VarDTC_minibatch(object):
 
     """
     const_jitter = 1e-6
-    def __init__(self, batchsize, limit=1):
+    def __init__(self, batchsize=None, limit=1, mpi_comm=None):
         
         self.batchsize = batchsize
+        self.mpi_comm = mpi_comm
+        self.limit = limit
         
         # Cache functions
         from ...util.caching import Cacher
@@ -31,6 +39,21 @@ class VarDTC_minibatch(object):
         
         self.midRes = {}
         self.batch_pos = 0 # the starting position of the current mini-batch
+        self.Y_speedup = False # Replace Y with the cholesky factor of YY.T, but the posterior inference will be wrong
+        
+    def __getstate__(self):
+        # has to be overridden, as Cacher objects cannot be pickled. 
+        return self.batchsize, self.limit, self.Y_speedup
+
+    def __setstate__(self, state):
+        # has to be overridden, as Cacher objects cannot be pickled. 
+        self.batchsize, self.limit, self.Y_speedup = state
+        self.mpi_comm = None
+        self.midRes = {}
+        self.batch_pos = 0
+        from ...util.caching import Cacher
+        self.get_trYYT = Cacher(self._get_trYYT, self.limit)
+        self.get_YYTfactor = Cacher(self._get_YYTfactor, self.limit)
 
     def set_limit(self, limit):
         self.get_trYYT.limit = limit
@@ -51,6 +74,67 @@ class VarDTC_minibatch(object):
         else:
             return jitchol(tdot(Y))
         
+    def gatherPsiStat(self, kern, X, Z, Y, beta, uncertain_inputs):
+        
+        het_noise = beta.size > 1
+
+        trYYT = self.get_trYYT(Y)
+        if self.Y_speedup and not het_noise:
+            Y =  self.get_YYTfactor(Y)
+        
+        num_inducing = Z.shape[0]        
+        num_data, output_dim = Y.shape
+        if self.batchsize == None:
+            self.batchsize = num_data
+
+        psi2_full = np.zeros((num_inducing,num_inducing))
+        psi1Y_full = np.zeros((output_dim,num_inducing)) # DxM
+        psi0_full = 0.
+        YRY_full = 0.
+        
+        for n_start in xrange(0,num_data,self.batchsize):            
+            n_end = min(self.batchsize+n_start, num_data)
+            if (n_end-n_start)==num_data:
+                Y_slice = Y
+                X_slice = X
+            else:
+                Y_slice = Y[n_start:n_end]
+                X_slice = X[n_start:n_end]
+            
+            if het_noise:
+                b = beta[n_start]
+                YRY_full += np.inner(Y_slice, Y_slice)*b
+            else:
+                b = beta
+            
+            if uncertain_inputs:
+                psi0 = kern.psi0(Z, X_slice)
+                psi1 = kern.psi1(Z, X_slice)
+                psi2_full += kern.psi2(Z, X_slice)*b
+            else:
+                psi0 = kern.Kdiag(X_slice)
+                psi1 = kern.K(X_slice, Z)
+                psi2_full += np.dot(psi1.T,psi1)*b
+                
+            psi0_full += psi0.sum()*b
+            psi1Y_full += np.dot(Y_slice.T,psi1)*b # DxM                
+
+        if not het_noise:
+            YRY_full = trYYT*beta
+        
+        if self.mpi_comm != None:
+            psi0_all = np.array(psi0_full)
+            psi1Y_all = psi1Y_full.copy()
+            psi2_all = psi2_full.copy()
+            YRY_all = np.array(YRY_full)
+            self.mpi_comm.Allreduce([psi0_full, MPI.DOUBLE], [psi0_all, MPI.DOUBLE])
+            self.mpi_comm.Allreduce([psi1Y_full, MPI.DOUBLE], [psi1Y_all, MPI.DOUBLE])
+            self.mpi_comm.Allreduce([psi2_full, MPI.DOUBLE], [psi2_all, MPI.DOUBLE])
+            self.mpi_comm.Allreduce([YRY_full, MPI.DOUBLE], [YRY_all, MPI.DOUBLE])
+            return psi0_all, psi1Y_all, psi2_all, YRY_all
+            
+        return psi0_full, psi1Y_full, psi2_full, YRY_full
+        
     def inference_likelihood(self, kern, X, Z, likelihood, Y):
         """
         The first phase of inference:
@@ -59,8 +143,11 @@ class VarDTC_minibatch(object):
         Cached intermediate results: Kmm, KmmInv,
         """
         
-        num_inducing = Z.shape[0]        
         num_data, output_dim = Y.shape 
+        if self.mpi_comm != None:
+            num_data_all = np.array(num_data,dtype=np.int32)
+            self.mpi_comm.Allreduce([np.int32(num_data), MPI.INT], [num_data_all, MPI.INT])
+            num_data = num_data_all
 
         if isinstance(X, VariationalPosterior):
             uncertain_inputs = True
@@ -70,75 +157,31 @@ class VarDTC_minibatch(object):
         #see whether we've got a different noise variance for each datum
         beta = 1./np.fmax(likelihood.variance, 1e-6)
         het_noise = beta.size > 1
-        # VVT_factor is a matrix such that tdot(VVT_factor) = VVT...this is for efficiency!
-        #self.YYTfactor = beta*self.get_YYTfactor(Y)
-        YYT_factor = Y
-        trYYT = self.get_trYYT(Y)
-        
-        
-        psi2_full = np.zeros((num_inducing,num_inducing))
-        psi1Y_full = np.zeros((output_dim,num_inducing)) # DxM
-        psi0_full = 0
-        YRY_full = 0
-        
-        for n_start in xrange(0,num_data,self.batchsize):
-            
-            n_end = min(self.batchsize+n_start, num_data)
-            
-            Y_slice = YYT_factor[n_start:n_end]
-            X_slice = X[n_start:n_end]
-            
-            if uncertain_inputs:
-                psi0 = kern.psi0(Z, X_slice)
-                psi1 = kern.psi1(Z, X_slice)
-                psi2 = kern.psi2(Z, X_slice)
-            else:
-                psi0 = kern.Kdiag(X_slice)
-                psi1 = kern.K(X_slice, Z)
-                psi2 = None
-                
         if het_noise:
-                beta_slice = beta[n_start:n_end]
+            self.batchsize = 1
-                psi0_full += (beta_slice*psi0).sum()
-                psi1Y_full += np.dot(beta_slice*Y_slice.T,psi1) # DxM
-                YRY_full += (beta_slice*np.square(Y_slice).sum(axis=-1)).sum()
-            else:
-                psi0_full += psi0.sum()
-                psi1Y_full += np.dot(Y_slice.T,psi1) # DxM
 
-                
+        psi0_full, psi1Y_full, psi2_full, YRY_full = self.gatherPsiStat(kern, X, Z, Y, beta, uncertain_inputs)
-            if uncertain_inputs:
-                if het_noise:
-                    psi2_full += np.einsum('n,nmo->mo',beta_slice,psi2)
-                else:
-                    psi2_full += psi2.sum(axis=0)
-            else:
-                if het_noise:
-                    psi2_full += np.einsum('n,nm,no->mo',beta_slice,psi1,psi1)
-                else:
-                    psi2_full += tdot(psi1.T)
-                
-        if not het_noise:
-            psi0_full *= beta
-            psi1Y_full *= beta
-            psi2_full *= beta
-            YRY_full = trYYT*beta
         
         #======================================================================
         # Compute Common Components
         #======================================================================
         
+        from ...util.debug import checkFullRank
+        
         Kmm = kern.K(Z).copy()
         diag.add(Kmm, self.const_jitter)
+        checkFullRank(Kmm)
         Lm = jitchol(Kmm)
         
-        Lambda = Kmm+psi2_full
+        LmInvPsi2LmInvT = backsub_both_sides(Lm,psi2_full,transpose='right')
+        Lambda = np.eye(Kmm.shape[0])+LmInvPsi2LmInvT
+        checkFullRank(Lambda)
         LL = jitchol(Lambda)
+        LL = np.dot(Lm,LL)
         b,_ = dtrtrs(LL, psi1Y_full.T)
         bbt = np.square(b).sum()
         v,_ = dtrtrs(LL.T,b,lower=False)
         vvt = np.einsum('md,od->mo',v,v)
-        LmInvPsi2LmInvT = backsub_both_sides(Lm,psi2_full,transpose='right')
         
         Psi2LLInvT = dtrtrs(LL,psi2_full)[0].T
         LmInvPsi2LLInvT= dtrtrs(Lm,Psi2LLInvT)[0]
@@ -170,12 +213,18 @@ class VarDTC_minibatch(object):
         # Compute the Posterior distribution of inducing points p(u|Y)
         #======================================================================
         
-#         phi_u_mean = np.dot(Kmm,v)
+        if not self.Y_speedup or het_noise:
-#         LLInvKmm,_ = dtrtrs(LL,Kmm)
-# #        phi_u_var = np.einsum('ma,mb->ab',LLInvKmm,LLInvKmm)
-#         phi_u_var = Kmm - np.dot(LLInvKmm.T,LLInvKmm)
-        
             post = Posterior(woodbury_inv=KmmInvPsi2P, woodbury_vector=v, K=Kmm, mean=None, cov=None, K_chol=Lm)
+        else:
+            post = None
+        
+        #======================================================================
+        # Compute dL_dthetaL for uncertian input and non-heter noise
+        #======================================================================        
+        
+        if not het_noise:
+            dL_dthetaL = (YRY_full*beta + beta*output_dim*psi0_full - num_data*output_dim*beta)/2. - beta*(dL_dpsi2R*psi2_full).sum() - beta*(v.T*psi1Y_full).sum()
+            self.midRes['dL_dthetaL'] = dL_dthetaL
 
         return logL, dL_dKmm, post
 
@@ -198,6 +247,9 @@ class VarDTC_minibatch(object):
         het_noise = beta.size > 1
         # VVT_factor is a matrix such that tdot(VVT_factor) = VVT...this is for efficiency!
         #self.YYTfactor = beta*self.get_YYTfactor(Y)
+        if self.Y_speedup and not het_noise:
+            YYT_factor = self.get_YYTfactor(Y)
+        else:
             YYT_factor = Y
         
         n_start = self.batch_pos
@@ -209,24 +261,24 @@ class VarDTC_minibatch(object):
             isEnd = False
             self.batch_pos = n_end
         
-        num_slice = n_end-n_start
         Y_slice = YYT_factor[n_start:n_end]
         X_slice = X[n_start:n_end]
         
-        if uncertain_inputs:
+        if not uncertain_inputs:
-            psi0 = kern.psi0(Z, X_slice)
-            psi1 = kern.psi1(Z, X_slice)
-            psi2 = kern.psi2(Z, X_slice)
-        else:
             psi0 = kern.Kdiag(X_slice)
             psi1 = kern.K(X_slice, Z)
             psi2 = None
+            betapsi1 = np.einsum('n,nm->nm',beta,psi1)
+        elif het_noise:
+            psi0 = kern.psi0(Z, X_slice)
+            psi1 = kern.psi1(Z, X_slice)
+            psi2 = kern.psi2(Z, X_slice)
+            betapsi1 = np.einsum('n,nm->nm',beta,psi1)
             
         if het_noise:
-            beta = beta[n_start:n_end]
+            beta = beta[n_start] # assuming batchsize==1
 
         betaY = beta*Y_slice
-        betapsi1 = np.einsum('n,nm->nm',beta,psi1)
         
         #======================================================================
         # Load Intermediate Results
@@ -239,12 +291,12 @@ class VarDTC_minibatch(object):
         # Compute dL_dpsi
         #======================================================================
         
-        dL_dpsi0 = -0.5 * output_dim * (beta * np.ones((n_end-n_start,)))
+        dL_dpsi0 = -output_dim * (beta * np.ones((n_end-n_start,)))/2.
         
         dL_dpsi1 = np.dot(betaY,v.T)
         
         if uncertain_inputs:
-            dL_dpsi2 = np.einsum('n,mo->nmo',beta * np.ones((n_end-n_start,)),dL_dpsi2R)
+            dL_dpsi2 = beta* dL_dpsi2R
         else:
             dL_dpsi1 += np.dot(betapsi1,dL_dpsi2R)*2.
             dL_dpsi2 = None
@@ -255,18 +307,16 @@ class VarDTC_minibatch(object):
 
         if het_noise:
             if uncertain_inputs:
-                psiR = np.einsum('mo,nmo->n',dL_dpsi2R,psi2)
+                psiR = np.einsum('mo,mo->',dL_dpsi2R,psi2)
-            else:
-                psiR = np.einsum('nm,no,mo->n',psi1,psi1,dL_dpsi2R)
-            
-            dL_dthetaL = ((np.square(betaY)).sum(axis=-1) + np.square(beta)*(output_dim*psi0)-output_dim*beta)/2. - np.square(beta)*psiR- (betaY*np.dot(betapsi1,v)).sum(axis=-1)
-        else:
-            if uncertain_inputs:
-                psiR = np.einsum('mo,nmo->',dL_dpsi2R,psi2)
             else:
                 psiR = np.einsum('nm,no,mo->',psi1,psi1,dL_dpsi2R)
             
-            dL_dthetaL = ((np.square(betaY)).sum() + np.square(beta)*output_dim*(psi0.sum())-num_slice*output_dim*beta)/2. - np.square(beta)*psiR- (betaY*np.dot(betapsi1,v)).sum()
+            dL_dthetaL = ((np.square(betaY)).sum(axis=-1) + np.square(beta)*(output_dim*psi0)-output_dim*beta)/2. - np.square(beta)*psiR- (betaY*np.dot(betapsi1,v)).sum(axis=-1)
+        else:
+            if isEnd:
+                dL_dthetaL = self.midRes['dL_dthetaL']
+            else:
+                dL_dthetaL = 0.
                 
         if uncertain_inputs:
             grad_dict = {'dL_dpsi0':dL_dpsi0,
@@ -281,36 +331,45 @@ class VarDTC_minibatch(object):
         return isEnd, (n_start,n_end), grad_dict
 
 
-def update_gradients(model):
+def update_gradients(model, mpi_comm=None):
-    model._log_marginal_likelihood, dL_dKmm, model.posterior = model.inference_method.inference_likelihood(model.kern, model.X, model.Z, model.likelihood, model.Y)
+    if mpi_comm == None:
+        Y = model.Y
+        X = model.X
+    else:
+        Y = model.Y_local
+        X = model.X[model.N_range[0]:model.N_range[1]]
+
+    model._log_marginal_likelihood, dL_dKmm, model.posterior = model.inference_method.inference_likelihood(model.kern, X, model.Z, model.likelihood, Y)
     
     het_noise = model.likelihood.variance.size > 1
     
     if het_noise:
         dL_dthetaL = np.empty((model.Y.shape[0],))
     else:
-        dL_dthetaL = 0
+        dL_dthetaL = np.float64(0.)
     
-    #gradients w.r.t. kernel
-    model.kern.update_gradients_full(dL_dKmm, model.Z, None)
     kern_grad = model.kern.gradient.copy()
-            
+    kern_grad[:] = 0.
-    #gradients w.r.t. Z
+    model.Z.gradient = 0.
-    model.Z.gradient[:,model.kern.active_dims] = model.kern.gradients_X(dL_dKmm, model.Z)
     
     isEnd = False
     while not isEnd:
-        isEnd, n_range, grad_dict = model.inference_method.inference_minibatch(model.kern, model.X, model.Z, model.likelihood, model.Y)
+        isEnd, n_range, grad_dict = model.inference_method.inference_minibatch(model.kern, X, model.Z, model.likelihood, Y)
         if isinstance(model.X, VariationalPosterior):
+            if (n_range[1]-n_range[0])==X.shape[0]:
+                X_slice = X
+            elif mpi_comm ==None:
                 X_slice = model.X[n_range[0]:n_range[1]]
+            else:
+                X_slice = model.X[model.N_range[0]+n_range[0]:model.N_range[0]+n_range[1]]
             
             #gradients w.r.t. kernel
             model.kern.update_gradients_expectations(variational_posterior=X_slice, Z=model.Z, dL_dpsi0=grad_dict['dL_dpsi0'], dL_dpsi1=grad_dict['dL_dpsi1'], dL_dpsi2=grad_dict['dL_dpsi2'])
             kern_grad += model.kern.gradient
     
             #gradients w.r.t. Z
-            model.Z.gradient[:,model.kern.active_dims] += model.kern.gradients_Z_expectations(
+            model.Z.gradient += model.kern.gradients_Z_expectations(
-                               grad_dict['dL_dpsi1'], grad_dict['dL_dpsi2'], Z=model.Z, variational_posterior=X_slice)
+                               dL_dpsi0=grad_dict['dL_dpsi0'], dL_dpsi1=grad_dict['dL_dpsi1'], dL_dpsi2=grad_dict['dL_dpsi2'], Z=model.Z, variational_posterior=X_slice)
         
             #gradients w.r.t. posterior parameters of X
             X_grad = model.kern.gradients_qX_expectations(variational_posterior=X_slice, Z=model.Z, dL_dpsi0=grad_dict['dL_dpsi0'], dL_dpsi1=grad_dict['dL_dpsi1'], dL_dpsi2=grad_dict['dL_dpsi2'])
@@ -321,13 +380,39 @@ def update_gradients(model):
             else:
                 dL_dthetaL += grad_dict['dL_dthetaL']
     
-    # Set the gradients w.r.t. kernel
+    # Gather the gradients from multiple MPI nodes
-    model.kern.gradient = kern_grad
+    if mpi_comm != None:
+        if het_noise:
+            raise "het_noise not implemented!"
+        kern_grad_all = kern_grad.copy()
+        Z_grad_all = model.Z.gradient.copy()
+        mpi_comm.Allreduce([kern_grad, MPI.DOUBLE], [kern_grad_all, MPI.DOUBLE])
+        mpi_comm.Allreduce([model.Z.gradient, MPI.DOUBLE], [Z_grad_all, MPI.DOUBLE])
+        kern_grad = kern_grad_all
+        model.Z.gradient = Z_grad_all
+    
+    #gradients w.r.t. kernel
+    model.kern.update_gradients_full(dL_dKmm, model.Z, None)
+    model.kern.gradient += kern_grad
+
+    #gradients w.r.t. Z
+    model.Z.gradient += model.kern.gradients_X(dL_dKmm, model.Z)
     
     # Update Log-likelihood
-    model._log_marginal_likelihood -= model.variational_prior.KL_divergence(model.X)
+    KL_div = model.variational_prior.KL_divergence(X)
     # update for the KL divergence
-    model.variational_prior.update_gradients_KL(model.X)
+    model.variational_prior.update_gradients_KL(X)
+
+    if mpi_comm != None:
+        KL_div_all = np.array(KL_div)
+        mpi_comm.Allreduce([np.float64(KL_div), MPI.DOUBLE], [KL_div_all, MPI.DOUBLE])
+        KL_div = KL_div_all
+        [mpi_comm.Allgatherv([pp.copy(), MPI.DOUBLE], [pa, (model.N_list*pa.shape[-1], None), MPI.DOUBLE]) for pp,pa in zip(model.get_X_gradients(X),model.get_X_gradients(model.X))]
+#         from ...models import SSGPLVM
+#         if isinstance(model, SSGPLVM):
+#             grad_pi = np.array(model.variational_prior.pi.gradient)
+#             mpi_comm.Allreduce([grad_pi.copy(), MPI.DOUBLE], [model.variational_prior.pi.gradient, MPI.DOUBLE])
+    model._log_marginal_likelihood -= KL_div
 
     # dL_dthetaL
     model.likelihood.update_gradients(dL_dthetaL)

GPy/inference/optimization/__init__.py

@@ -1,2 +1,3 @@
 from scg import SCG
 from optimization import *
+from hmc import HMC,HMC_shortcut

GPy/inference/optimization/hmc.py

@@ -0,0 +1,157 @@
+"""HMC implementation"""
+
+import numpy as np
+
+
+class HMC:
+    def __init__(self,model,M=None,stepsize=1e-1):
+        self.model = model
+        self.stepsize = stepsize
+        self.p = np.empty_like(model.optimizer_array.copy())
+        if M is None:
+            self.M = np.eye(self.p.size)
+        else:
+            self.M = M
+        self.Minv = np.linalg.inv(self.M)
+
+    def sample(self, m_iters=1000, hmc_iters=20):
+        params = np.empty((m_iters,self.p.size))
+        for i in xrange(m_iters):
+            self.p[:] = np.random.multivariate_normal(np.zeros(self.p.size),self.M)
+            H_old = self._computeH()
+            theta_old = self.model.optimizer_array.copy()
+            params[i] = self.model.unfixed_param_array
+            #Matropolis
+            self._update(hmc_iters)
+            H_new = self._computeH()
+
+            if H_old>H_new:
+                k = 1.
+            else:
+                k = np.exp(H_old-H_new)
+            if np.random.rand()<k:
+                params[i] = self.model.unfixed_param_array
+            else:
+                self.model.optimizer_array = theta_old
+        return params
+
+    def _update(self, hmc_iters):
+        for i in xrange(hmc_iters):
+            self.p[:] += -self.stepsize/2.*self.model._transform_gradients(self.model.objective_function_gradients())
+            self.model.optimizer_array = self.model.optimizer_array + self.stepsize*np.dot(self.Minv, self.p)
+            self.p[:] += -self.stepsize/2.*self.model._transform_gradients(self.model.objective_function_gradients())
+
+    def _computeH(self,):
+        return self.model.objective_function()+self.p.size*np.log(2*np.pi)/2.+np.log(np.linalg.det(self.M))/2.+np.dot(self.p, np.dot(self.Minv,self.p[:,None]))/2.
+
+class HMC_shortcut:
+    def __init__(self,model,M=None,stepsize_range=[1e-6, 1e-1],groupsize=5, Hstd_th=[1e-5, 3.]):
+        self.model = model
+        self.stepsize_range = np.log(stepsize_range)
+        self.p = np.empty_like(model.optimizer_array.copy())
+        self.groupsize = groupsize
+        self.Hstd_th = Hstd_th
+        if M is None:
+            self.M = np.eye(self.p.size)
+        else:
+            self.M = M
+        self.Minv = np.linalg.inv(self.M)
+
+    def sample(self, m_iters=1000, hmc_iters=20):
+        params = np.empty((m_iters,self.p.size))
+        for i in xrange(m_iters):
+            # sample a stepsize from the uniform distribution
+            stepsize = np.exp(np.random.rand()*(self.stepsize_range[1]-self.stepsize_range[0])+self.stepsize_range[0])
+            self.p[:] = np.random.multivariate_normal(np.zeros(self.p.size),self.M)
+            H_old = self._computeH()
+            params[i] = self.model.unfixed_param_array
+            theta_old = self.model.optimizer_array.copy()
+            #Matropolis
+            self._update(hmc_iters, stepsize)
+            H_new = self._computeH()
+
+            if H_old>H_new:
+                k = 1.
+            else:
+                k = np.exp(H_old-H_new)
+            if np.random.rand()<k:
+                params[i] = self.model.unfixed_param_array
+            else:
+                self.model.optimizer_array = theta_old
+        return params
+
+    def _update(self, hmc_iters, stepsize):
+        theta_buf = np.empty((2*hmc_iters+1,self.model.optimizer_array.size))
+        p_buf = np.empty((2*hmc_iters+1,self.p.size))
+        H_buf = np.empty((2*hmc_iters+1,))
+        # Set initial position
+        theta_buf[hmc_iters] = self.model.optimizer_array
+        p_buf[hmc_iters] = self.p
+        H_buf[hmc_iters] = self._computeH()
+
+        reversal = []
+        pos = 1
+        i=0
+        while i<hmc_iters:
+            self.p[:] += -stepsize/2.*self.model._transform_gradients(self.model.objective_function_gradients())
+            self.model.optimizer_array = self.model.optimizer_array + stepsize*np.dot(self.Minv, self.p)
+            self.p[:] += -stepsize/2.*self.model._transform_gradients(self.model.objective_function_gradients())
+
+            theta_buf[hmc_iters+pos] = self.model.optimizer_array
+            p_buf[hmc_iters+pos] = self.p
+            H_buf[hmc_iters+pos] = self._computeH()
+            i+=1
+
+            if i<self.groupsize:
+                pos += 1
+                continue
+            else:
+                if len(reversal)==0:
+                    Hlist = range(hmc_iters+pos,hmc_iters+pos-self.groupsize,-1)
+                    if self._testH(H_buf[Hlist]):
+                        pos += 1
+                    else:
+                        # Reverse the trajectory for the 1st time
+                        reversal.append(pos)
+                        if hmc_iters-i>pos:
+                            pos = -1
+                            i += pos
+                            self.model.optimizer_array = theta_buf[hmc_iters]
+                            self.p[:] = -p_buf[hmc_iters]
+                        else:
+                            pos_new = pos-hmc_iters+i
+                            self.model.optimizer_array = theta_buf[hmc_iters+pos_new]
+                            self.p[:] = -p_buf[hmc_iters+pos_new]
+                            break
+                else:
+                    Hlist = range(hmc_iters+pos,hmc_iters+pos+self.groupsize)
+#                    print Hlist
+#                    print self._testH(H_buf[Hlist])
+
+                    if self._testH(H_buf[Hlist]):
+                        pos += -1
+                    else:
+                        # Reverse the trajectory for the 2nd time
+                        r = (hmc_iters - i)%((reversal[0]-pos)*2)
+                        if r>(reversal[0]-pos):
+                            pos_new = 2*reversal[0] - r - pos
+                        else:
+                            pos_new = pos + r
+                        self.model.optimizer_array = theta_buf[hmc_iters+pos_new]
+                        self.p[:] = p_buf[hmc_iters+pos_new] # the sign of momentum might be wrong!
+#                        print reversal[0],pos,pos_new
+#                        print H_buf
+                        break
+
+    def _testH(self, Hlist):
+        Hstd = np.std(Hlist)
+#        print Hlist
+#        print Hstd
+        if Hstd<self.Hstd_th[0] or Hstd>self.Hstd_th[1]:
+            return False
+        else:
+            return True
+
+    def _computeH(self,):
+        return self.model.objective_function()+self.p.size*np.log(2*np.pi)/2.+np.log(np.linalg.det(self.M))/2.+np.dot(self.p, np.dot(self.Minv,self.p[:,None]))/2.
+

GPy/inference/optimization/scg.py

@@ -32,7 +32,7 @@ def print_out(len_maxiters, fnow, current_grad, beta, iteration):
     sys.stdout.flush()
 
 def exponents(fnow, current_grad):
-    exps = [np.abs(fnow), current_grad]
+    exps = [np.abs(np.float(fnow)), current_grad]
     return np.sign(exps) * np.log10(exps).astype(int)
 
 def SCG(f, gradf, x, optargs=(), maxiters=500, max_f_eval=np.inf, display=True, xtol=None, ftol=None, gtol=None):
@@ -56,13 +56,13 @@ def SCG(f, gradf, x, optargs=(), maxiters=500, max_f_eval=np.inf, display=True,
     if gtol is None:
         gtol = 1e-5
 
-    sigma0 = 1.0e-8
+    sigma0 = 1.0e-7
     fold = f(x, *optargs) # Initial function value.
     function_eval = 1
     fnow = fold
     gradnew = gradf(x, *optargs) # Initial gradient.
-    if any(np.isnan(gradnew)):
+    #if any(np.isnan(gradnew)):
-        raise UnexpectedInfOrNan, "Gradient contribution resulted in a NaN value"
+    #    raise UnexpectedInfOrNan, "Gradient contribution resulted in a NaN value"
     current_grad = np.dot(gradnew, gradnew)
     gradold = gradnew.copy()
     d = -gradnew # Initial search direction.
@@ -168,13 +168,13 @@ def SCG(f, gradf, x, optargs=(), maxiters=500, max_f_eval=np.inf, display=True,
         if Delta < 0.25:
             beta = min(4.0 * beta, betamax)
         if Delta > 0.75:
-            beta = max(0.5 * beta, betamin)
+            beta = max(0.25 * beta, betamin)
 
         # Update search direction using Polak-Ribiere formula, or re-start
         # in direction of negative gradient after nparams steps.
         if nsuccess == x.size:
             d = -gradnew
-#             beta = 1.  # TODO: betareset!!
+            beta = 1. # This is not in the original paper
             nsuccess = 0
         elif success:
             Gamma = np.dot(gradold - gradnew, gradnew) / (mu)

GPy/installation.cfg

@@ -0,0 +1,2 @@
+# This is the local installation configuration file for GPy
+

GPy/kern/__init__.py

@@ -3,16 +3,20 @@ from _src.rbf import RBF
 from _src.linear import Linear, LinearFull
 from _src.static import Bias, White
 from _src.brownian import Brownian
-from _src.stationary import Exponential, Matern32, Matern52, ExpQuad, RatQuad, Cosine
+from _src.stationary import Exponential, OU, Matern32, Matern52, ExpQuad, RatQuad, Cosine
 from _src.mlp import MLP
 from _src.periodic import PeriodicExponential, PeriodicMatern32, PeriodicMatern52
 from _src.independent_outputs import IndependentOutputs, Hierarchical
 from _src.coregionalize import Coregionalize
-from _src.ssrbf import SSRBF # TODO: ZD: did you remove this?
 from _src.ODE_UY import ODE_UY
+from _src.ODE_UYC import ODE_UYC
+from _src.ODE_st import ODE_st
+from _src.ODE_t import ODE_t
 from _src.poly import Poly
-#from _src.ODE_UYC import ODE_UYC ADD THIS FILE TO THE REPO!!
+
-#from _src.ODE_st import ODE_st
+from _src.trunclinear import TruncLinear,TruncLinear_inf
+from _src.splitKern import SplitKern,DiffGenomeKern
+
 # TODO: put this in an init file somewhere
 #I'm commenting this out because the files were not added. JH. Remember to add the files before commiting
 try:

GPy/kern/_src/ODE_UYC.py

@@ -0,0 +1,290 @@
+# Copyright (c) 2013, GPy authors (see AUTHORS.txt).
+# Licensed under the BSD 3-clause license (see LICENSE.txt)
+
+from kern import Kern
+from ...core.parameterization import Param
+from ...core.parameterization.transformations import Logexp
+import numpy as np
+from independent_outputs import index_to_slices
+
+class ODE_UYC(Kern):
+    def __init__(self, input_dim, variance_U=3., variance_Y=1., lengthscale_U=1., lengthscale_Y=1., ubias =1. ,active_dims=None, name='ode_uyc'):
+        assert input_dim ==2, "only defined for 2 input dims"
+        super(ODE_UYC, self).__init__(input_dim, active_dims, name)
+
+        self.variance_Y = Param('variance_Y', variance_Y, Logexp())
+        self.variance_U = Param('variance_U', variance_U, Logexp())
+        self.lengthscale_Y = Param('lengthscale_Y', lengthscale_Y, Logexp())
+        self.lengthscale_U = Param('lengthscale_U', lengthscale_U, Logexp())
+        self.ubias = Param('ubias', ubias, Logexp())
+
+        self.add_parameters(self.variance_Y, self.variance_U, self.lengthscale_Y, self.lengthscale_U, self.ubias)
+
+    def K(self, X, X2=None):
+        # model :   a * dy/dt + b * y = U
+        #lu=sqrt(3)/theta1  ly=1/theta2  theta2= a/b :thetay   sigma2=1/(2ab) :sigmay
+
+        X,slices = X[:,:-1],index_to_slices(X[:,-1])
+        if X2 is None:
+            X2,slices2 = X,slices
+            K = np.zeros((X.shape[0], X.shape[0]))
+        else:
+            X2,slices2 = X2[:,:-1],index_to_slices(X2[:,-1])
+            K = np.zeros((X.shape[0], X2.shape[0]))
+
+        #stop
+        #rdist = X[:,0][:,None] - X2[:,0][:,None].T
+        rdist = X - X2.T
+        ly=1/self.lengthscale_Y
+        lu=np.sqrt(3)/self.lengthscale_U
+        #iu=self.input_lengthU  #dimention of U
+        Vu=self.variance_U
+        Vy=self.variance_Y
+        #Vy=ly/2
+        #stop
+
+
+        # kernel for kuu  matern3/2
+        kuu = lambda dist:Vu * (1 + lu* np.abs(dist)) * np.exp(-lu * np.abs(dist)) +self.ubias
+
+        # kernel for kyy
+        k1 = lambda dist:np.exp(-ly*np.abs(dist))*(2*lu+ly)/(lu+ly)**2
+        k2 = lambda dist:(np.exp(-lu*dist)*(ly-2*lu+lu*ly*dist-lu**2*dist) + np.exp(-ly*dist)*(2*lu-ly) ) / (ly-lu)**2
+        k3 = lambda dist:np.exp(-lu*dist) * ( (1+lu*dist)/(lu+ly) + (lu)/(lu+ly)**2 )
+        kyy = lambda dist:Vu*Vy*(k1(dist) + k2(dist) + k3(dist))
+
+
+        # cross covariance function
+        kyu3 = lambda dist:np.exp(-lu*dist)/(lu+ly)*(1+lu*(dist+1/(lu+ly)))
+        #kyu3 = lambda dist: 0
+
+        k1cros = lambda dist:np.exp(ly*dist)/(lu-ly) * ( 1- np.exp( (lu-ly)*dist) + lu* ( dist*np.exp( (lu-ly)*dist ) + (1- np.exp( (lu-ly)*dist ) ) /(lu-ly)   )    )
+        #k1cros = lambda dist:0
+
+        k2cros = lambda dist:np.exp(ly*dist)*( 1/(lu+ly) + lu/(lu+ly)**2 )
+        #k2cros = lambda dist:0
+
+        Vyu=np.sqrt(Vy*ly*2)
+
+        # cross covariance kuy
+        kuyp = lambda dist:Vu*Vyu*(kyu3(dist))       #t>0 kuy
+        kuyn = lambda dist:Vu*Vyu*(k1cros(dist)+k2cros(dist))      #t<0 kuy
+        # cross covariance kyu
+        kyup = lambda dist:Vu*Vyu*(k1cros(-dist)+k2cros(-dist))    #t>0 kyu
+        kyun = lambda dist:Vu*Vyu*(kyu3(-dist))       #t<0 kyu
+
+
+        for i, s1 in enumerate(slices):
+            for j, s2 in enumerate(slices2):
+                for ss1 in s1:
+                    for ss2 in s2:
+                        if i==0 and j==0:
+                            K[ss1,ss2] = kuu(np.abs(rdist[ss1,ss2]))
+                        elif i==0 and j==1:
+                            #K[ss1,ss2]=  np.where(  rdist[ss1,ss2]>0 , kuyp(np.abs(rdist[ss1,ss2])), kuyn(np.abs(rdist[ss1,ss2]) )   )
+                            K[ss1,ss2]=  np.where(  rdist[ss1,ss2]>0 , kuyp(rdist[ss1,ss2]), kuyn(rdist[ss1,ss2] )   )
+                        elif i==1 and j==1:
+                            K[ss1,ss2] = kyy(np.abs(rdist[ss1,ss2]))
+                        else:
+                            #K[ss1,ss2]= 0
+                            #K[ss1,ss2]= np.where(  rdist[ss1,ss2]>0 , kyup(np.abs(rdist[ss1,ss2])), kyun(np.abs(rdist[ss1,ss2]) )   )
+                            K[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , kyup(rdist[ss1,ss2]), kyun(rdist[ss1,ss2] )   )
+        return K
+
+
+
+    def Kdiag(self, X):
+        """Compute the diagonal of the covariance matrix associated to X."""
+        Kdiag = np.zeros(X.shape[0])
+        ly=1/self.lengthscale_Y
+        lu=np.sqrt(3)/self.lengthscale_U
+
+        Vu = self.variance_U
+        Vy=self.variance_Y
+
+        k1 = (2*lu+ly)/(lu+ly)**2
+        k2 = (ly-2*lu + 2*lu-ly ) / (ly-lu)**2
+        k3 = 1/(lu+ly) + (lu)/(lu+ly)**2
+
+        slices = index_to_slices(X[:,-1])
+
+        for i, ss1 in enumerate(slices):
+            for s1 in ss1:
+                if i==0:
+                    Kdiag[s1]+= self.variance_U + self.ubias
+                elif i==1:
+                    Kdiag[s1]+= Vu*Vy*(k1+k2+k3)
+                else:
+                    raise ValueError, "invalid input/output index"
+        #Kdiag[slices[0][0]]+= self.variance_U   #matern32 diag
+        #Kdiag[slices[1][0]]+= self.variance_U*self.variance_Y*(k1+k2+k3)  #  diag
+        return Kdiag
+
+
+    def update_gradients_full(self, dL_dK, X, X2=None):
+        """derivative of the covariance matrix with respect to the parameters."""
+        X,slices = X[:,:-1],index_to_slices(X[:,-1])
+        if X2 is None:
+            X2,slices2 = X,slices
+        else:
+            X2,slices2 = X2[:,:-1],index_to_slices(X2[:,-1])
+        #rdist = X[:,0][:,None] - X2[:,0][:,None].T
+
+        rdist = X - X2.T
+        ly=1/self.lengthscale_Y
+        lu=np.sqrt(3)/self.lengthscale_U
+
+        Vu=self.variance_U
+        Vy=self.variance_Y
+        Vyu = np.sqrt(Vy*ly*2)
+        dVdly = 0.5/np.sqrt(ly)*np.sqrt(2*Vy)
+        dVdVy = 0.5/np.sqrt(Vy)*np.sqrt(2*ly)
+
+        rd=rdist.shape[0]
+        dktheta1 = np.zeros([rd,rd])
+        dktheta2 = np.zeros([rd,rd])
+        dkUdvar = np.zeros([rd,rd])
+        dkYdvar = np.zeros([rd,rd])
+
+        dkdubias = np.zeros([rd,rd])
+
+        # dk dtheta for UU
+        UUdtheta1 = lambda dist: np.exp(-lu* dist)*dist + (-dist)*np.exp(-lu* dist)*(1+lu*dist)
+        UUdtheta2 = lambda dist: 0
+        #UUdvar = lambda dist: (1 + lu*dist)*np.exp(-lu*dist)
+        UUdvar = lambda dist: (1 + lu* np.abs(dist)) * np.exp(-lu * np.abs(dist))
+
+        # dk dtheta for YY
+
+        dk1theta1 = lambda dist: np.exp(-ly*dist)*2*(-lu)/(lu+ly)**3
+
+        dk2theta1 = lambda dist: (1.0)*(
+            np.exp(-lu*dist)*dist*(-ly+2*lu-lu*ly*dist+dist*lu**2)*(ly-lu)**(-2) + np.exp(-lu*dist)*(-2+ly*dist-2*dist*lu)*(ly-lu)**(-2)
+            +np.exp(-dist*lu)*(ly-2*lu+ly*lu*dist-dist*lu**2)*2*(ly-lu)**(-3)
+            +np.exp(-dist*ly)*2*(ly-lu)**(-2)
+            +np.exp(-dist*ly)*2*(2*lu-ly)*(ly-lu)**(-3)
+            )
+
+        dk3theta1 = lambda dist: np.exp(-dist*lu)*(lu+ly)**(-2)*((2*lu+ly+dist*lu**2+lu*ly*dist)*(-dist-2/(lu+ly))+2+2*lu*dist+ly*dist)
+
+        #dktheta1 = lambda dist: self.variance_U*self.variance_Y*(dk1theta1+dk2theta1+dk3theta1)
+
+
+
+
+        dk1theta2 = lambda dist: np.exp(-ly*dist) * ((lu+ly)**(-2)) * (  (-dist)*(2*lu+ly)  +  1  +  (-2)*(2*lu+ly)/(lu+ly)  )
+
+        dk2theta2 =lambda dist:  1*(
+            np.exp(-dist*lu)*(ly-lu)**(-2) * ( 1+lu*dist+(-2)*(ly-2*lu+lu*ly*dist-dist*lu**2)*(ly-lu)**(-1) )
+            +np.exp(-dist*ly)*(ly-lu)**(-2) * ( (-dist)*(2*lu-ly) -1+(2*lu-ly)*(-2)*(ly-lu)**(-1) )
+            )
+
+        dk3theta2 = lambda dist: np.exp(-dist*lu) * (-3*lu-ly-dist*lu**2-lu*ly*dist)/(lu+ly)**3
+
+        #dktheta2 = lambda dist: self.variance_U*self.variance_Y*(dk1theta2 + dk2theta2 +dk3theta2)
+
+        # kyy kernel
+
+        k1 = lambda dist: np.exp(-ly*dist)*(2*lu+ly)/(lu+ly)**2
+        k2 = lambda dist: (np.exp(-lu*dist)*(ly-2*lu+lu*ly*dist-lu**2*dist) + np.exp(-ly*dist)*(2*lu-ly) ) / (ly-lu)**2
+        k3 = lambda dist: np.exp(-lu*dist) * ( (1+lu*dist)/(lu+ly) + (lu)/(lu+ly)**2 )
+        #dkdvar = k1+k2+k3
+
+
+
+        # cross covariance function
+        kyu3 = lambda dist:np.exp(-lu*dist)/(lu+ly)*(1+lu*(dist+1/(lu+ly)))
+
+        k1cros = lambda dist:np.exp(ly*dist)/(lu-ly) * ( 1- np.exp( (lu-ly)*dist) + lu* ( dist*np.exp( (lu-ly)*dist ) + (1- np.exp( (lu-ly)*dist ) ) /(lu-ly)   )    )
+
+        k2cros = lambda dist:np.exp(ly*dist)*( 1/(lu+ly) + lu/(lu+ly)**2 )
+        # cross covariance kuy
+        kuyp = lambda dist:(kyu3(dist))       #t>0 kuy
+        kuyn = lambda dist:(k1cros(dist)+k2cros(dist))      #t<0 kuy
+        # cross covariance kyu
+        kyup = lambda dist:(k1cros(-dist)+k2cros(-dist))    #t>0 kyu
+        kyun = lambda dist:(kyu3(-dist))       #t<0 kyu
+
+        # dk dtheta for UY
+
+
+        dkyu3dtheta2 = lambda dist: np.exp(-lu*dist) * ( (-1)*(lu+ly)**(-2)*(1+lu*dist+lu*(lu+ly)**(-1)) + (lu+ly)**(-1)*(-lu)*(lu+ly)**(-2) )
+        dkyu3dtheta1 = lambda dist: np.exp(-lu*dist)*(lu+ly)**(-1)* ( (-dist)*(1+dist*lu+lu*(lu+ly)**(-1)) -\
+         (lu+ly)**(-1)*(1+dist*lu+lu*(lu+ly)**(-1)) +dist+(lu+ly)**(-1)-lu*(lu+ly)**(-2) )
+
+        dkcros2dtheta1 = lambda dist: np.exp(ly*dist)* ( -(ly+lu)**(-2) + (ly+lu)**(-2) + (-2)*lu*(lu+ly)**(-3)  )
+        dkcros2dtheta2 = lambda dist: np.exp(ly*dist)*dist* ( (ly+lu)**(-1) + lu*(lu+ly)**(-2) ) + \
+                                      np.exp(ly*dist)*( -(lu+ly)**(-2) + lu*(-2)*(lu+ly)**(-3)  )
+
+        dkcros1dtheta1 = lambda dist: np.exp(ly*dist)*(     -(lu-ly)**(-2)*(  1-np.exp((lu-ly)*dist) + lu*dist*np.exp((lu-ly)*dist)+ \
+          lu*(1-np.exp((lu-ly)*dist))/(lu-ly)  )  +  (lu-ly)**(-1)*(  -np.exp( (lu-ly)*dist )*dist + dist*np.exp( (lu-ly)*dist)+\
+          lu*dist**2*np.exp((lu-ly)*dist)+(1-np.exp((lu-ly)*dist))/(lu-ly) - lu*np.exp((lu-ly)*dist)*dist/(lu-ly) -\
+          lu*(1-np.exp((lu-ly)*dist))/(lu-ly)**2  )   )
+
+        dkcros1dtheta2 = lambda t: np.exp(ly*t)*t/(lu-ly)*( 1-np.exp((lu-ly)*t) +lu*t*np.exp((lu-ly)*t)+\
+            lu*(1-np.exp((lu-ly)*t))/(lu-ly)  )+\
+            np.exp(ly*t)/(lu-ly)**2* ( 1-np.exp((lu-ly)*t) +lu*t*np.exp((lu-ly)*t) + lu*( 1-np.exp((lu-ly)*t) )/(lu-ly)  )+\
+            np.exp(ly*t)/(lu-ly)*( np.exp((lu-ly)*t)*t -lu*t*t*np.exp((lu-ly)*t) +lu*t*np.exp((lu-ly)*t)/(lu-ly)+\
+            lu*( 1-np.exp((lu-ly)*t) )/(lu-ly)**2 )
+
+        dkuypdtheta1 = lambda dist:(dkyu3dtheta1(dist))       #t>0 kuy
+        dkuyndtheta1 = lambda dist:(dkcros1dtheta1(dist)+dkcros2dtheta1(dist))      #t<0 kuy
+        # cross covariance kyu
+        dkyupdtheta1 = lambda dist:(dkcros1dtheta1(-dist)+dkcros2dtheta1(-dist))    #t>0 kyu
+        dkyundtheta1 = lambda dist:(dkyu3dtheta1(-dist))       #t<0 kyu
+
+        dkuypdtheta2 = lambda dist:(dkyu3dtheta2(dist))       #t>0 kuy
+        dkuyndtheta2 = lambda dist:(dkcros1dtheta2(dist)+dkcros2dtheta2(dist))      #t<0 kuy
+        # cross covariance kyu
+        dkyupdtheta2 = lambda dist:(dkcros1dtheta2(-dist)+dkcros2dtheta2(-dist))    #t>0 kyu
+        dkyundtheta2 = lambda dist:(dkyu3dtheta2(-dist))       #t<0 kyu
+
+
+        for i, s1 in enumerate(slices):
+            for j, s2 in enumerate(slices2):
+                for ss1 in s1:
+                    for ss2 in s2:
+                        if i==0 and j==0:
+                            #target[ss1,ss2] = kuu(np.abs(rdist[ss1,ss2]))
+                            dktheta1[ss1,ss2] = Vu*UUdtheta1(np.abs(rdist[ss1,ss2]))
+                            dktheta2[ss1,ss2] = 0
+                            dkUdvar[ss1,ss2] = UUdvar(np.abs(rdist[ss1,ss2]))
+                            dkYdvar[ss1,ss2] = 0
+                            dkdubias[ss1,ss2] = 1
+                        elif i==0 and j==1:
+                            ########target[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , kuyp(np.abs(rdist[ss1,ss2])), kuyn(np.abs(rdist[s1[0],s2[0]]) )   )
+                            #np.where(  rdist[ss1,ss2]>0 , kuyp(np.abs(rdist[ss1,ss2])), kuyn(np.abs(rdist[s1[0],s2[0]]) )   )
+                            #dktheta1[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , self.variance_U*self.variance_Y*dkcrtheta1(np.abs(rdist[ss1,ss2])) ,self.variance_U*self.variance_Y*(dk1theta1(np.abs(rdist[ss1,ss2]))+dk2theta1(np.abs(rdist[ss1,ss2])))    )
+                            #dktheta2[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , self.variance_U*self.variance_Y*dkcrtheta2(np.abs(rdist[ss1,ss2])) ,self.variance_U*self.variance_Y*(dk1theta2(np.abs(rdist[ss1,ss2]))+dk2theta2(np.abs(rdist[ss1,ss2])))    )
+                            dktheta1[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , Vu*Vyu*dkuypdtheta1(rdist[ss1,ss2]),Vu*Vyu*dkuyndtheta1(rdist[ss1,ss2]) )
+                            dkUdvar[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , Vyu*kuyp(rdist[ss1,ss2]), Vyu* kuyn(rdist[ss1,ss2])  )
+                            dktheta2[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , Vu*Vyu*dkuypdtheta2(rdist[ss1,ss2])+Vu*dVdly*kuyp(rdist[ss1,ss2]),Vu*Vyu*dkuyndtheta2(rdist[ss1,ss2])+Vu*dVdly*kuyn(rdist[ss1,ss2]) )
+                            dkYdvar[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , Vu*dVdVy*kuyp(rdist[ss1,ss2]), Vu*dVdVy* kuyn(rdist[ss1,ss2])  )
+                            dkdubias[ss1,ss2] = 0
+                        elif i==1 and j==1:
+                            #target[ss1,ss2] = kyy(np.abs(rdist[ss1,ss2]))
+                            dktheta1[ss1,ss2] = self.variance_U*self.variance_Y*(dk1theta1(np.abs(rdist[ss1,ss2]))+dk2theta1(np.abs(rdist[ss1,ss2]))+dk3theta1(np.abs(rdist[ss1,ss2])))
+                            dktheta2[ss1,ss2] = self.variance_U*self.variance_Y*(dk1theta2(np.abs(rdist[ss1,ss2])) + dk2theta2(np.abs(rdist[ss1,ss2])) +dk3theta2(np.abs(rdist[ss1,ss2])))
+                            dkUdvar[ss1,ss2] = self.variance_Y*(k1(np.abs(rdist[ss1,ss2]))+k2(np.abs(rdist[ss1,ss2]))+k3(np.abs(rdist[ss1,ss2])) )
+                            dkYdvar[ss1,ss2] = self.variance_U*(k1(np.abs(rdist[ss1,ss2]))+k2(np.abs(rdist[ss1,ss2]))+k3(np.abs(rdist[ss1,ss2])) )
+                            dkdubias[ss1,ss2] = 0
+                        else:
+                            #######target[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , kyup(np.abs(rdist[ss1,ss2])), kyun(np.abs(rdist[s1[0],s2[0]]) )   )
+                            #dktheta1[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 ,self.variance_U*self.variance_Y*(dk1theta1(np.abs(rdist[ss1,ss2]))+dk2theta1(np.abs(rdist[ss1,ss2]))) , self.variance_U*self.variance_Y*dkcrtheta1(np.abs(rdist[ss1,ss2])) )
+                            #dktheta2[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 ,self.variance_U*self.variance_Y*(dk1theta2(np.abs(rdist[ss1,ss2]))+dk2theta2(np.abs(rdist[ss1,ss2]))) , self.variance_U*self.variance_Y*dkcrtheta2(np.abs(rdist[ss1,ss2])) )
+                            dktheta1[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , Vu*Vyu*dkyupdtheta1(rdist[ss1,ss2]),Vu*Vyu*dkyundtheta1(rdist[ss1,ss2])  )
+                            dkUdvar[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , Vyu*kyup(rdist[ss1,ss2]),Vyu*kyun(rdist[ss1,ss2]))
+                            dktheta2[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , Vu*Vyu*dkyupdtheta2(rdist[ss1,ss2])+Vu*dVdly*kyup(rdist[ss1,ss2]),Vu*Vyu*dkyundtheta2(rdist[ss1,ss2])+Vu*dVdly*kyun(rdist[ss1,ss2])  )
+                            dkYdvar[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , Vu*dVdVy*kyup(rdist[ss1,ss2]), Vu*dVdVy*kyun(rdist[ss1,ss2]))
+                            dkdubias[ss1,ss2] = 0
+        #stop
+        self.variance_U.gradient = np.sum(dkUdvar * dL_dK)     # Vu
+
+        self.variance_Y.gradient = np.sum(dkYdvar * dL_dK)     # Vy
+
+        self.lengthscale_U.gradient = np.sum(dktheta1*(-np.sqrt(3)*self.lengthscale_U**(-2))* dL_dK)     #lu
+
+        self.lengthscale_Y.gradient = np.sum(dktheta2*(-self.lengthscale_Y**(-2)) * dL_dK)              #ly
+
+        self.ubias.gradient = np.sum(dkdubias * dL_dK) 
+

GPy/kern/_src/ODE_st.py

@@ -0,0 +1,267 @@
+# Copyright (c) 2012, GPy authors (see AUTHORS.txt).
+# Licensed under the BSD 3-clause license (see LICENSE.txt)
+from kern import Kern
+from ...core.parameterization import Param
+from ...core.parameterization.transformations import Logexp
+import numpy as np
+from independent_outputs import index_to_slices
+
+
+class ODE_st(Kern):
+    """
+    kernel resultiong from a first order ODE with OU driving GP
+
+    :param input_dim: the number of input dimension, has to be equal to one
+    :type input_dim: int
+    :param varianceU: variance of the driving GP
+    :type varianceU: float
+    :param lengthscaleU: lengthscale of the driving GP  (sqrt(3)/lengthscaleU)
+    :type lengthscaleU: float
+    :param varianceY: 'variance' of the transfer function
+    :type varianceY: float
+    :param lengthscaleY: 'lengthscale' of the transfer function (1/lengthscaleY)
+    :type lengthscaleY: float
+    :rtype: kernel object
+
+    """
+    
+    def __init__(self, input_dim, a=1.,b=1., c=1.,variance_Yx=3.,variance_Yt=1.5, lengthscale_Yx=1.5, lengthscale_Yt=1.5, active_dims=None, name='ode_st'):
+        assert input_dim ==3, "only defined for 3 input dims"
+        super(ODE_st, self).__init__(input_dim, active_dims, name)
+
+        self.variance_Yt = Param('variance_Yt', variance_Yt, Logexp())
+        self.variance_Yx = Param('variance_Yx', variance_Yx, Logexp())
+        self.lengthscale_Yt = Param('lengthscale_Yt', lengthscale_Yt, Logexp())
+        self.lengthscale_Yx = Param('lengthscale_Yx', lengthscale_Yx, Logexp())        
+
+        self.a= Param('a', a, Logexp())
+        self.b = Param('b', b, Logexp())
+        self.c = Param('c', c, Logexp())
+
+        self.add_parameters(self.a, self.b, self.c, self.variance_Yt, self.variance_Yx, self.lengthscale_Yt,self.lengthscale_Yx)
+
+
+    def K(self, X, X2=None):        
+    # model :   -a d^2y/dx^2  + b dy/dt + c * y = U
+    # kernel Kyy rbf spatiol temporal
+    # vyt Y temporal variance  vyx Y spatiol variance   lyt Y temporal lengthscale   lyx Y spatiol lengthscale
+    # kernel Kuu doper( doper(Kyy))
+    # a   b    c    lyt   lyx    vyx*vyt
+        """Compute the covariance matrix between X and X2."""        
+        X,slices = X[:,:-1],index_to_slices(X[:,-1])
+        if X2 is None:
+            X2,slices2 = X,slices
+            K = np.zeros((X.shape[0], X.shape[0]))
+        else:
+            X2,slices2 = X2[:,:-1],index_to_slices(X2[:,-1])
+            K = np.zeros((X.shape[0], X2.shape[0]))
+
+
+        tdist = (X[:,0][:,None] - X2[:,0][None,:])**2
+        xdist = (X[:,1][:,None] - X2[:,1][None,:])**2
+
+        ttdist = (X[:,0][:,None] - X2[:,0][None,:])
+        #rdist = [tdist,xdist]
+        #dist = np.abs(X - X2.T)
+        vyt = self.variance_Yt
+        vyx = self.variance_Yx
+        
+        lyt=1/(2*self.lengthscale_Yt)
+        lyx=1/(2*self.lengthscale_Yx)
+
+        a = self.a ## -a is used in the model, negtive diffusion
+        b = self.b
+        c = self.c
+
+        kyy = lambda tdist,xdist: np.exp(-lyt*(tdist) -lyx*(xdist))
+
+        k1 = lambda tdist: (2*lyt - 4*lyt**2 * (tdist) )
+
+        k2 = lambda xdist: ( 4*lyx**2 * (xdist)  - 2*lyx )
+
+        k3 = lambda xdist: ( 3*4*lyx**2 - 6*8*xdist*lyx**3 + 16*xdist**2*lyx**4 )
+
+        k4 = lambda ttdist: 2*lyt*(ttdist)
+
+        for i, s1 in enumerate(slices):
+            for j, s2 in enumerate(slices2):
+                for ss1 in s1:
+                    for ss2 in s2:
+                        if i==0 and j==0:
+                            K[ss1,ss2] = vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
+                        elif i==0 and j==1:
+                            K[ss1,ss2] = (-a*k2(xdist[ss1,ss2]) + b*k4(ttdist[ss1,ss2]) + c)*vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
+                            #K[ss1,ss2]=  np.where(  rdist[ss1,ss2]>0 , kuyp(np.abs(rdist[ss1,ss2])), kuyn(np.abs(rdist[ss1,ss2]) )   )
+                            #K[ss1,ss2]=  np.where(  rdist[ss1,ss2]>0 , kuyp(rdist[ss1,ss2]), kuyn(rdist[ss1,ss2] )   )
+                        elif i==1 and j==1:
+                            K[ss1,ss2] = ( b**2*k1(tdist[ss1,ss2]) - 2*a*c*k2(xdist[ss1,ss2]) + a**2*k3(xdist[ss1,ss2]) + c**2 )* vyt*vyx* kyy(tdist[ss1,ss2],xdist[ss1,ss2])
+                        else:
+                            K[ss1,ss2] = (-a*k2(xdist[ss1,ss2]) - b*k4(ttdist[ss1,ss2]) + c)*vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
+                            #K[ss1,ss2]= np.where(  rdist[ss1,ss2]>0 , kyup(np.abs(rdist[ss1,ss2])), kyun(np.abs(rdist[ss1,ss2]) )   )
+                            #K[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , kyup(rdist[ss1,ss2]), kyun(rdist[ss1,ss2] )   )
+        
+        #stop
+        return K
+
+    def Kdiag(self, X):
+        """Compute the diagonal of the covariance matrix associated to X."""
+        vyt = self.variance_Yt
+        vyx = self.variance_Yx
+
+        lyt = 1./(2*self.lengthscale_Yt)
+        lyx = 1./(2*self.lengthscale_Yx)
+
+        a = self.a
+        b = self.b
+        c = self.c
+
+        ## dk^2/dtdt'
+        k1 = (2*lyt )*vyt*vyx
+        ## dk^2/dx^2
+        k2 = ( - 2*lyx )*vyt*vyx
+        ## dk^4/dx^2dx'^2
+        k3 = ( 4*3*lyx**2 )*vyt*vyx
+
+
+        Kdiag = np.zeros(X.shape[0])
+        slices = index_to_slices(X[:,-1])
+
+        for i, ss1 in enumerate(slices):
+            for s1 in ss1:
+                if i==0:
+                    Kdiag[s1]+= vyt*vyx
+                elif i==1:
+                    #i=1
+                    Kdiag[s1]+= b**2*k1 - 2*a*c*k2 + a**2*k3 + c**2*vyt*vyx
+                    #Kdiag[s1]+= Vu*Vy*(k1+k2+k3)
+                else:
+                    raise ValueError, "invalid input/output index"
+
+        return Kdiag
+        
+
+    def update_gradients_full(self, dL_dK, X, X2=None):
+    #def dK_dtheta(self, dL_dK, X, X2, target):
+        """derivative of the covariance matrix with respect to the parameters."""
+        X,slices = X[:,:-1],index_to_slices(X[:,-1])
+        if X2 is None:
+            X2,slices2 = X,slices
+            K = np.zeros((X.shape[0], X.shape[0]))
+        else:
+            X2,slices2 = X2[:,:-1],index_to_slices(X2[:,-1])
+        
+        vyt = self.variance_Yt
+        vyx = self.variance_Yx
+
+        lyt = 1./(2*self.lengthscale_Yt)
+        lyx = 1./(2*self.lengthscale_Yx)
+
+        a = self.a
+        b = self.b
+        c = self.c
+
+        tdist = (X[:,0][:,None] - X2[:,0][None,:])**2
+        xdist = (X[:,1][:,None] - X2[:,1][None,:])**2
+        #rdist = [tdist,xdist]
+        ttdist = (X[:,0][:,None] - X2[:,0][None,:])
+        
+        rd=tdist.shape[0]
+
+        dka = np.zeros([rd,rd])
+        dkb = np.zeros([rd,rd])
+        dkc = np.zeros([rd,rd])
+        dkYdvart = np.zeros([rd,rd])
+        dkYdvarx = np.zeros([rd,rd])
+        dkYdlent = np.zeros([rd,rd])
+        dkYdlenx = np.zeros([rd,rd])
+
+
+        kyy = lambda tdist,xdist: np.exp(-lyt*(tdist) -lyx*(xdist))
+        #k1 = lambda tdist: (lyt - lyt**2 * (tdist) )
+        #k2 = lambda xdist: ( lyx**2 * (xdist)  - lyx )
+        #k3 = lambda xdist: ( 3*lyx**2 - 6*xdist*lyx**3 + xdist**2*lyx**4 )
+        #k4 = lambda tdist: -lyt*np.sqrt(tdist)
+
+        k1 = lambda tdist: (2*lyt - 4*lyt**2 * (tdist) )
+
+        k2 = lambda xdist: ( 4*lyx**2 * (xdist)  - 2*lyx )
+
+        k3 = lambda xdist: ( 3*4*lyx**2 - 6*8*xdist*lyx**3 + 16*xdist**2*lyx**4 )
+
+        k4 = lambda ttdist: 2*lyt*(ttdist)
+
+        dkyydlyx = lambda tdist,xdist: kyy(tdist,xdist)*(-xdist)
+        dkyydlyt = lambda tdist,xdist: kyy(tdist,xdist)*(-tdist)
+
+        dk1dlyt = lambda tdist: 2. - 4*2.*lyt*tdist
+        dk2dlyx = lambda xdist: (4.*2.*lyx*xdist -2.)
+        dk3dlyx = lambda xdist: (6.*4.*lyx - 18.*8*xdist*lyx**2 + 4*16*xdist**2*lyx**3)
+
+        dk4dlyt = lambda ttdist: 2*(ttdist)
+
+        for i, s1 in enumerate(slices):
+            for j, s2 in enumerate(slices2):
+                for ss1 in s1:
+                    for ss2 in s2:
+                        if i==0 and j==0:
+                            dka[ss1,ss2] = 0
+                            dkb[ss1,ss2] = 0
+                            dkc[ss1,ss2] = 0
+                            dkYdvart[ss1,ss2] = vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
+                            dkYdvarx[ss1,ss2] = vyt*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
+                            dkYdlenx[ss1,ss2] = vyt*vyx*dkyydlyx(tdist[ss1,ss2],xdist[ss1,ss2])
+                            dkYdlent[ss1,ss2] = vyt*vyx*dkyydlyt(tdist[ss1,ss2],xdist[ss1,ss2])
+                        elif i==0 and j==1:
+                            dka[ss1,ss2] = -k2(xdist[ss1,ss2])*vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
+                            dkb[ss1,ss2] = k4(ttdist[ss1,ss2])*vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
+                            dkc[ss1,ss2] = vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
+                            #dkYdvart[ss1,ss2] = 0
+                            #dkYdvarx[ss1,ss2] = 0
+                            #dkYdlent[ss1,ss2] = 0
+                            #dkYdlenx[ss1,ss2] = 0
+                            dkYdvart[ss1,ss2] = (-a*k2(xdist[ss1,ss2])+b*k4(ttdist[ss1,ss2])+c)*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
+                            dkYdvarx[ss1,ss2] = (-a*k2(xdist[ss1,ss2])+b*k4(ttdist[ss1,ss2])+c)*vyt*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
+                            dkYdlent[ss1,ss2] = vyt*vyx*dkyydlyt(tdist[ss1,ss2],xdist[ss1,ss2])* (-a*k2(xdist[ss1,ss2])+b*k4(ttdist[ss1,ss2])+c)+\
+                            vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])*b*dk4dlyt(ttdist[ss1,ss2])
+                            dkYdlenx[ss1,ss2] = vyt*vyx*dkyydlyx(tdist[ss1,ss2],xdist[ss1,ss2])*(-a*k2(xdist[ss1,ss2])+b*k4(ttdist[ss1,ss2])+c)+\
+                            vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])*(-a*dk2dlyx(xdist[ss1,ss2]))
+                        elif i==1 and j==1:
+                            dka[ss1,ss2] = (2*a*k3(xdist[ss1,ss2]) - 2*c*k2(xdist[ss1,ss2]))*vyt*vyx* kyy(tdist[ss1,ss2],xdist[ss1,ss2])
+                            dkb[ss1,ss2] = 2*b*k1(tdist[ss1,ss2])*vyt*vyx* kyy(tdist[ss1,ss2],xdist[ss1,ss2])
+                            dkc[ss1,ss2] = (-2*a*k2(xdist[ss1,ss2]) + 2*c )*vyt*vyx* kyy(tdist[ss1,ss2],xdist[ss1,ss2])
+                            dkYdvart[ss1,ss2] = ( b**2*k1(tdist[ss1,ss2]) - 2*a*c*k2(xdist[ss1,ss2]) + a**2*k3(xdist[ss1,ss2]) + c**2 )*vyx* kyy(tdist[ss1,ss2],xdist[ss1,ss2])
+                            dkYdvarx[ss1,ss2] = ( b**2*k1(tdist[ss1,ss2]) - 2*a*c*k2(xdist[ss1,ss2]) + a**2*k3(xdist[ss1,ss2]) + c**2 )*vyt* kyy(tdist[ss1,ss2],xdist[ss1,ss2])
+                            dkYdlent[ss1,ss2] = vyt*vyx*dkyydlyt(tdist[ss1,ss2],xdist[ss1,ss2])*( b**2*k1(tdist[ss1,ss2]) - 2*a*c*k2(xdist[ss1,ss2]) + a**2*k3(xdist[ss1,ss2]) + c**2 ) +\
+                            vyx*vyt*kyy(tdist[ss1,ss2],xdist[ss1,ss2])*b**2*dk1dlyt(tdist[ss1,ss2])
+                            dkYdlenx[ss1,ss2] = vyt*vyx*dkyydlyx(tdist[ss1,ss2],xdist[ss1,ss2])*( b**2*k1(tdist[ss1,ss2]) - 2*a*c*k2(xdist[ss1,ss2]) + a**2*k3(xdist[ss1,ss2]) + c**2 ) +\
+                            vyx*vyt*kyy(tdist[ss1,ss2],xdist[ss1,ss2])* (-2*a*c*dk2dlyx(xdist[ss1,ss2]) + a**2*dk3dlyx(xdist[ss1,ss2]) )
+                        else:
+                            dka[ss1,ss2] = -k2(xdist[ss1,ss2])*vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
+                            dkb[ss1,ss2] = -k4(ttdist[ss1,ss2])*vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
+                            dkc[ss1,ss2] = vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
+                            #dkYdvart[ss1,ss2] = 0
+                            #dkYdvarx[ss1,ss2] = 0
+                            #dkYdlent[ss1,ss2] = 0
+                            #dkYdlenx[ss1,ss2] = 0
+                            dkYdvart[ss1,ss2] = (-a*k2(xdist[ss1,ss2])-b*k4(ttdist[ss1,ss2])+c)*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
+                            dkYdvarx[ss1,ss2] = (-a*k2(xdist[ss1,ss2])-b*k4(ttdist[ss1,ss2])+c)*vyt*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
+                            dkYdlent[ss1,ss2] = vyt*vyx*dkyydlyt(tdist[ss1,ss2],xdist[ss1,ss2])* (-a*k2(xdist[ss1,ss2])-b*k4(ttdist[ss1,ss2])+c)+\
+                            vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])*(-1)*b*dk4dlyt(ttdist[ss1,ss2])
+                            dkYdlenx[ss1,ss2] = vyt*vyx*dkyydlyx(tdist[ss1,ss2],xdist[ss1,ss2])*(-a*k2(xdist[ss1,ss2])-b*k4(ttdist[ss1,ss2])+c)+\
+                            vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])*(-a*dk2dlyx(xdist[ss1,ss2])) 
+
+        self.a.gradient = np.sum(dka * dL_dK)  
+
+        self.b.gradient = np.sum(dkb * dL_dK) 
+
+        self.c.gradient = np.sum(dkc * dL_dK)
+
+
+        self.variance_Yt.gradient = np.sum(dkYdvart * dL_dK)  # Vy
+
+        self.variance_Yx.gradient = np.sum(dkYdvarx * dL_dK)
+
+        self.lengthscale_Yt.gradient = np.sum(dkYdlent*(-0.5*self.lengthscale_Yt**(-2)) * dL_dK)    #ly np.sum(dktheta2*(-self.lengthscale_Y**(-2)) * dL_dK) 
+
+        self.lengthscale_Yx.gradient =  np.sum(dkYdlenx*(-0.5*self.lengthscale_Yx**(-2)) * dL_dK)
+

GPy/kern/_src/ODE_t.py

@@ -0,0 +1,165 @@
+from kern import Kern
+from ...core.parameterization import Param
+from ...core.parameterization.transformations import Logexp
+import numpy as np
+from independent_outputs import index_to_slices
+
+
+class ODE_t(Kern):
+
+        def __init__(self, input_dim, a=1., c=1.,variance_Yt=3., lengthscale_Yt=1.5,ubias =1., active_dims=None, name='ode_st'):
+                assert input_dim ==2, "only defined for 2 input dims"
+                super(ODE_t, self).__init__(input_dim, active_dims, name)
+
+                self.variance_Yt = Param('variance_Yt', variance_Yt, Logexp())
+                self.lengthscale_Yt = Param('lengthscale_Yt', lengthscale_Yt, Logexp())        
+
+                self.a= Param('a', a, Logexp())
+                self.c = Param('c', c, Logexp())
+                self.ubias = Param('ubias', ubias, Logexp())
+                self.add_parameters(self.a, self.c, self.variance_Yt, self.lengthscale_Yt,self.ubias)
+
+        def K(self, X, X2=None):
+                """Compute the covariance matrix between X and X2."""        
+                X,slices = X[:,:-1],index_to_slices(X[:,-1])
+                if X2 is None:
+                        X2,slices2 = X,slices
+                        K = np.zeros((X.shape[0], X.shape[0]))
+                else:
+                        X2,slices2 = X2[:,:-1],index_to_slices(X2[:,-1])
+                        K = np.zeros((X.shape[0], X2.shape[0]))
+
+                tdist = (X[:,0][:,None] - X2[:,0][None,:])**2
+                ttdist = (X[:,0][:,None] - X2[:,0][None,:])
+                
+                vyt = self.variance_Yt
+                
+                lyt=1/(2*self.lengthscale_Yt)
+
+                a = -self.a
+                c = self.c
+
+                kyy = lambda tdist: np.exp(-lyt*(tdist))
+
+                k1 = lambda tdist: (2*lyt - 4*lyt**2 *(tdist) )
+
+                k4 = lambda tdist: 2*lyt*(tdist)
+
+                for i, s1 in enumerate(slices):
+                        for j, s2 in enumerate(slices2):
+                                for ss1 in s1:
+                                    for ss2 in s2:
+                                        if i==0 and j==0:
+                                            K[ss1,ss2] = vyt*kyy(tdist[ss1,ss2])
+                                        elif i==0 and j==1:
+                                            K[ss1,ss2] = (k4(ttdist[ss1,ss2])+1)*vyt*kyy(tdist[ss1,ss2])
+                                            #K[ss1,ss2] = (2*lyt*(ttdist[ss1,ss2])+1)*vyt*kyy(tdist[ss1,ss2])
+                                        elif i==1 and j==1:
+                                            K[ss1,ss2] = ( k1(tdist[ss1,ss2]) + 1. )*vyt* kyy(tdist[ss1,ss2])+self.ubias
+                                        else:
+                                            K[ss1,ss2] = (-k4(ttdist[ss1,ss2])+1)*vyt*kyy(tdist[ss1,ss2])
+                                            #K[ss1,ss2] = (-2*lyt*(ttdist[ss1,ss2])+1)*vyt*kyy(tdist[ss1,ss2])
+                #stop
+                return K
+
+
+        def Kdiag(self, X):
+
+                vyt = self.variance_Yt
+                lyt = 1./(2*self.lengthscale_Yt)
+
+                a = -self.a
+                c = self.c        
+                
+                k1 = (2*lyt )*vyt
+                
+                Kdiag = np.zeros(X.shape[0])
+                slices = index_to_slices(X[:,-1])
+
+                for i, ss1 in enumerate(slices):
+                    for s1 in ss1:
+                        if i==0:
+                            Kdiag[s1]+= vyt
+                        elif i==1:
+                            #i=1
+                            Kdiag[s1]+= k1 + vyt+self.ubias
+                            #Kdiag[s1]+= Vu*Vy*(k1+k2+k3)
+                        else:
+                            raise ValueError, "invalid input/output index"
+
+                return Kdiag
+
+        def update_gradients_full(self, dL_dK, X, X2=None):
+                """derivative of the covariance matrix with respect to the parameters."""
+                X,slices = X[:,:-1],index_to_slices(X[:,-1])
+                if X2 is None:
+                    X2,slices2 = X,slices
+                    K = np.zeros((X.shape[0], X.shape[0]))
+                else:
+                    X2,slices2 = X2[:,:-1],index_to_slices(X2[:,-1])
+
+
+                vyt = self.variance_Yt
+
+                lyt = 1./(2*self.lengthscale_Yt)
+
+                tdist = (X[:,0][:,None] - X2[:,0][None,:])**2
+                ttdist = (X[:,0][:,None] - X2[:,0][None,:])
+                #rdist = [tdist,xdist]
+                
+                rd=tdist.shape[0]
+
+                dka = np.zeros([rd,rd])
+                dkc = np.zeros([rd,rd])
+                dkYdvart = np.zeros([rd,rd])
+                dkYdlent = np.zeros([rd,rd])
+
+                dkdubias = np.zeros([rd,rd])
+
+                kyy = lambda tdist: np.exp(-lyt*(tdist))
+                dkyydlyt = lambda tdist: kyy(tdist)*(-tdist)
+
+                k1 = lambda tdist: (2*lyt - 4*lyt**2 * (tdist) )
+
+                k4 = lambda ttdist: 2*lyt*(ttdist)
+
+                dk1dlyt = lambda tdist: 2. - 4*2.*lyt*tdist
+
+                dk4dlyt = lambda ttdist: 2*(ttdist)
+
+                for i, s1 in enumerate(slices):
+                    for j, s2 in enumerate(slices2):
+                        for ss1 in s1:
+                            for ss2 in s2:
+                                if i==0 and j==0:
+                                    dkYdvart[ss1,ss2] = kyy(tdist[ss1,ss2])
+                                    dkYdlent[ss1,ss2] = vyt*dkyydlyt(tdist[ss1,ss2])
+                                    dkdubias[ss1,ss2] = 0
+                                elif i==0 and j==1:
+                                    dkYdvart[ss1,ss2] = (k4(ttdist[ss1,ss2])+1)*kyy(tdist[ss1,ss2])
+                                    #dkYdvart[ss1,ss2] = ((2*lyt*ttdist[ss1,ss2])+1)*kyy(tdist[ss1,ss2])
+                                    dkYdlent[ss1,ss2] = vyt*dkyydlyt(tdist[ss1,ss2])* (k4(ttdist[ss1,ss2])+1.)+\
+                                    vyt*kyy(tdist[ss1,ss2])*(dk4dlyt(ttdist[ss1,ss2]))
+                                    #dkYdlent[ss1,ss2] = vyt*dkyydlyt(tdist[ss1,ss2])* (2*lyt*(ttdist[ss1,ss2])+1.)+\
+                                    #vyt*kyy(tdist[ss1,ss2])*(2*ttdist[ss1,ss2])
+                                    dkdubias[ss1,ss2] = 0
+                                elif i==1 and j==1:
+                                    dkYdvart[ss1,ss2] = (k1(tdist[ss1,ss2]) + 1. )* kyy(tdist[ss1,ss2])
+                                    dkYdlent[ss1,ss2] = vyt*dkyydlyt(tdist[ss1,ss2])*( k1(tdist[ss1,ss2]) + 1. ) +\
+                          			vyt*kyy(tdist[ss1,ss2])*dk1dlyt(tdist[ss1,ss2])
+                                    dkdubias[ss1,ss2] = 1
+                                else:
+                                    dkYdvart[ss1,ss2] = (-k4(ttdist[ss1,ss2])+1)*kyy(tdist[ss1,ss2])
+                                    #dkYdvart[ss1,ss2] = (-2*lyt*(ttdist[ss1,ss2])+1)*kyy(tdist[ss1,ss2])
+                                    dkYdlent[ss1,ss2] = vyt*dkyydlyt(tdist[ss1,ss2])* (-k4(ttdist[ss1,ss2])+1.)+\
+                                    vyt*kyy(tdist[ss1,ss2])*(-dk4dlyt(ttdist[ss1,ss2]) )
+                                    dkdubias[ss1,ss2] = 0
+                                    #dkYdlent[ss1,ss2] = vyt*dkyydlyt(tdist[ss1,ss2])* (-2*lyt*(ttdist[ss1,ss2])+1.)+\
+                                    #vyt*kyy(tdist[ss1,ss2])*(-2)*(ttdist[ss1,ss2])
+   
+
+                self.variance_Yt.gradient = np.sum(dkYdvart * dL_dK)
+
+                self.lengthscale_Yt.gradient =  np.sum(dkYdlent*(-0.5*self.lengthscale_Yt**(-2)) * dL_dK)
+
+                self.ubias.gradient = np.sum(dkdubias * dL_dK) 

GPy/kern/_src/add.py

@@ -14,6 +14,13 @@ class Add(CombinationKernel):
     This kernel will take over the active dims of it's subkernels passed in.
     """
     def __init__(self, subkerns, name='add'):
+        for i, kern in enumerate(subkerns[:]):
+            if isinstance(kern, Add):
+                del subkerns[i]
+                for part in kern.parts[::-1]:
+                    kern.remove_parameter(part)
+                    subkerns.insert(i, part)
+
         super(Add, self).__init__(subkerns, name)
 
     @Cache_this(limit=2, force_kwargs=['which_parts'])
@@ -40,7 +47,7 @@ class Add(CombinationKernel):
         return reduce(np.add, (p.Kdiag(X) for p in which_parts))
 
     def update_gradients_full(self, dL_dK, X, X2=None):
-        [p.update_gradients_full(dL_dK, X, X2) for p in self.parts]
+        [p.update_gradients_full(dL_dK, X, X2) for p in self.parts if not p.is_fixed]
 
     def update_gradients_diag(self, dL_dK, X):
         [p.update_gradients_diag(dL_dK, X) for p in self.parts]
@@ -64,12 +71,15 @@ class Add(CombinationKernel):
         [target.__iadd__(p.gradients_X_diag(dL_dKdiag, X)) for p in self.parts]
         return target
     
+    @Cache_this(limit=2, force_kwargs=['which_parts'])
     def psi0(self, Z, variational_posterior):
         return reduce(np.add, (p.psi0(Z, variational_posterior) for p in self.parts))
     
+    @Cache_this(limit=2, force_kwargs=['which_parts'])
     def psi1(self, Z, variational_posterior):
         return reduce(np.add, (p.psi1(Z, variational_posterior) for p in self.parts))
 
+    @Cache_this(limit=2, force_kwargs=['which_parts'])
     def psi2(self, Z, variational_posterior):
         psi2 = reduce(np.add, (p.psi2(Z, variational_posterior) for p in self.parts))
         #return psi2
@@ -88,17 +98,18 @@ class Add(CombinationKernel):
             # rbf X bias
             #elif isinstance(p1, (Bias, Fixed)) and isinstance(p2, (RBF, RBFInv)):
             elif isinstance(p1,  Bias) and isinstance(p2, (RBF, Linear)):
-                tmp = p2.psi1(Z, variational_posterior)
+                tmp = p2.psi1(Z, variational_posterior).sum(axis=0)
-                psi2 += p1.variance * (tmp[:, :, None] + tmp[:, None, :])
+                psi2 += p1.variance * (tmp[:,None]+tmp[None,:]) #(tmp[:, :, None] + tmp[:, None, :])
             #elif isinstance(p2, (Bias, Fixed)) and isinstance(p1, (RBF, RBFInv)):
             elif isinstance(p2, Bias) and isinstance(p1, (RBF, Linear)):
-                tmp = p1.psi1(Z, variational_posterior)
+                tmp = p1.psi1(Z, variational_posterior).sum(axis=0)
-                psi2 += p2.variance * (tmp[:, :, None] + tmp[:, None, :])
+                psi2 += p2.variance * (tmp[:,None]+tmp[None,:]) #(tmp[:, :, None] + tmp[:, None, :])
             elif isinstance(p2, (RBF, Linear)) and isinstance(p1, (RBF, Linear)):
                 assert np.intersect1d(p1.active_dims, p2.active_dims).size == 0, "only non overlapping kernel dimensions allowed so far"
                 tmp1 = p1.psi1(Z, variational_posterior)
                 tmp2 = p2.psi1(Z, variational_posterior)
-                psi2 += (tmp1[:, :, None] * tmp2[:, None, :]) + (tmp2[:, :, None] * tmp1[:, None, :])
+                psi2 += np.einsum('nm,no->mo',tmp1,tmp2)+np.einsum('nm,no->mo',tmp2,tmp1)
+                #(tmp1[:, :, None] * tmp2[:, None, :]) + (tmp2[:, :, None] * tmp1[:, None, :])
             else:
                 raise NotImplementedError, "psi2 cannot be computed for this kernel"
         return psi2
@@ -114,12 +125,12 @@ class Add(CombinationKernel):
                 if isinstance(p2, White):
                     continue
                 elif isinstance(p2, Bias):
-                    eff_dL_dpsi1 += dL_dpsi2.sum(1) * p2.variance * 2.
+                    eff_dL_dpsi1 += dL_dpsi2.sum(0) * p2.variance * 2.
                 else:# np.setdiff1d(p1.active_dims, ar2, assume_unique): # TODO: Careful, not correct for overlapping active_dims
-                    eff_dL_dpsi1 += dL_dpsi2.sum(1) * p2.psi1(Z, variational_posterior) * 2.
+                    eff_dL_dpsi1 += dL_dpsi2.sum(0) * p2.psi1(Z, variational_posterior) * 2.
             p1.update_gradients_expectations(dL_dpsi0, eff_dL_dpsi1, dL_dpsi2, Z, variational_posterior)
 
-    def gradients_Z_expectations(self, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
+    def gradients_Z_expectations(self, dL_psi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
         from static import White, Bias
         target = np.zeros(Z.shape)
         for p1 in self.parts:
@@ -131,36 +142,34 @@ class Add(CombinationKernel):
                 if isinstance(p2, White):
                     continue
                 elif isinstance(p2, Bias):
-                    eff_dL_dpsi1 += dL_dpsi2.sum(1) * p2.variance * 2.
+                    eff_dL_dpsi1 += dL_dpsi2.sum(0) * p2.variance * 2.
                 else:
-                    eff_dL_dpsi1 += dL_dpsi2.sum(1) * p2.psi1(Z, variational_posterior) * 2.
+                    eff_dL_dpsi1 += dL_dpsi2.sum(0) * p2.psi1(Z, variational_posterior) * 2.
-            target += p1.gradients_Z_expectations(eff_dL_dpsi1, dL_dpsi2, Z, variational_posterior)
+            target += p1.gradients_Z_expectations(dL_psi0, eff_dL_dpsi1, dL_dpsi2, Z, variational_posterior)
         return target
 
     def gradients_qX_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
         from static import White, Bias
-        target_mu = np.zeros(variational_posterior.shape)
+        target_grads = [np.zeros(v.shape) for v in variational_posterior.parameters]
-        target_S = np.zeros(variational_posterior.shape)
+        for p1 in self.parameters:
-        for p1 in self._parameters_:
             #compute the effective dL_dpsi1. extra terms appear becaue of the cross terms in psi2!
             eff_dL_dpsi1 = dL_dpsi1.copy()
-            for p2 in self._parameters_:
+            for p2 in self.parameters:
                 if p2 is p1:
                     continue
                 if isinstance(p2, White):
                     continue
                 elif isinstance(p2, Bias):
-                    eff_dL_dpsi1 += dL_dpsi2.sum(1) * p2.variance * 2.
+                    eff_dL_dpsi1 += dL_dpsi2.sum(0) * p2.variance * 2.
                 else:
-                    eff_dL_dpsi1 += dL_dpsi2.sum(1) * p2.psi1(Z, variational_posterior) * 2.
+                    eff_dL_dpsi1 += dL_dpsi2.sum(0) * p2.psi1(Z, variational_posterior) * 2.
-            a, b = p1.gradients_qX_expectations(dL_dpsi0, eff_dL_dpsi1, dL_dpsi2, Z, variational_posterior)
+            grads = p1.gradients_qX_expectations(dL_dpsi0, eff_dL_dpsi1, dL_dpsi2, Z, variational_posterior)
-            target_mu += a
+            [np.add(target_grads[i],grads[i],target_grads[i]) for i in xrange(len(grads))]
-            target_S += b
+        return target_grads
-        return target_mu, target_S
 
-    def add(self, other, name='sum'):
+    def add(self, other):
         if isinstance(other, Add):
-            other_params = other._parameters_[:]
+            other_params = other.parameters[:]
             for p in other_params:
                 other.remove_parameter(p)
             self.add_parameters(*other_params)
@@ -169,8 +178,11 @@ class Add(CombinationKernel):
         self.input_dim, self.active_dims = self.get_input_dim_active_dims(self.parts)
         return self
 
-    def input_sensitivity(self):
+    def input_sensitivity(self, summarize=True):
-        in_sen = np.zeros(self.input_dim)
+        if summarize:
-        for i, p in enumerate(self.parts):
+            return reduce(np.add, [k.input_sensitivity(summarize) for k in self.parts])
-            in_sen[p.active_dims] += p.input_sensitivity()
+        else:
-        return in_sen
+            i_s = np.zeros((len(self.parts), self.input_dim))
+            from operator import setitem
+            [setitem(i_s, (i, Ellipsis), k.input_sensitivity(summarize)) for i, k in enumerate(self.parts)]
+            return i_s

GPy/kern/_src/independent_outputs.py

@@ -20,9 +20,11 @@ def index_to_slices(index):
     returns
     >>> [[slice(0,2,None),slice(4,5,None)],[slice(2,4,None),slice(8,10,None)],[slice(5,8,None)]]
     """
+    if len(index)==0:
+        return[]
 
     #contruct the return structure
-    ind = np.asarray(index,dtype=np.int64)
+    ind = np.asarray(index,dtype=np.int)
     ret = [[] for i in range(ind.max()+1)]
 
     #find the switchpoints
@@ -32,7 +34,7 @@ def index_to_slices(index):
     [ret[ind_i].append(slice(*indexes_i)) for ind_i,indexes_i in zip(ind[switchpoints[:-1]],zip(switchpoints,switchpoints[1:]))]
     return ret
 
-class IndependentOutputs(Kern):
+class IndependentOutputs(CombinationKernel):
     """
     A kernel which can represent several independent functions.  this kernel
     'switches off' parts of the matrix where the output indexes are different.
@@ -180,6 +182,9 @@ class Hierarchical(CombinationKernel):
     def Kdiag(self,X):
         return np.diag(self.K(X))
 
+    def gradients_X(self, dL_dK, X, X2=None):
+        raise NotImplementedError
+
     def update_gradients_full(self,dL_dK,X,X2=None):
         slices = [index_to_slices(X[:,i]) for i in self.extra_dims]
         if X2 is None:

GPy/kern/_src/kern.py

@@ -34,40 +34,28 @@ class Kern(Parameterized):
 
             is the active_dimensions of inputs X we will work on.
             All kernels will get sliced Xes as inputs, if active_dims is not None
+            Only positive integers are allowed in active_dims!
             if active_dims is None, slicing is switched off and all X will be passed through as given.
 
         :param int input_dim: the number of input dimensions to the function
-        :param array-like|slice|None active_dims: list of indices on which dimensions this kernel works on, or none if no slicing
+        :param array-like|None active_dims: list of indices on which dimensions this kernel works on, or none if no slicing
 
         Do not instantiate.
         """
         super(Kern, self).__init__(name=name, *a, **kw)
-        try:
         self.input_dim = int(input_dim)
-            self.active_dims = active_dims# if active_dims is not None else slice(0, input_dim, 1)
-        except TypeError:
-            # input_dim is something else then an integer
-            self.input_dim = input_dim
-            if active_dims is not None:
-                print "WARNING: given input_dim={} is not an integer and active_dims={} is given, switching off slicing"
-            self.active_dims = None
 
-        if self.active_dims is not None and self.input_dim is not None:
+        if active_dims is None:
-            assert isinstance(self.active_dims, (slice, list, tuple, np.ndarray)), 'active_dims needs to be an array-like or slice object over dimensions, {} given'.format(self.active_dims.__class__)
+            active_dims = np.arange(input_dim)
-            if isinstance(self.active_dims, slice):
+
-                self.active_dims = slice(self.active_dims.start or 0, self.active_dims.stop or self.input_dim, self.active_dims.step or 1)
+        self.active_dims = np.array(active_dims, dtype=int)
-                active_dim_size = int(np.round((self.active_dims.stop-self.active_dims.start)/self.active_dims.step))
+
-            elif isinstance(self.active_dims, np.ndarray):
+        assert self.active_dims.size == self.input_dim, "input_dim={} does not match len(active_dim)={}, active_dims={}".format(self.input_dim, self.active_dims.size, self.active_dims)
-                #assert np.all(self.active_dims >= 0), 'active dimensions need to be positive. negative indexing is not allowed'
+
-                assert self.active_dims.ndim == 1, 'only flat indices allowed, given active_dims.shape={}, provide only indexes to the dimensions (columns) of the input'.format(self.active_dims.shape)
-                active_dim_size = self.active_dims.size
-            else:
-                active_dim_size = len(self.active_dims)
-            assert active_dim_size == self.input_dim, "input_dim={} does not match len(active_dim)={}, active_dims={}".format(self.input_dim, active_dim_size, self.active_dims)
         self._sliced_X = 0
         self.useGPU = self._support_GPU and useGPU
 
-    @Cache_this(limit=10)
+    @Cache_this(limit=20)
     def _slice_X(self, X):
         return X[:, self.active_dims]
 
@@ -115,7 +103,7 @@ class Kern(Parameterized):
         """
         raise NotImplementedError
 
-    def gradients_Z_expectations(self, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
+    def gradients_Z_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
         """
         Returns the derivative of the objective wrt Z, using the chain rule
         through the expectation variables.
@@ -146,7 +134,7 @@ class Kern(Parameterized):
         from ...plotting.matplot_dep import kernel_plots
         return kernel_plots.plot_ARD(self,*args,**kw)
 
-    def input_sensitivity(self):
+    def input_sensitivity(self, summarize=True):
         """
         Returns the sensitivity for each dimension of this kernel.
         """
@@ -156,6 +144,9 @@ class Kern(Parameterized):
         """ Overloading of the '+' operator. for more control, see self.add """
         return self.add(other)
 
+    def __iadd__(self, other):
+        return self.add(other)
+
     def add(self, other, name='add'):
         """
         Add another kernel to this one.
@@ -176,8 +167,8 @@ class Kern(Parameterized):
         """
         Shortcut for tensor `prod`.
         """
-        assert self.active_dims == range(self.input_dim), "Can only use kernels, which have their input_dims defined from 0"
+        assert np.all(self.active_dims == range(self.input_dim)), "Can only use kernels, which have their input_dims defined from 0"
-        assert other.active_dims == range(other.input_dim), "Can only use kernels, which have their input_dims defined from 0"
+        assert np.all(other.active_dims == range(other.input_dim)), "Can only use kernels, which have their input_dims defined from 0"
         other.active_dims += self.input_dim
         return self.prod(other)
 
@@ -195,17 +186,17 @@ class Kern(Parameterized):
         assert isinstance(other, Kern), "only kernels can be added to kernels..."
         from prod import Prod
         #kernels = []
-        #if isinstance(self, Prod): kernels.extend(self._parameters_)
+        #if isinstance(self, Prod): kernels.extend(self.parameters)
         #else: kernels.append(self)
-        #if isinstance(other, Prod): kernels.extend(other._parameters_)
+        #if isinstance(other, Prod): kernels.extend(other.parameters)
         #else: kernels.append(other)
         return Prod([self, other], name)
 
     def _check_input_dim(self, X):
-        assert X.shape[1] == self.input_dim, "You did not specify active_dims and X has wrong shape: X_dim={}, whereas input_dim={}".format(X.shape[1], self.input_dim)
+        assert X.shape[1] == self.input_dim, "{} did not specify active_dims and X has wrong shape: X_dim={}, whereas input_dim={}".format(self.name, X.shape[1], self.input_dim)
 
     def _check_active_dims(self, X):
-        assert X.shape[1] >= len(np.r_[self.active_dims]), "At least {} dimensional X needed, X.shape={!s}".format(len(np.r_[self.active_dims]), X.shape)
+        assert X.shape[1] >= len(self.active_dims), "At least {} dimensional X needed, X.shape={!s}".format(len(self.active_dims), X.shape)
 
 
 class CombinationKernel(Kern):
@@ -222,9 +213,10 @@ class CombinationKernel(Kern):
 
         :param list kernels: List of kernels to combine (can be only one element)
         :param str name: name of the combination kernel
-        :param array-like|slice extra_dims: if needed extra dimensions for the combination kernel to work on
+        :param array-like extra_dims: if needed extra dimensions for the combination kernel to work on
         """
         assert all([isinstance(k, Kern) for k in kernels])
+        extra_dims = np.array(extra_dims, dtype=int)
         input_dim, active_dims = self.get_input_dim_active_dims(kernels, extra_dims)
         # initialize the kernel with the full input_dim
         super(CombinationKernel, self).__init__(input_dim, active_dims, name)
@@ -233,21 +225,28 @@ class CombinationKernel(Kern):
 
     @property
     def parts(self):
-        return self._parameters_
+        return self.parameters
 
     def get_input_dim_active_dims(self, kernels, extra_dims = None):
         #active_dims = reduce(np.union1d, (np.r_[x.active_dims] for x in kernels), np.array([], dtype=int))
         #active_dims = np.array(np.concatenate((active_dims, extra_dims if extra_dims is not None else [])), dtype=int)
-        input_dim = np.array([k.input_dim for k in kernels])
+        input_dim = reduce(max, (k.active_dims.max() for k in kernels)) + 1
-        if np.all(input_dim[0]==input_dim):
+
-            input_dim = input_dim[0]
+        if extra_dims is not None:
-        active_dims = None
+            input_dim += extra_dims.size
+
+        active_dims = np.arange(input_dim)
         return input_dim, active_dims
 
-    def input_sensitivity(self):
+    def input_sensitivity(self, summarize=True):
+        """
+        If summize is true, we want to get the summerized view of the sensitivities,
+        otherwise put everything into an array with shape (#kernels, input_dim)
+        in the order of appearance of the kernels in the parameterized object.
+        """
         raise NotImplementedError("Choose the kernel you want to get the sensitivity for. You need to override the default behaviour for getting the input sensitivity to be able to get the input sensitivity. For sum kernel it is the sum of all sensitivities, TODO: product kernel? Other kernels?, also TODO: shall we return all the sensitivities here in the combination kernel? So we can combine them however we want? This could lead to just plot all the sensitivities here...")
 
-    def _check_input_dim(self, X):
+    def _check_active_dims(self, X):
         return
 
     def _check_input_dim(self, X):

GPy/kern/_src/kernel_slice_operations.py

@@ -124,9 +124,9 @@ def _slice_update_gradients_expectations(f):
 
 def _slice_gradients_Z_expectations(f):
     @wraps(f)
-    def wrap(self, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
+    def wrap(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
         with _Slice_wrap(self, Z, variational_posterior) as s:
-            ret = s.handle_return_array(f(self, dL_dpsi1, dL_dpsi2, s.X, s.X2))
+            ret = s.handle_return_array(f(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, s.X, s.X2))
         return ret
     return wrap
 

GPy/kern/_src/linear.py

@@ -3,15 +3,13 @@
 
 
 import numpy as np
-from scipy import weave
 from kern import Kern
 from ...util.linalg import tdot
-from ...util.misc import param_to_array
 from ...core.parameterization import Param
 from ...core.parameterization.transformations import Logexp
 from ...util.caching import Cache_this
-from ...core.parameterization import variational
+from ...util.config import *
-from psi_comp import linear_psi_comp
+from .psi_comp import PSICOMP_Linear
 
 class Linear(Kern):
     """
@@ -52,6 +50,7 @@ class Linear(Kern):
 
         self.variances = Param('variances', variances, Logexp())
         self.add_parameter(self.variances)
+        self.psicomp = PSICOMP_Linear()
 
     @Cache_this(limit=2)
     def K(self, X, X2=None):
@@ -77,10 +76,12 @@ class Linear(Kern):
     def update_gradients_full(self, dL_dK, X, X2=None):
         if self.ARD:
             if X2 is None:
-                self.variances.gradient = np.array([np.sum(dL_dK * tdot(X[:, i:i + 1])) for i in range(self.input_dim)])
+                #self.variances.gradient = np.array([np.sum(dL_dK * tdot(X[:, i:i + 1])) for i in range(self.input_dim)])
+                self.variances.gradient = np.einsum('ij,iq,jq->q', dL_dK, X, X)
             else:
-                product = X[:, None, :] * X2[None, :, :]
+                #product = X[:, None, :] * X2[None, :, :]
-                self.variances.gradient = (dL_dK[:, :, None] * product).sum(0).sum(0)
+                #self.variances.gradient = (dL_dK[:, :, None] * product).sum(0).sum(0)
+                self.variances.gradient = np.einsum('ij,iq,jq->q', dL_dK, X, X2)
         else:
             self.variances.gradient = np.sum(self._dot_product(X, X2) * dL_dK)
 
@@ -94,225 +95,42 @@ class Linear(Kern):
 
     def gradients_X(self, dL_dK, X, X2=None):
         if X2 is None:
-            return 2.*(((X[None,:, :] * self.variances)) * dL_dK[:, :, None]).sum(1)
+            return np.einsum('jq,q,ij->iq', X, 2*self.variances, dL_dK)
         else:
-            return (((X2[None,:, :] * self.variances)) * dL_dK[:, :, None]).sum(1)
+            #return (((X2[None,:, :] * self.variances)) * dL_dK[:, :, None]).sum(1)
+            return np.einsum('jq,q,ij->iq', X2, self.variances, dL_dK)
 
     def gradients_X_diag(self, dL_dKdiag, X):
         return 2.*self.variances*dL_dKdiag[:,None]*X
 
+    def input_sensitivity(self):
+        return np.ones(self.input_dim) * self.variances
+
     #---------------------------------------#
     #             PSI statistics            #
     #---------------------------------------#
 
     def psi0(self, Z, variational_posterior):
-        if isinstance(variational_posterior, variational.SpikeAndSlabPosterior):
+        return self.psicomp.psicomputations(self.variances, Z, variational_posterior)[0]
-            gamma = variational_posterior.binary_prob
-            mu = variational_posterior.mean
-            S = variational_posterior.variance
-
-            return np.einsum('q,nq,nq->n',self.variances,gamma,np.square(mu)+S)
-#            return (self.variances*gamma*(np.square(mu)+S)).sum(axis=1)
-        else:
-            return np.sum(self.variances * self._mu2S(variational_posterior), 1)
 
     def psi1(self, Z, variational_posterior):
-        if isinstance(variational_posterior, variational.SpikeAndSlabPosterior):
+        return self.psicomp.psicomputations(self.variances, Z, variational_posterior)[1]
-            gamma = variational_posterior.binary_prob
-            mu = variational_posterior.mean
-            return np.einsum('nq,q,mq,nq->nm',gamma,self.variances,Z,mu)
-#            return (self.variances*gamma*mu).sum(axis=1)
-        else:
-            return self.K(variational_posterior.mean, Z) #the variance, it does nothing
 
-    @Cache_this(limit=1)
     def psi2(self, Z, variational_posterior):
-        if isinstance(variational_posterior, variational.SpikeAndSlabPosterior):
+        return self.psicomp.psicomputations(self.variances, Z, variational_posterior)[2]
-            gamma = variational_posterior.binary_prob
-            mu = variational_posterior.mean
-            S = variational_posterior.variance
-            mu2 = np.square(mu)
-            variances2 = np.square(self.variances)
-            tmp = np.einsum('nq,q,mq,nq->nm',gamma,self.variances,Z,mu)
-            return np.einsum('nq,q,mq,oq,nq->nmo',gamma,variances2,Z,Z,mu2+S)+\
-                np.einsum('nm,no->nmo',tmp,tmp) - np.einsum('nq,q,mq,oq,nq->nmo',np.square(gamma),variances2,Z,Z,mu2)
-        else:
-            ZA = Z * self.variances
-            ZAinner = self._ZAinner(variational_posterior, Z)
-            return np.dot(ZAinner, ZA.T)
 
     def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
-        if isinstance(variational_posterior, variational.SpikeAndSlabPosterior):
+        dL_dvar = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variances, Z, variational_posterior)[0]
-            gamma = variational_posterior.binary_prob
-            mu = variational_posterior.mean
-            S = variational_posterior.variance
-            mu2S = np.square(mu)+S
-            _dpsi2_dvariance, _, _, _, _ = linear_psi_comp._psi2computations(self.variances, Z, mu, S, gamma)
-            grad = np.einsum('n,nq,nq->q',dL_dpsi0,gamma,mu2S) + np.einsum('nm,nq,mq,nq->q',dL_dpsi1,gamma,Z,mu) +\
-                 np.einsum('nmo,nmoq->q',dL_dpsi2,_dpsi2_dvariance)
         if self.ARD:
-                self.variances.gradient = grad
+            self.variances.gradient = dL_dvar
         else:
-                self.variances.gradient = grad.sum()
+            self.variances.gradient = dL_dvar.sum()
-        else:
-            #psi1
-            self.update_gradients_full(dL_dpsi1, variational_posterior.mean, Z)
-            # psi0:
-            tmp = dL_dpsi0[:, None] * self._mu2S(variational_posterior)
-            if self.ARD: self.variances.gradient += tmp.sum(0)
-            else: self.variances.gradient += tmp.sum()
-            #psi2
-            if self.ARD:
-                tmp = dL_dpsi2[:, :, :, None] * (self._ZAinner(variational_posterior, Z)[:, :, None, :] * Z[None, None, :, :])
-                self.variances.gradient += 2.*tmp.sum(0).sum(0).sum(0)
-            else:
-                self.variances.gradient += 2.*np.sum(dL_dpsi2 * self.psi2(Z, variational_posterior))/self.variances
 
-    def gradients_Z_expectations(self, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
+    def gradients_Z_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
-        if isinstance(variational_posterior, variational.SpikeAndSlabPosterior):
+        return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variances, Z, variational_posterior)[1]
-            gamma = variational_posterior.binary_prob
-            mu = variational_posterior.mean
-            S = variational_posterior.variance
-            _, _, _, _, _dpsi2_dZ = linear_psi_comp._psi2computations(self.variances, Z, mu, S, gamma)
-
-            grad =  np.einsum('nm,nq,q,nq->mq',dL_dpsi1,gamma, self.variances,mu) +\
-                 np.einsum('nmo,noq->mq',dL_dpsi2,_dpsi2_dZ)
-
-            return grad
-        else:
-            #psi1
-            grad = self.gradients_X(dL_dpsi1.T, Z, variational_posterior.mean)
-            #psi2
-            self._weave_dpsi2_dZ(dL_dpsi2, Z, variational_posterior, grad)
-            return grad
 
     def gradients_qX_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
-        if isinstance(variational_posterior, variational.SpikeAndSlabPosterior):
+        return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variances, Z, variational_posterior)[2:]
-            gamma = variational_posterior.binary_prob
-            mu = variational_posterior.mean
-            S = variational_posterior.variance
-            mu2S = np.square(mu)+S
-            _, _dpsi2_dgamma, _dpsi2_dmu, _dpsi2_dS, _ = linear_psi_comp._psi2computations(self.variances, Z, mu, S, gamma)
-
-            grad_gamma = np.einsum('n,q,nq->nq',dL_dpsi0,self.variances,mu2S) + np.einsum('nm,q,mq,nq->nq',dL_dpsi1,self.variances,Z,mu) +\
-                 np.einsum('nmo,nmoq->nq',dL_dpsi2,_dpsi2_dgamma)
-            grad_mu = np.einsum('n,nq,q,nq->nq',dL_dpsi0,gamma,2.*self.variances,mu) + np.einsum('nm,nq,q,mq->nq',dL_dpsi1,gamma,self.variances,Z) +\
-                 np.einsum('nmo,nmoq->nq',dL_dpsi2,_dpsi2_dmu)
-            grad_S = np.einsum('n,nq,q->nq',dL_dpsi0,gamma,self.variances) + np.einsum('nmo,nmoq->nq',dL_dpsi2,_dpsi2_dS)
-
-            return grad_mu, grad_S, grad_gamma
-        else:
-            grad_mu, grad_S = np.zeros(variational_posterior.mean.shape), np.zeros(variational_posterior.mean.shape)
-            # psi0
-            grad_mu += dL_dpsi0[:, None] * (2.0 * variational_posterior.mean * self.variances)
-            grad_S += dL_dpsi0[:, None] * self.variances
-            # psi1
-            grad_mu += (dL_dpsi1[:, :, None] * (Z * self.variances)).sum(1)
-            # psi2
-            self._weave_dpsi2_dmuS(dL_dpsi2, Z, variational_posterior, grad_mu, grad_S)
-
-            return grad_mu, grad_S
-
-    #--------------------------------------------------#
-    #            Helpers for psi statistics            #
-    #--------------------------------------------------#
-
-
-    def _weave_dpsi2_dmuS(self, dL_dpsi2, Z, vp, target_mu, target_S):
-        # Think N,num_inducing,num_inducing,input_dim
-        ZA = Z * self.variances
-        AZZA = ZA.T[:, None, :, None] * ZA[None, :, None, :]
-        AZZA = AZZA + AZZA.swapaxes(1, 2)
-        AZZA_2 = AZZA/2.
-
-        #Using weave, we can exploit the symmetry of this problem:
-        code = """
-        int n, m, mm,q,qq;
-        double factor,tmp;
-        #pragma omp parallel for private(m,mm,q,qq,factor,tmp)
-        for(n=0;n<N;n++){
-          for(m=0;m<num_inducing;m++){
-            for(mm=0;mm<=m;mm++){
-              //add in a factor of 2 for the off-diagonal terms (and then count them only once)
-              if(m==mm)
-                factor = dL_dpsi2(n,m,mm);
-              else
-                factor = 2.0*dL_dpsi2(n,m,mm);
-
-              for(q=0;q<input_dim;q++){
-
-                //take the dot product of mu[n,:] and AZZA[:,m,mm,q] TODO: blas!
-                tmp = 0.0;
-                for(qq=0;qq<input_dim;qq++){
-                  tmp += mu(n,qq)*AZZA(qq,m,mm,q);
-                }
-
-                target_mu(n,q) += factor*tmp;
-                target_S(n,q) += factor*AZZA_2(q,m,mm,q);
-              }
-            }
-          }
-        }
-        """
-        support_code = """
-        #include <omp.h>
-        #include <math.h>
-        """
-        weave_options = {'headers'           : ['<omp.h>'],
-                         'extra_compile_args': ['-fopenmp -O3'],  #-march=native'],
-                         'extra_link_args'   : ['-lgomp']}
-        mu = vp.mean
-        N,num_inducing,input_dim,mu = mu.shape[0],Z.shape[0],mu.shape[1],param_to_array(mu)
-        weave.inline(code, support_code=support_code, libraries=['gomp'],
-                     arg_names=['N','num_inducing','input_dim','mu','AZZA','AZZA_2','target_mu','target_S','dL_dpsi2'],
-                     type_converters=weave.converters.blitz,**weave_options)
-
-
-    def _weave_dpsi2_dZ(self, dL_dpsi2, Z, vp, target):
-        AZA = self.variances*self._ZAinner(vp, Z)
-        code="""
-        int n,m,mm,q;
-        #pragma omp parallel for private(n,mm,q)
-        for(m=0;m<num_inducing;m++){
-          for(q=0;q<input_dim;q++){
-            for(mm=0;mm<num_inducing;mm++){
-              for(n=0;n<N;n++){
-                target(m,q) += 2*dL_dpsi2(n,m,mm)*AZA(n,mm,q);
-              }
-            }
-          }
-        }
-        """
-        support_code = """
-        #include <omp.h>
-        #include <math.h>
-        """
-        weave_options = {'headers'           : ['<omp.h>'],
-                         'extra_compile_args': ['-fopenmp -O3'],  #-march=native'],
-                         'extra_link_args'   : ['-lgomp']}
-
-        N,num_inducing,input_dim = vp.mean.shape[0],Z.shape[0],vp.mean.shape[1]
-        mu = param_to_array(vp.mean)
-        weave.inline(code, support_code=support_code, libraries=['gomp'],
-                     arg_names=['N','num_inducing','input_dim','AZA','target','dL_dpsi2'],
-                     type_converters=weave.converters.blitz,**weave_options)
-
-
-    @Cache_this(limit=1, ignore_args=(0,))
-    def _mu2S(self, vp):
-        return np.square(vp.mean) + vp.variance
-
-    @Cache_this(limit=1)
-    def _ZAinner(self, vp, Z):
-        ZA = Z*self.variances
-        inner = (vp.mean[:, None, :] * vp.mean[:, :, None])
-        diag_indices = np.diag_indices(vp.mean.shape[1], 2)
-        inner[:, diag_indices[0], diag_indices[1]] += vp.variance
-
-        return np.dot(ZA, inner).swapaxes(0, 1)  # NOTE: self.ZAinner \in [num_inducing x num_data x input_dim]!
-
-    def input_sensitivity(self):
-        return np.ones(self.input_dim) * self.variances
 
 class LinearFull(Kern):
     def __init__(self, input_dim, rank, W=None, kappa=None, active_dims=None, name='linear_full'):

GPy/kern/_src/periodic.py

@@ -101,6 +101,7 @@ class PeriodicExponential(Periodic):
         Flower = np.array(self._cos(self.basis_alpha,self.basis_omega,self.basis_phi)(self.lower))[:,None]
         return(self.lengthscale/(2*self.variance) * Gint + 1./self.variance*np.dot(Flower,Flower.T))
 
+    @silence_errors
     def update_gradients_full(self, dL_dK, X, X2=None):
         """derivative of the covariance matrix with respect to the parameters (shape is N x num_inducing x num_params)"""
         if X2 is None: X2 = X
@@ -213,7 +214,7 @@ class PeriodicMatern32(Periodic):
         return(self.lengthscale**3/(12*np.sqrt(3)*self.variance) * Gint + 1./self.variance*np.dot(Flower,Flower.T) + self.lengthscale**2/(3.*self.variance)*np.dot(F1lower,F1lower.T))
 
 
-    #@silence_errors
+    @silence_errors
     def update_gradients_full(self,dL_dK,X,X2):
         """derivative of the covariance matrix with respect to the parameters (shape is num_data x num_inducing x num_params)"""
         if X2 is None: X2 = X

GPy/kern/_src/psi_comp/__init__.py

@@ -1,2 +1,53 @@
 # Copyright (c) 2012, GPy authors (see AUTHORS.txt).
 # Licensed under the BSD 3-clause license (see LICENSE.txt)
+
+from ....core.parameterization.parameter_core import Pickleable
+from GPy.util.caching import Cache_this
+from ....core.parameterization import variational
+import rbf_psi_comp
+import ssrbf_psi_comp
+import sslinear_psi_comp
+import linear_psi_comp
+
+class PSICOMP_RBF(Pickleable):
+
+    @Cache_this(limit=2, ignore_args=(0,))
+    def psicomputations(self, variance, lengthscale, Z, variational_posterior):
+        if isinstance(variational_posterior, variational.NormalPosterior):
+            return rbf_psi_comp.psicomputations(variance, lengthscale, Z, variational_posterior)
+        elif isinstance(variational_posterior, variational.SpikeAndSlabPosterior):
+            return ssrbf_psi_comp.psicomputations(variance, lengthscale, Z, variational_posterior)
+        else:
+            raise ValueError, "unknown distriubtion received for psi-statistics"
+                
+    @Cache_this(limit=2, ignore_args=(0,1,2,3))
+    def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior):
+        if isinstance(variational_posterior, variational.NormalPosterior):
+            return rbf_psi_comp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior)
+        elif isinstance(variational_posterior, variational.SpikeAndSlabPosterior):
+            return ssrbf_psi_comp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior)
+        else:
+            raise ValueError, "unknown distriubtion received for psi-statistics"
+        
+class PSICOMP_Linear(Pickleable):
+
+    @Cache_this(limit=2, ignore_args=(0,))
+    def psicomputations(self, variance, Z, variational_posterior):
+        if isinstance(variational_posterior, variational.NormalPosterior):
+            return linear_psi_comp.psicomputations(variance, Z, variational_posterior)
+        elif isinstance(variational_posterior, variational.SpikeAndSlabPosterior):
+            return sslinear_psi_comp.psicomputations(variance, Z, variational_posterior)
+        else:
+            raise ValueError, "unknown distriubtion received for psi-statistics"
+                
+    @Cache_this(limit=2, ignore_args=(0,1,2,3))
+    def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior):
+        if isinstance(variational_posterior, variational.NormalPosterior):
+            return linear_psi_comp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior)
+        elif isinstance(variational_posterior, variational.SpikeAndSlabPosterior):
+            return sslinear_psi_comp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior)
+        else:
+            raise ValueError, "unknown distriubtion received for psi-statistics"
+        
+    def _setup_observers(self):
+        pass

GPy/kern/_src/psi_comp/linear_psi_comp.py

@@ -2,14 +2,45 @@
 # Licensed under the BSD 3-clause license (see LICENSE.txt)
 
 """
-The package for the Psi statistics computation of the linear kernel for SSGPLVM
+The package for the Psi statistics computation of the linear kernel for Bayesian GPLVM
 """
 
 import numpy as np
-from GPy.util.caching import Cache_this
 
-#@Cache_this(limit=1)
+def psicomputations(variance, Z, variational_posterior):
-def _psi2computations(variance, Z, mu, S, gamma):
+    """
+    Compute psi-statistics for ss-linear kernel
+    """
+    # here are the "statistics" for psi0, psi1 and psi2
+    # Produced intermediate results:
+    # psi0    N
+    # psi1    NxM
+    # psi2    MxM
+    mu = variational_posterior.mean
+    S = variational_posterior.variance
+
+    psi0 = np.einsum('q,nq->n',variance,np.square(mu)+S)
+    psi1 = np.einsum('q,mq,nq->nm',variance,Z,mu)
+    psi2 = np.einsum('q,mq,oq,nq->mo',np.square(variance),Z,Z,S) + np.einsum('nm,no->mo',psi1,psi1)
+
+    return psi0, psi1, psi2
+
+def psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior):
+    mu = variational_posterior.mean
+    S = variational_posterior.variance
+
+    dL_dvar, dL_dmu, dL_dS, dL_dZ = _psi2computations(dL_dpsi2, variance, Z, mu, S)
+
+    # Compute for psi0 and psi1
+    mu2S = np.square(mu)+S
+    dL_dvar += np.einsum('n,nq->q',dL_dpsi0,mu2S) + np.einsum('nm,mq,nq->q',dL_dpsi1,Z,mu)
+    dL_dmu += np.einsum('n,q,nq->nq',dL_dpsi0,2.*variance,mu) + np.einsum('nm,q,mq->nq',dL_dpsi1,variance,Z)
+    dL_dS += np.einsum('n,q->nq',dL_dpsi0,variance)
+    dL_dZ +=  np.einsum('nm,q,nq->mq',dL_dpsi1, variance,mu)
+    
+    return dL_dvar, dL_dZ, dL_dmu, dL_dS
+
+def _psi2computations(dL_dpsi2, variance, Z, mu, S):
     """
     Z - MxQ
     mu - NxQ
@@ -18,34 +49,23 @@ def _psi2computations(variance, Z, mu, S, gamma):
     """
     # here are the "statistics" for psi1 and psi2
     # Produced intermediate results:
-    # _psi2                NxMxM
+    # _psi2_dvariance      Q
-    # _psi2_dvariance      NxMxMxQ
+    # _psi2_dZ             MxQ
-    # _psi2_dZ             NxMxQ
+    # _psi2_dmu            NxQ
-    # _psi2_dgamma         NxMxMxQ
+    # _psi2_dS             NxQ
-    # _psi2_dmu            NxMxMxQ
-    # _psi2_dS             NxMxMxQ
     
-    mu2 = np.square(mu)
-    gamma2 = np.square(gamma)
     variance2 = np.square(variance)
-    mu2S = mu2+S # NxQ
+    common_sum = np.einsum('q,mq,nq->nm',variance,Z,mu) # NxM
-    common_sum = np.einsum('nq,q,mq,nq->nm',gamma,variance,Z,mu) # NxM
+    Z_expect = np.einsum('mo,mq,oq->q',dL_dpsi2,Z,Z)
+    common_expect = np.einsum('mo,mq,no->nq',dL_dpsi2+dL_dpsi2.T,Z,common_sum)
 
-    _dpsi2_dvariance = np.einsum('nq,q,mq,oq->nmoq',2.*(gamma*mu2S-gamma2*mu2),variance,Z,Z)+\
+    dL_dvar = np.einsum('q,nq,q->q',Z_expect,2.*S,variance)+ np.einsum('nq,nq->q',common_expect,mu)
-        np.einsum('nq,mq,nq,no->nmoq',gamma,Z,mu,common_sum)+\
-        np.einsum('nq,oq,nq,nm->nmoq',gamma,Z,mu,common_sum)
             
-    _dpsi2_dgamma = np.einsum('q,mq,oq,nq->nmoq',variance2,Z,Z,(mu2S-2.*gamma*mu2))+\
+    dL_dmu = np.einsum('nq,q->nq',common_expect,variance)
-        np.einsum('q,mq,nq,no->nmoq',variance,Z,mu,common_sum)+\
-        np.einsum('q,oq,nq,nm->nmoq',variance,Z,mu,common_sum)
     
-    _dpsi2_dmu = np.einsum('q,mq,oq,nq,nq->nmoq',variance2,Z,Z,mu,2.*(gamma-gamma2))+\
+    dL_dS = np.empty(S.shape)
-        np.einsum('nq,q,mq,no->nmoq',gamma,variance,Z,common_sum)+\
+    dL_dS[:] = np.einsum('q,q->q',Z_expect,variance2)
-        np.einsum('nq,q,oq,nm->nmoq',gamma,variance,Z,common_sum)
     
-    _dpsi2_dS = np.einsum('nq,q,mq,oq->nmoq',gamma,variance2,Z,Z)
+    dL_dZ = 2.*(np.einsum('om,q,mq,nq->oq',dL_dpsi2,variance2,Z,S)+np.einsum('om,q,nq,nm->oq',dL_dpsi2,variance,mu,common_sum))
 
-    _dpsi2_dZ = 2.*(np.einsum('nq,q,mq,nq->nmq',gamma,variance2,Z,mu2S)+np.einsum('nq,q,nq,nm->nmq',gamma,variance,mu,common_sum)
+    return dL_dvar, dL_dmu, dL_dS, dL_dZ
-                    -np.einsum('nq,q,mq,nq->nmq',gamma2,variance2,Z,mu2))
-
-    return _dpsi2_dvariance, _dpsi2_dgamma, _dpsi2_dmu, _dpsi2_dS, _dpsi2_dZ

GPy/kern/_src/psi_comp/rbf_psi_comp.py

@@ -0,0 +1,162 @@
+"""
+The module for psi-statistics for RBF kernel
+"""
+
+import numpy as np
+from GPy.util.caching import Cacher
+
+def psicomputations(variance, lengthscale, Z, variational_posterior):
+    """
+    Z - MxQ
+    mu - NxQ
+    S - NxQ
+    gamma - NxQ
+    """
+    # here are the "statistics" for psi0, psi1 and psi2
+    # Produced intermediate results:
+    # _psi1                NxM
+    mu = variational_posterior.mean
+    S = variational_posterior.variance
+    
+    psi0 = np.empty(mu.shape[0])
+    psi0[:] = variance
+    psi1 = _psi1computations(variance, lengthscale, Z, mu, S)
+    psi2 = _psi2computations(variance, lengthscale, Z, mu, S).sum(axis=0)
+    return psi0, psi1, psi2
+
+def __psi1computations(variance, lengthscale, Z, mu, S):
+    """
+    Z - MxQ
+    mu - NxQ
+    S - NxQ
+    gamma - NxQ
+    """
+    # here are the "statistics" for psi1
+    # Produced intermediate results:
+    # _psi1                NxM
+
+    lengthscale2 = np.square(lengthscale)
+
+    # psi1
+    _psi1_logdenom = np.log(S/lengthscale2+1.).sum(axis=-1) # N
+    _psi1_log = (_psi1_logdenom[:,None]+np.einsum('nmq,nq->nm',np.square(mu[:,None,:]-Z[None,:,:]),1./(S+lengthscale2)))/(-2.)
+    _psi1 = variance*np.exp(_psi1_log)
+        
+    return _psi1
+
+def __psi2computations(variance, lengthscale, Z, mu, S):
+    """
+    Z - MxQ
+    mu - NxQ
+    S - NxQ
+    gamma - NxQ
+    """
+    # here are the "statistics" for psi2
+    # Produced intermediate results:
+    # _psi2                MxM
+    
+    lengthscale2 = np.square(lengthscale)
+    
+    _psi2_logdenom = np.log(2.*S/lengthscale2+1.).sum(axis=-1)/(-2.) # N
+    _psi2_exp1 = (np.square(Z[:,None,:]-Z[None,:,:])/lengthscale2).sum(axis=-1)/(-4.) #MxM
+    Z_hat = (Z[:,None,:]+Z[None,:,:])/2. #MxMxQ
+    denom = 1./(2.*S+lengthscale2)
+    _psi2_exp2 = -(np.square(mu)*denom).sum(axis=-1)[:,None,None]+2.*np.einsum('nq,moq,nq->nmo',mu,Z_hat,denom)-np.einsum('moq,nq->nmo',np.square(Z_hat),denom)
+    _psi2 = variance*variance*np.exp(_psi2_logdenom[:,None,None]+_psi2_exp1[None,:,:]+_psi2_exp2)
+    
+
+    return _psi2
+
+def psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior):
+    ARD = (len(lengthscale)!=1)
+    
+    dvar_psi1, dl_psi1, dZ_psi1, dmu_psi1, dS_psi1 = _psi1compDer(dL_dpsi1, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance)
+    dvar_psi2, dl_psi2, dZ_psi2, dmu_psi2, dS_psi2 = _psi2compDer(dL_dpsi2, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance)
+
+    dL_dvar = np.sum(dL_dpsi0) + dvar_psi1 + dvar_psi2
+    
+    dL_dlengscale = dl_psi1 + dl_psi2
+    if not ARD:
+        dL_dlengscale = dL_dlengscale.sum()
+
+    dL_dmu = dmu_psi1 + dmu_psi2
+    dL_dS = dS_psi1 + dS_psi2
+    dL_dZ = dZ_psi1 + dZ_psi2
+    
+    return dL_dvar, dL_dlengscale, dL_dZ, dL_dmu, dL_dS
+
+def _psi1compDer(dL_dpsi1, variance, lengthscale, Z, mu, S):
+    """
+    dL_dpsi1 - NxM
+    Z - MxQ
+    mu - NxQ
+    S - NxQ
+    gamma - NxQ
+    """
+    # here are the "statistics" for psi1
+    # Produced intermediate results: dL_dparams w.r.t. psi1
+    # _dL_dvariance     1
+    # _dL_dlengthscale  Q
+    # _dL_dZ            MxQ
+    # _dL_dgamma        NxQ
+    # _dL_dmu           NxQ
+    # _dL_dS            NxQ
+    
+    lengthscale2 = np.square(lengthscale)
+    
+    _psi1 = _psi1computations(variance, lengthscale, Z, mu, S)
+    Lpsi1 = dL_dpsi1*_psi1
+    Zmu = Z[None,:,:]-mu[:,None,:] # NxMxQ
+    denom = 1./(S+lengthscale2)
+    Zmu2_denom = np.square(Zmu)*denom[:,None,:] #NxMxQ
+    _dL_dvar = Lpsi1.sum()/variance
+    _dL_dmu = np.einsum('nm,nmq,nq->nq',Lpsi1,Zmu,denom)
+    _dL_dS = np.einsum('nm,nmq,nq->nq',Lpsi1,(Zmu2_denom-1.),denom)/2.
+    _dL_dZ = -np.einsum('nm,nmq,nq->mq',Lpsi1,Zmu,denom)
+    _dL_dl = np.einsum('nm,nmq,nq->q',Lpsi1,(Zmu2_denom+(S/lengthscale2)[:,None,:]),denom*lengthscale)
+    
+    return _dL_dvar, _dL_dl, _dL_dZ, _dL_dmu, _dL_dS
+
+def _psi2compDer(dL_dpsi2, variance, lengthscale, Z, mu, S):
+    """
+    Z - MxQ
+    mu - NxQ
+    S - NxQ
+    gamma - NxQ
+    dL_dpsi2 - MxM
+    """
+    # here are the "statistics" for psi2
+    # Produced the derivatives w.r.t. psi2:
+    # _dL_dvariance      1
+    # _dL_dlengthscale   Q
+    # _dL_dZ             MxQ
+    # _dL_dgamma         NxQ
+    # _dL_dmu            NxQ
+    # _dL_dS             NxQ
+    
+    lengthscale2 = np.square(lengthscale)
+    denom = 1./(2*S+lengthscale2)
+    denom2 = np.square(denom)
+
+    _psi2 = _psi2computations(variance, lengthscale, Z, mu, S) # NxMxM
+    
+    Lpsi2 = dL_dpsi2[None,:,:]*_psi2
+    Lpsi2sum = np.einsum('nmo->n',Lpsi2) #N
+    Lpsi2Z = np.einsum('nmo,oq->nq',Lpsi2,Z) #NxQ
+    Lpsi2Z2 = np.einsum('nmo,oq,oq->nq',Lpsi2,Z,Z) #NxQ
+    Lpsi2Z2p = np.einsum('nmo,mq,oq->nq',Lpsi2,Z,Z) #NxQ
+    Lpsi2Zhat = Lpsi2Z
+    Lpsi2Zhat2 = (Lpsi2Z2+Lpsi2Z2p)/2
+    
+    _dL_dvar = Lpsi2sum.sum()*2/variance
+    _dL_dmu = (-2*denom) * (mu*Lpsi2sum[:,None]-Lpsi2Zhat)
+    _dL_dS = (2*np.square(denom))*(np.square(mu)*Lpsi2sum[:,None]-2*mu*Lpsi2Zhat+Lpsi2Zhat2) - denom*Lpsi2sum[:,None]
+    _dL_dZ = -np.einsum('nmo,oq->oq',Lpsi2,Z)/lengthscale2+np.einsum('nmo,oq->mq',Lpsi2,Z)/lengthscale2+ \
+             2*np.einsum('nmo,nq,nq->mq',Lpsi2,mu,denom) - np.einsum('nmo,nq,mq->mq',Lpsi2,denom,Z) - np.einsum('nmo,oq,nq->mq',Lpsi2,Z,denom)
+    _dL_dl = 2*lengthscale* ((S/lengthscale2*denom+np.square(mu*denom))*Lpsi2sum[:,None]+(Lpsi2Z2-Lpsi2Z2p)/(2*np.square(lengthscale2))-
+                             (2*mu*denom2)*Lpsi2Zhat+denom2*Lpsi2Zhat2).sum(axis=0)
+
+    return _dL_dvar, _dL_dl, _dL_dZ, _dL_dmu, _dL_dS
+    
+_psi1computations = Cacher(__psi1computations, limit=1)
+_psi2computations = Cacher(__psi2computations, limit=1)

GPy/kern/_src/psi_comp/rbf_psi_gpucomp.py

@@ -0,0 +1,411 @@
+"""
+The module for psi-statistics for RBF kernel
+"""
+
+import numpy as np
+from ....util.caching import Cache_this
+from . import PSICOMP_RBF
+from ....util import gpu_init
+
+try:
+    import pycuda.gpuarray as gpuarray
+    from pycuda.compiler import SourceModule
+    from ....util.linalg_gpu import sum_axis
+except:
+    pass    
+
+gpu_code = """
+    // define THREADNUM
+
+    #define IDX_NMQ(n,m,q) ((q*M+m)*N+n)
+    #define IDX_NMM(n,m1,m2) ((m2*M+m1)*N+n)
+    #define IDX_NQ(n,q) (q*N+n)
+    #define IDX_NM(n,m) (m*N+n)
+    #define IDX_MQ(m,q) (q*M+m)
+    #define IDX_MM(m1,m2) (m2*M+m1)
+    #define IDX_NQB(n,q,b) ((b*Q+q)*N+n)
+    #define IDX_QB(q,b) (b*Q+q)
+
+    // Divide data evenly
+    __device__ void divide_data(int total_data, int psize, int pidx, int *start, int *end) {
+        int residue = (total_data)%psize;
+        if(pidx<residue) {
+            int size = total_data/psize+1;
+            *start = size*pidx;
+            *end = *start+size;
+        } else {
+            int size = total_data/psize;
+            *start = size*pidx+residue;
+            *end = *start+size;
+        }
+    }
+    
+    __device__ void reduce_sum(double* array, int array_size) {
+        int s;
+        if(array_size >= blockDim.x) {
+            for(int i=blockDim.x+threadIdx.x; i<array_size; i+= blockDim.x) {
+                array[threadIdx.x] += array[i];
+            }
+            array_size = blockDim.x;
+        }
+        __syncthreads();
+        for(int i=1; i<=array_size;i*=2) {s=i;}
+        if(threadIdx.x < array_size-s) {array[threadIdx.x] += array[s+threadIdx.x];}
+        __syncthreads();
+        for(s=s/2;s>=1;s=s/2) {
+            if(threadIdx.x < s) {array[threadIdx.x] += array[s+threadIdx.x];}
+            __syncthreads();
+        }
+    }
+
+    __global__ void compDenom(double *log_denom1, double *log_denom2, double *l, double *S, int N, int Q)
+    {
+        int n_start, n_end;
+        divide_data(N, gridDim.x, blockIdx.x, &n_start, &n_end);
+        
+        for(int i=n_start*Q+threadIdx.x; i<n_end*Q; i+=blockDim.x) {
+            int n=i/Q;
+            int q=i%Q;
+
+            double Snq = S[IDX_NQ(n,q)];
+            double lq = l[q]*l[q];
+            log_denom1[IDX_NQ(n,q)] = log(Snq/lq+1.);
+            log_denom2[IDX_NQ(n,q)] = log(2.*Snq/lq+1.);
+        }
+    }
+
+    __global__ void psi1computations(double *psi1, double *log_denom1, double var, double *l, double *Z, double *mu, double *S, int N, int M, int Q)
+    {
+        int m_start, m_end;
+        divide_data(M, gridDim.x, blockIdx.x, &m_start, &m_end);
+        
+        for(int m=m_start; m<m_end; m++) {
+            for(int n=threadIdx.x; n<N; n+= blockDim.x) {            
+                double log_psi1 = 0;
+                for(int q=0;q<Q;q++) {
+                    double muZ = mu[IDX_NQ(n,q)]-Z[IDX_MQ(m,q)];
+                    double Snq = S[IDX_NQ(n,q)];
+                    double lq = l[q]*l[q];
+                    log_psi1 += (muZ*muZ/(Snq+lq)+log_denom1[IDX_NQ(n,q)])/(-2.);
+                }
+                psi1[IDX_NM(n,m)] = var*exp(log_psi1);
+            }
+        }
+    }
+    
+    __global__ void psi2computations(double *psi2, double *psi2n, double *log_denom2, double var, double *l, double *Z, double *mu, double *S, int N, int M, int Q)
+    {
+        int psi2_idx_start, psi2_idx_end;
+        __shared__ double psi2_local[THREADNUM];
+        divide_data((M+1)*M/2, gridDim.x, blockIdx.x, &psi2_idx_start, &psi2_idx_end);
+        
+        for(int psi2_idx=psi2_idx_start; psi2_idx<psi2_idx_end; psi2_idx++) {
+            int m1 = int((sqrt(8.*psi2_idx+1.)-1.)/2.);
+            int m2 = psi2_idx - (m1+1)*m1/2;
+            
+            psi2_local[threadIdx.x] = 0;
+            for(int n=threadIdx.x;n<N;n+=blockDim.x) {
+                double log_psi2_n = 0;
+                for(int q=0;q<Q;q++) {
+                    double dZ = Z[IDX_MQ(m1,q)] - Z[IDX_MQ(m2,q)];
+                    double muZhat = mu[IDX_NQ(n,q)]- (Z[IDX_MQ(m1,q)]+Z[IDX_MQ(m2,q)])/2.;
+                    double Snq = S[IDX_NQ(n,q)];
+                    double lq = l[q]*l[q];
+                    log_psi2_n += dZ*dZ/(-4.*lq)-muZhat*muZhat/(2.*Snq+lq) + log_denom2[IDX_NQ(n,q)]/(-2.);
+                }
+                double exp_psi2_n = exp(log_psi2_n);
+                psi2n[IDX_NMM(n,m1,m2)] = var*var*exp_psi2_n;
+                if(m1!=m2) { psi2n[IDX_NMM(n,m2,m1)] = var*var*exp_psi2_n;}
+                psi2_local[threadIdx.x] += exp_psi2_n;
+            }
+            __syncthreads();
+            reduce_sum(psi2_local, THREADNUM);
+            if(threadIdx.x==0) {
+                psi2[IDX_MM(m1,m2)] = var*var*psi2_local[0];
+                if(m1!=m2) { psi2[IDX_MM(m2,m1)] = var*var*psi2_local[0]; }
+            }
+            __syncthreads();
+        }
+    }
+    
+    __global__ void psi1compDer(double *dvar, double *dl, double *dZ, double *dmu, double *dS, double *dL_dpsi1, double *psi1, double var, double *l, double *Z, double *mu, double *S, int N, int M, int Q)
+    {
+        int m_start, m_end;
+        __shared__ double g_local[THREADNUM];
+        divide_data(M, gridDim.x, blockIdx.x, &m_start, &m_end);
+        int P = int(ceil(double(N)/THREADNUM));
+
+        double dvar_local = 0;
+        for(int q=0;q<Q;q++) {
+            double lq_sqrt = l[q];
+            double lq = lq_sqrt*lq_sqrt;
+            double dl_local = 0;
+            for(int p=0;p<P;p++) {
+                int n = p*THREADNUM + threadIdx.x;
+                double dmu_local = 0;
+                double dS_local = 0;
+                double Snq,mu_nq;
+                if(n<N) {Snq = S[IDX_NQ(n,q)]; mu_nq=mu[IDX_NQ(n,q)];}
+                for(int m=m_start; m<m_end; m++) {
+                    if(n<N) {
+                        double lpsi1 = psi1[IDX_NM(n,m)]*dL_dpsi1[IDX_NM(n,m)];
+                        if(q==0) {dvar_local += lpsi1;}
+                        
+                        double Zmu = Z[IDX_MQ(m,q)] - mu_nq;
+                        double denom = Snq+lq;
+                        double Zmu2_denom = Zmu*Zmu/denom;
+                        
+                        dmu_local += lpsi1*Zmu/denom;
+                        dS_local += lpsi1*(Zmu2_denom-1.)/denom;
+                        dl_local += lpsi1*(Zmu2_denom+Snq/lq)/denom;
+                        g_local[threadIdx.x] = -lpsi1*Zmu/denom;
+                    }
+                    __syncthreads();
+                    reduce_sum(g_local, p<P-1?THREADNUM:N-(P-1)*THREADNUM);
+                    if(threadIdx.x==0) {dZ[IDX_MQ(m,q)] += g_local[0];}
+                }
+                if(n<N) {
+                    dmu[IDX_NQB(n,q,blockIdx.x)] += dmu_local;
+                    dS[IDX_NQB(n,q,blockIdx.x)] += dS_local/2.;
+                }
+                __threadfence_block();
+            }
+            g_local[threadIdx.x] = dl_local*lq_sqrt;
+            __syncthreads();
+            reduce_sum(g_local, THREADNUM);
+            if(threadIdx.x==0) {dl[IDX_QB(q,blockIdx.x)] += g_local[0];}
+        }
+        g_local[threadIdx.x] = dvar_local;
+        __syncthreads();
+        reduce_sum(g_local, THREADNUM);
+        if(threadIdx.x==0) {dvar[blockIdx.x] += g_local[0]/var;}        
+    }
+    
+    __global__ void psi2compDer(double *dvar, double *dl, double *dZ, double *dmu, double *dS, double *dL_dpsi2, double *psi2n, double var, double *l, double *Z, double *mu, double *S, int N, int M, int Q)
+    {
+        int m_start, m_end;
+        __shared__ double g_local[THREADNUM];
+        divide_data(M, gridDim.x, blockIdx.x, &m_start, &m_end);
+        int P = int(ceil(double(N)/THREADNUM));
+
+        double dvar_local = 0;
+        for(int q=0;q<Q;q++) {
+            double lq_sqrt = l[q];
+            double lq = lq_sqrt*lq_sqrt;
+            double dl_local = 0;
+            for(int p=0;p<P;p++) {
+                int n = p*THREADNUM + threadIdx.x;
+                double dmu_local = 0;
+                double dS_local = 0;
+                double Snq,mu_nq;
+                if(n<N) {Snq = S[IDX_NQ(n,q)]; mu_nq=mu[IDX_NQ(n,q)];}
+                for(int m1=m_start; m1<m_end; m1++) {
+                    g_local[threadIdx.x] = 0;
+                    for(int m2=0;m2<M;m2++) {
+                        if(n<N) {
+                            double lpsi2 = psi2n[IDX_NMM(n,m1,m2)]*dL_dpsi2[IDX_MM(m1,m2)];
+                            if(q==0) {dvar_local += lpsi2;}
+                            
+                            double dZ = Z[IDX_MQ(m1,q)] - Z[IDX_MQ(m2,q)];
+                            double muZhat =  mu_nq - (Z[IDX_MQ(m1,q)] + Z[IDX_MQ(m2,q)])/2.;
+                            double denom = 2.*Snq+lq;
+                            double muZhat2_denom = muZhat*muZhat/denom;
+                            
+                            dmu_local += lpsi2*muZhat/denom;
+                            dS_local += lpsi2*(2.*muZhat2_denom-1.)/denom;
+                            dl_local += lpsi2*((Snq/lq+muZhat2_denom)/denom+dZ*dZ/(4.*lq*lq));
+                            g_local[threadIdx.x] += 2.*lpsi2*(muZhat/denom-dZ/(2*lq));
+                        }
+                    }
+                    __syncthreads();
+                    reduce_sum(g_local, p<P-1?THREADNUM:N-(P-1)*THREADNUM);
+                    if(threadIdx.x==0) {dZ[IDX_MQ(m1,q)] += g_local[0];}
+                }
+                if(n<N) {
+                    dmu[IDX_NQB(n,q,blockIdx.x)] += -2.*dmu_local;
+                    dS[IDX_NQB(n,q,blockIdx.x)] += dS_local;
+                }
+                __threadfence_block();
+            }
+            g_local[threadIdx.x] = dl_local*2.*lq_sqrt;
+            __syncthreads();
+            reduce_sum(g_local, THREADNUM);
+            if(threadIdx.x==0) {dl[IDX_QB(q,blockIdx.x)] += g_local[0];}
+        }
+        g_local[threadIdx.x] = dvar_local;
+        __syncthreads();
+        reduce_sum(g_local, THREADNUM);
+        if(threadIdx.x==0) {dvar[blockIdx.x] += g_local[0]*2/var;}
+    }
+    """
+
+class PSICOMP_RBF_GPU(PSICOMP_RBF):
+
+    def __init__(self, threadnum=128, blocknum=15, GPU_direct=False):
+        self.GPU_direct = GPU_direct
+        self.gpuCache = None
+        
+        self.threadnum = threadnum
+        self.blocknum = blocknum
+        module = SourceModule("#define THREADNUM "+str(self.threadnum)+"\n"+gpu_code)
+        self.g_psi1computations = module.get_function('psi1computations')
+        self.g_psi1computations.prepare('PPdPPPPiii')
+        self.g_psi2computations = module.get_function('psi2computations')
+        self.g_psi2computations.prepare('PPPdPPPPiii')
+        self.g_psi1compDer = module.get_function('psi1compDer')
+        self.g_psi1compDer.prepare('PPPPPPPdPPPPiii')
+        self.g_psi2compDer = module.get_function('psi2compDer')
+        self.g_psi2compDer.prepare('PPPPPPPdPPPPiii')
+        self.g_compDenom = module.get_function('compDenom')
+        self.g_compDenom.prepare('PPPPii')
+        
+    def __deepcopy__(self, memo):
+        s = PSICOMP_RBF_GPU(threadnum=self.threadnum, blocknum=self.blocknum, GPU_direct=self.GPU_direct)
+        memo[id(self)] = s 
+        return s
+    
+    def _initGPUCache(self, N, M, Q):            
+        if self.gpuCache == None:
+            self.gpuCache = {
+                             'l_gpu'                :gpuarray.empty((Q,),np.float64,order='F'),
+                             'Z_gpu'                :gpuarray.empty((M,Q),np.float64,order='F'),
+                             'mu_gpu'               :gpuarray.empty((N,Q),np.float64,order='F'),
+                             'S_gpu'                :gpuarray.empty((N,Q),np.float64,order='F'),
+                             'psi1_gpu'             :gpuarray.empty((N,M),np.float64,order='F'),
+                             'psi2_gpu'             :gpuarray.empty((M,M),np.float64,order='F'),
+                             'psi2n_gpu'            :gpuarray.empty((N,M,M),np.float64,order='F'),
+                             'dL_dpsi1_gpu'         :gpuarray.empty((N,M),np.float64,order='F'),
+                             'dL_dpsi2_gpu'         :gpuarray.empty((M,M),np.float64,order='F'),
+                             'log_denom1_gpu'       :gpuarray.empty((N,Q),np.float64,order='F'),
+                             'log_denom2_gpu'       :gpuarray.empty((N,Q),np.float64,order='F'),
+                             # derivatives
+                             'dvar_gpu'             :gpuarray.empty((self.blocknum,),np.float64, order='F'),
+                             'dl_gpu'               :gpuarray.empty((Q,self.blocknum),np.float64, order='F'),
+                             'dZ_gpu'               :gpuarray.empty((M,Q),np.float64, order='F'),
+                             'dmu_gpu'              :gpuarray.empty((N,Q,self.blocknum),np.float64, order='F'),
+                             'dS_gpu'               :gpuarray.empty((N,Q,self.blocknum),np.float64, order='F'),
+                             # grad
+                             'grad_l_gpu'               :gpuarray.empty((Q,),np.float64, order='F'),
+                             'grad_mu_gpu'              :gpuarray.empty((N,Q,),np.float64, order='F'),
+                             'grad_S_gpu'               :gpuarray.empty((N,Q,),np.float64, order='F'),
+                             }
+        else:
+            assert N==self.gpuCache['mu_gpu'].shape[0]
+            assert M==self.gpuCache['Z_gpu'].shape[0]
+            assert Q==self.gpuCache['l_gpu'].shape[0]
+    
+    def sync_params(self, lengthscale, Z, mu, S):
+        if len(lengthscale)==1:
+            self.gpuCache['l_gpu'].fill(lengthscale)
+        else:
+            self.gpuCache['l_gpu'].set(np.asfortranarray(lengthscale))
+        self.gpuCache['Z_gpu'].set(np.asfortranarray(Z))
+        self.gpuCache['mu_gpu'].set(np.asfortranarray(mu))
+        self.gpuCache['S_gpu'].set(np.asfortranarray(S))
+        N,Q = self.gpuCache['S_gpu'].shape
+        # t=self.g_compDenom(self.gpuCache['log_denom1_gpu'],self.gpuCache['log_denom2_gpu'],self.gpuCache['l_gpu'],self.gpuCache['S_gpu'], np.int32(N), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True)
+        # print 'g_compDenom '+str(t)
+        self.g_compDenom.prepared_call((self.blocknum,1),(self.threadnum,1,1), self.gpuCache['log_denom1_gpu'].gpudata,self.gpuCache['log_denom2_gpu'].gpudata,self.gpuCache['l_gpu'].gpudata,self.gpuCache['S_gpu'].gpudata, np.int32(N), np.int32(Q))
+        
+    def reset_derivative(self):
+        self.gpuCache['dvar_gpu'].fill(0.)
+        self.gpuCache['dl_gpu'].fill(0.)
+        self.gpuCache['dZ_gpu'].fill(0.)
+        self.gpuCache['dmu_gpu'].fill(0.)
+        self.gpuCache['dS_gpu'].fill(0.)
+        self.gpuCache['grad_l_gpu'].fill(0.)
+        self.gpuCache['grad_mu_gpu'].fill(0.)
+        self.gpuCache['grad_S_gpu'].fill(0.)
+    
+    def get_dimensions(self, Z, variational_posterior):
+        return variational_posterior.mean.shape[0], Z.shape[0], Z.shape[1]
+
+    @Cache_this(limit=1, ignore_args=(0,))
+    def psicomputations(self, variance, lengthscale, Z, variational_posterior):
+        """
+        Z - MxQ
+        mu - NxQ
+        S - NxQ
+        """
+        N,M,Q = self.get_dimensions(Z, variational_posterior)
+        self._initGPUCache(N,M,Q)
+        self.sync_params(lengthscale, Z, variational_posterior.mean, variational_posterior.variance)
+        
+        psi1_gpu = self.gpuCache['psi1_gpu']
+        psi2_gpu = self.gpuCache['psi2_gpu']
+        psi2n_gpu = self.gpuCache['psi2n_gpu']
+        l_gpu = self.gpuCache['l_gpu']
+        Z_gpu = self.gpuCache['Z_gpu']
+        mu_gpu = self.gpuCache['mu_gpu']
+        S_gpu = self.gpuCache['S_gpu']
+        log_denom1_gpu = self.gpuCache['log_denom1_gpu']
+        log_denom2_gpu = self.gpuCache['log_denom2_gpu']
+
+        psi0 = np.empty((N,))
+        psi0[:] = variance
+        self.g_psi1computations.prepared_call((self.blocknum,1),(self.threadnum,1,1),psi1_gpu.gpudata, log_denom1_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata, np.int32(N), np.int32(M), np.int32(Q))
+        self.g_psi2computations.prepared_call((self.blocknum,1),(self.threadnum,1,1),psi2_gpu.gpudata, psi2n_gpu.gpudata, log_denom2_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata, np.int32(N), np.int32(M), np.int32(Q))
+        # t = self.g_psi1computations(psi1_gpu, log_denom1_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True)
+        # print 'g_psi1computations '+str(t)
+        # t = self.g_psi2computations(psi2_gpu, psi2n_gpu, log_denom2_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True)
+        # print 'g_psi2computations '+str(t)
+         
+        if self.GPU_direct:
+            return psi0, psi1_gpu, psi2_gpu
+        else:
+            return psi0, psi1_gpu.get(), psi2_gpu.get()
+
+    @Cache_this(limit=1, ignore_args=(0,1,2,3))
+    def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior):
+        ARD = (len(lengthscale)!=1)
+        
+        N,M,Q = self.get_dimensions(Z, variational_posterior)
+        psi1_gpu = self.gpuCache['psi1_gpu']
+        psi2n_gpu = self.gpuCache['psi2n_gpu']
+        l_gpu = self.gpuCache['l_gpu']
+        Z_gpu = self.gpuCache['Z_gpu']
+        mu_gpu = self.gpuCache['mu_gpu']
+        S_gpu = self.gpuCache['S_gpu']
+        dvar_gpu = self.gpuCache['dvar_gpu']
+        dl_gpu = self.gpuCache['dl_gpu']
+        dZ_gpu = self.gpuCache['dZ_gpu']
+        dmu_gpu = self.gpuCache['dmu_gpu']
+        dS_gpu = self.gpuCache['dS_gpu']
+        grad_l_gpu = self.gpuCache['grad_l_gpu']
+        grad_mu_gpu = self.gpuCache['grad_mu_gpu']
+        grad_S_gpu = self.gpuCache['grad_S_gpu']
+        
+        if self.GPU_direct:
+            dL_dpsi1_gpu = dL_dpsi1
+            dL_dpsi2_gpu = dL_dpsi2
+            dL_dpsi0_sum = dL_dpsi0.get().sum() #gpuarray.sum(dL_dpsi0).get()
+        else:
+            dL_dpsi1_gpu = self.gpuCache['dL_dpsi1_gpu']
+            dL_dpsi2_gpu = self.gpuCache['dL_dpsi2_gpu']
+            dL_dpsi1_gpu.set(np.asfortranarray(dL_dpsi1))
+            dL_dpsi2_gpu.set(np.asfortranarray(dL_dpsi2))
+            dL_dpsi0_sum = dL_dpsi0.sum()
+
+        self.reset_derivative()
+        # t=self.g_psi1compDer(dvar_gpu,dl_gpu,dZ_gpu,dmu_gpu,dS_gpu,dL_dpsi1_gpu,psi1_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True)
+        # print 'g_psi1compDer '+str(t)
+        # t=self.g_psi2compDer(dvar_gpu,dl_gpu,dZ_gpu,dmu_gpu,dS_gpu,dL_dpsi2_gpu,psi2n_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True)
+        # print 'g_psi2compDer '+str(t)
+        self.g_psi1compDer.prepared_call((self.blocknum,1),(self.threadnum,1,1),dvar_gpu.gpudata,dl_gpu.gpudata,dZ_gpu.gpudata,dmu_gpu.gpudata,dS_gpu.gpudata,dL_dpsi1_gpu.gpudata,psi1_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata, np.int32(N), np.int32(M), np.int32(Q))
+        self.g_psi2compDer.prepared_call((self.blocknum,1),(self.threadnum,1,1),dvar_gpu.gpudata,dl_gpu.gpudata,dZ_gpu.gpudata,dmu_gpu.gpudata,dS_gpu.gpudata,dL_dpsi2_gpu.gpudata,psi2n_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata, np.int32(N), np.int32(M), np.int32(Q))
+
+        dL_dvar = dL_dpsi0_sum + dvar_gpu.get().sum()#gpuarray.sum(dvar_gpu).get()
+        sum_axis(grad_mu_gpu,dmu_gpu,N*Q,self.blocknum)
+        dL_dmu = grad_mu_gpu.get()
+        sum_axis(grad_S_gpu,dS_gpu,N*Q,self.blocknum)
+        dL_dS = grad_S_gpu.get()
+        dL_dZ = dZ_gpu.get()
+        if ARD:
+            sum_axis(grad_l_gpu,dl_gpu,Q,self.blocknum)
+            dL_dlengscale = grad_l_gpu.get()
+        else:
+            dL_dlengscale = dl_gpu.get().sum() #gpuarray.sum(dl_gpu).get()
+            
+        return dL_dvar, dL_dlengscale, dL_dZ, dL_dmu, dL_dS
+    
+

GPy/kern/_src/psi_comp/sslinear_psi_comp.py

@@ -0,0 +1,83 @@
+# Copyright (c) 2012, GPy authors (see AUTHORS.txt).
+# Licensed under the BSD 3-clause license (see LICENSE.txt)
+
+"""
+The package for the Psi statistics computation of the linear kernel for SSGPLVM
+"""
+
+import numpy as np
+
+def psicomputations(variance, Z, variational_posterior):
+    """
+    Compute psi-statistics for ss-linear kernel
+    """
+    # here are the "statistics" for psi0, psi1 and psi2
+    # Produced intermediate results:
+    # psi0    N
+    # psi1    NxM
+    # psi2    MxM
+    mu = variational_posterior.mean
+    S = variational_posterior.variance
+    gamma = variational_posterior.binary_prob
+
+    psi0 = np.einsum('q,nq,nq->n',variance,gamma,np.square(mu)+S)
+    psi1 = np.einsum('nq,q,mq,nq->nm',gamma,variance,Z,mu)
+    psi2 = np.einsum('nq,q,mq,oq,nq->mo',gamma,np.square(variance),Z,Z,(1-gamma)*np.square(mu)+S) +\
+                np.einsum('nm,no->mo',psi1,psi1) 
+
+    return psi0, psi1, psi2
+
+def psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior):
+    mu = variational_posterior.mean
+    S = variational_posterior.variance
+    gamma = variational_posterior.binary_prob
+
+    dL_dvar, dL_dgamma, dL_dmu, dL_dS, dL_dZ = _psi2computations(dL_dpsi2, variance, Z, mu, S, gamma)
+
+    # Compute for psi0 and psi1
+    mu2S = np.square(mu)+S
+    dL_dvar += np.einsum('n,nq,nq->q',dL_dpsi0,gamma,mu2S) + np.einsum('nm,nq,mq,nq->q',dL_dpsi1,gamma,Z,mu)
+    dL_dgamma += np.einsum('n,q,nq->nq',dL_dpsi0,variance,mu2S) + np.einsum('nm,q,mq,nq->nq',dL_dpsi1,variance,Z,mu)
+    dL_dmu += np.einsum('n,nq,q,nq->nq',dL_dpsi0,gamma,2.*variance,mu) + np.einsum('nm,nq,q,mq->nq',dL_dpsi1,gamma,variance,Z)
+    dL_dS += np.einsum('n,nq,q->nq',dL_dpsi0,gamma,variance)
+    dL_dZ +=  np.einsum('nm,nq,q,nq->mq',dL_dpsi1,gamma, variance,mu)
+    
+    return dL_dvar, dL_dZ, dL_dmu, dL_dS, dL_dgamma
+
+def _psi2computations(dL_dpsi2, variance, Z, mu, S, gamma):
+    """
+    Z - MxQ
+    mu - NxQ
+    S - NxQ
+    gamma - NxQ
+    """
+    # here are the "statistics" for psi1 and psi2
+    # Produced intermediate results:
+    # _psi2_dvariance      Q
+    # _psi2_dZ             MxQ
+    # _psi2_dgamma         NxQ
+    # _psi2_dmu            NxQ
+    # _psi2_dS             NxQ
+    
+    mu2 = np.square(mu)
+    gamma2 = np.square(gamma)
+    variance2 = np.square(variance)
+    mu2S = mu2+S # NxQ
+    common_sum = np.einsum('nq,q,mq,nq->nm',gamma,variance,Z,mu) # NxM
+    Z_expect = np.einsum('mo,mq,oq->q',dL_dpsi2,Z,Z)
+    common_expect = np.einsum('mo,mq,no->nq',dL_dpsi2+dL_dpsi2.T,Z,common_sum)
+    
+    dL_dvar = np.einsum('nq,q,q->q',2.*(gamma*mu2S-gamma2*mu2),variance,Z_expect)+\
+        np.einsum('nq,nq,nq->q',common_expect,gamma,mu)
+        
+    dL_dgamma = np.einsum('q,q,nq->nq',Z_expect,variance2,(mu2S-2.*gamma*mu2))+\
+        np.einsum('nq,q,nq->nq',common_expect,variance,mu)
+    
+    dL_dmu = np.einsum('q,q,nq,nq->nq',Z_expect,variance2,mu,2.*(gamma-gamma2))+\
+            np.einsum('nq,nq,q->nq',common_expect,gamma,variance)
+                    
+    dL_dS = np.einsum('q,nq,q->nq',Z_expect,gamma,variance2)
+    
+    dL_dZ = 2.*(np.einsum('om,nq,q,mq,nq->oq',dL_dpsi2,gamma,variance2,Z,(mu2S-gamma*mu2))+np.einsum('om,nq,q,nq,nm->oq',dL_dpsi2,gamma,variance,mu,common_sum))
+
+    return dL_dvar, dL_dgamma, dL_dmu, dL_dS, dL_dZ

GPy/kern/_src/psi_comp/ssrbf_psi_comp.py

@@ -6,15 +6,27 @@ The package for the psi statistics computation
 """
 
 import numpy as np
-from GPy.util.caching import Cache_this
 
-@Cache_this(limit=1)
+def psicomputations(variance, lengthscale, Z, variational_posterior):
-def _Z_distances(Z):
+    """
-    Zhat = 0.5 * (Z[:, None, :] + Z[None, :, :]) # M,M,Q
+    Z - MxQ
-    Zdist = 0.5 * (Z[:, None, :] - Z[None, :, :]) # M,M,Q
+    mu - NxQ
-    return Zhat, Zdist
+    S - NxQ
+    gamma - NxQ
+    """
+    # here are the "statistics" for psi0, psi1 and psi2
+    # Produced intermediate results:
+    # _psi1                NxM
+    mu = variational_posterior.mean
+    S = variational_posterior.variance
+    gamma = variational_posterior.binary_prob
+    
+    psi0 = np.empty(mu.shape[0])
+    psi0[:] = variance
+    psi1 = _psi1computations(variance, lengthscale, Z, mu, S, gamma)
+    psi2 = _psi2computations(variance, lengthscale, Z, mu, S, gamma)
+    return psi0, psi1, psi2
 
-@Cache_this(limit=1)
 def _psi1computations(variance, lengthscale, Z, mu, S, gamma):
     """
     Z - MxQ
@@ -22,15 +34,9 @@ def _psi1computations(variance, lengthscale, Z, mu, S, gamma):
     S - NxQ
     gamma - NxQ
     """
-    # here are the "statistics" for psi1 and psi2
+    # here are the "statistics" for psi1
     # Produced intermediate results:
     # _psi1                NxM
-    # _dpsi1_dvariance     NxM
-    # _dpsi1_dlengthscale  NxMxQ
-    # _dpsi1_dZ            NxMxQ
-    # _dpsi1_dgamma        NxMxQ
-    # _dpsi1_dmu           NxMxQ
-    # _dpsi1_dS            NxMxQ
 
     lengthscale2 = np.square(lengthscale)
 
@@ -40,25 +46,15 @@ def _psi1computations(variance, lengthscale, Z, mu, S, gamma):
     _psi1_dist = Z[None, :, :] - mu[:, None, :]  # NxMxQ
     _psi1_dist_sq = np.square(_psi1_dist) / (lengthscale2 * _psi1_denom) # NxMxQ
     _psi1_common = gamma[:,None,:] / (lengthscale2*_psi1_denom*_psi1_denom_sqrt) #Nx1xQ
-    _psi1_exponent1 = np.log(gamma[:,None,:]) -0.5 * (_psi1_dist_sq + np.log(_psi1_denom)) # NxMxQ
+    _psi1_exponent1 = np.log(gamma[:,None,:]) - (_psi1_dist_sq + np.log(_psi1_denom))/2. # NxMxQ
-    _psi1_exponent2 = np.log(1.-gamma[:,None,:]) -0.5 * (np.square(Z[None,:,:])/lengthscale2) # NxMxQ
+    _psi1_exponent2 = np.log(1.-gamma[:,None,:]) - (np.square(Z[None,:,:])/lengthscale2)/2. # NxMxQ
     _psi1_exponent_max = np.maximum(_psi1_exponent1,_psi1_exponent2)
     _psi1_exponent = _psi1_exponent_max+np.log(np.exp(_psi1_exponent1-_psi1_exponent_max) + np.exp(_psi1_exponent2-_psi1_exponent_max)) #NxMxQ
     _psi1_exp_sum = _psi1_exponent.sum(axis=-1) #NxM
-    _psi1_exp_dist_sq = np.exp(-0.5*_psi1_dist_sq) # NxMxQ
-    _psi1_exp_Z = np.exp(-0.5*np.square(Z[None,:,:])/lengthscale2) # 1xMxQ
-    _psi1_q = variance * np.exp(_psi1_exp_sum[:,:,None] - _psi1_exponent) # NxMxQ
     _psi1 = variance * np.exp(_psi1_exp_sum) # NxM
-    _dpsi1_dvariance = _psi1 / variance # NxM
-    _dpsi1_dgamma = _psi1_q * (_psi1_exp_dist_sq/_psi1_denom_sqrt-_psi1_exp_Z) # NxMxQ
-    _dpsi1_dmu = _psi1_q * (_psi1_exp_dist_sq * _psi1_dist * _psi1_common) # NxMxQ
-    _dpsi1_dS = _psi1_q * (_psi1_exp_dist_sq * _psi1_common * 0.5 * (_psi1_dist_sq - 1.)) # NxMxQ
-    _dpsi1_dZ = _psi1_q * (- _psi1_common * _psi1_dist * _psi1_exp_dist_sq - (1-gamma[:,None,:])/lengthscale2*Z[None,:,:]*_psi1_exp_Z) # NxMxQ
-    _dpsi1_dlengthscale = 2.*lengthscale*_psi1_q * (0.5*_psi1_common*(S[:,None,:]/lengthscale2+_psi1_dist_sq)*_psi1_exp_dist_sq + 0.5*(1-gamma[:,None,:])*np.square(Z[None,:,:]/lengthscale2)*_psi1_exp_Z) # NxMxQ
 
-    return _psi1, _dpsi1_dvariance, _dpsi1_dgamma, _dpsi1_dmu, _dpsi1_dS, _dpsi1_dZ, _dpsi1_dlengthscale
+    return _psi1
 
-@Cache_this(limit=1)
 def _psi2computations(variance, lengthscale, Z, mu, S, gamma):
     """
     Z - MxQ
@@ -66,19 +62,14 @@ def _psi2computations(variance, lengthscale, Z, mu, S, gamma):
     S - NxQ
     gamma - NxQ
     """
-    # here are the "statistics" for psi1 and psi2
+    # here are the "statistics" for psi2
     # Produced intermediate results:
-    # _psi2                NxMxM
+    # _psi2                MxM
-    # _psi2_dvariance      NxMxM
-    # _psi2_dlengthscale   NxMxMxQ
-    # _psi2_dZ             NxMxMxQ
-    # _psi2_dgamma         NxMxMxQ
-    # _psi2_dmu            NxMxMxQ
-    # _psi2_dS             NxMxMxQ
     
     lengthscale2 = np.square(lengthscale)
     
-    _psi2_Zhat, _psi2_Zdist = _Z_distances(Z)
+    _psi2_Zhat = 0.5 * (Z[:, None, :] + Z[None, :, :]) # M,M,Q
+    _psi2_Zdist = 0.5 * (Z[:, None, :] - Z[None, :, :]) # M,M,Q
     _psi2_Zdist_sq = np.square(_psi2_Zdist / lengthscale) # M,M,Q
     _psi2_Z_sq_sum = (np.square(Z[:,None,:])+np.square(Z[None,:,:]))/lengthscale2 # MxMxQ
 
@@ -93,15 +84,116 @@ def _psi2computations(variance, lengthscale, Z, mu, S, gamma):
     _psi2_exponent_max = np.maximum(_psi2_exponent1, _psi2_exponent2)
     _psi2_exponent = _psi2_exponent_max+np.log(np.exp(_psi2_exponent1-_psi2_exponent_max) + np.exp(_psi2_exponent2-_psi2_exponent_max))
     _psi2_exp_sum = _psi2_exponent.sum(axis=-1) #NxM
-    _psi2_q = np.square(variance) * np.exp(_psi2_exp_sum[:,:,:,None]-_psi2_exponent) # NxMxMxQ 
+    _psi2 = variance*variance * (np.exp(_psi2_exp_sum).sum(axis=0)) # MxM
+
+    return _psi2
+
+def psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior):
+    ARD = (len(lengthscale)!=1)
+    
+    dvar_psi1, dl_psi1, dZ_psi1, dmu_psi1, dS_psi1, dgamma_psi1 = _psi1compDer(dL_dpsi1, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
+    dvar_psi2, dl_psi2, dZ_psi2, dmu_psi2, dS_psi2, dgamma_psi2 = _psi2compDer(dL_dpsi2, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
+
+    dL_dvar = np.sum(dL_dpsi0) + dvar_psi1 + dvar_psi2
+    
+    dL_dlengscale = dl_psi1 + dl_psi2
+    if not ARD:
+        dL_dlengscale = dL_dlengscale.sum()
+
+    dL_dgamma = dgamma_psi1 + dgamma_psi2
+    dL_dmu = dmu_psi1 + dmu_psi2
+    dL_dS = dS_psi1 + dS_psi2
+    dL_dZ = dZ_psi1 + dZ_psi2
+    
+    return dL_dvar, dL_dlengscale, dL_dZ, dL_dmu, dL_dS, dL_dgamma
+
+def _psi1compDer(dL_dpsi1, variance, lengthscale, Z, mu, S, gamma):
+    """
+    dL_dpsi1 - NxM
+    Z - MxQ
+    mu - NxQ
+    S - NxQ
+    gamma - NxQ
+    """
+    # here are the "statistics" for psi1
+    # Produced intermediate results: dL_dparams w.r.t. psi1
+    # _dL_dvariance     1
+    # _dL_dlengthscale  Q
+    # _dL_dZ            MxQ
+    # _dL_dgamma        NxQ
+    # _dL_dmu           NxQ
+    # _dL_dS            NxQ
+    
+    lengthscale2 = np.square(lengthscale)
+
+    # psi1
+    _psi1_denom = S / lengthscale2 + 1.  # NxQ
+    _psi1_denom_sqrt = np.sqrt(_psi1_denom) #NxQ
+    _psi1_dist = Z[None, :, :] - mu[:, None, :]  # NxMxQ
+    _psi1_dist_sq = np.square(_psi1_dist) / (lengthscale2 * _psi1_denom[:,None,:]) # NxMxQ
+    _psi1_common = gamma / (lengthscale2*_psi1_denom*_psi1_denom_sqrt) #NxQ
+    _psi1_exponent1 = np.log(gamma[:,None,:]) -0.5 * (_psi1_dist_sq + np.log(_psi1_denom[:, None,:])) # NxMxQ
+    _psi1_exponent2 = np.log(1.-gamma[:,None,:]) -0.5 * (np.square(Z[None,:,:])/lengthscale2) # NxMxQ
+    _psi1_exponent_max = np.maximum(_psi1_exponent1,_psi1_exponent2)
+    _psi1_exponent = _psi1_exponent_max+np.log(np.exp(_psi1_exponent1-_psi1_exponent_max) + np.exp(_psi1_exponent2-_psi1_exponent_max)) #NxMxQ
+    _psi1_exp_sum = _psi1_exponent.sum(axis=-1) #NxM
+    _psi1_exp_dist_sq = np.exp(-0.5*_psi1_dist_sq) # NxMxQ
+    _psi1_exp_Z = np.exp(-0.5*np.square(Z[None,:,:])/lengthscale2) # 1xMxQ
+    _psi1_q = variance * np.exp(_psi1_exp_sum[:,:,None] - _psi1_exponent) # NxMxQ
+    _psi1 = variance * np.exp(_psi1_exp_sum) # NxM
+    _dL_dvariance = np.einsum('nm,nm->',dL_dpsi1, _psi1)/variance # 1
+    _dL_dgamma = np.einsum('nm,nmq,nmq->nq',dL_dpsi1, _psi1_q, (_psi1_exp_dist_sq/_psi1_denom_sqrt[:,None,:]-_psi1_exp_Z)) # NxQ
+    _dL_dmu = np.einsum('nm, nmq, nmq, nmq, nq->nq',dL_dpsi1,_psi1_q,_psi1_exp_dist_sq,_psi1_dist,_psi1_common)  # NxQ
+    _dL_dS = np.einsum('nm,nmq,nmq,nq,nmq->nq',dL_dpsi1,_psi1_q,_psi1_exp_dist_sq,_psi1_common,(_psi1_dist_sq-1.))/2.  # NxQ
+    _dL_dZ = np.einsum('nm,nmq,nmq->mq',dL_dpsi1,_psi1_q, (- _psi1_common[:,None,:] * _psi1_dist * _psi1_exp_dist_sq - (1-gamma[:,None,:])/lengthscale2*Z[None,:,:]*_psi1_exp_Z))
+    _dL_dlengthscale = lengthscale* np.einsum('nm,nmq,nmq->q',dL_dpsi1,_psi1_q,(_psi1_common[:,None,:]*(S[:,None,:]/lengthscale2+_psi1_dist_sq)*_psi1_exp_dist_sq + (1-gamma[:,None,:])*np.square(Z[None,:,:]/lengthscale2)*_psi1_exp_Z))
+
+    return _dL_dvariance, _dL_dlengthscale, _dL_dZ, _dL_dmu, _dL_dS, _dL_dgamma 
+
+def _psi2compDer(dL_dpsi2, variance, lengthscale, Z, mu, S, gamma):
+    """
+    Z - MxQ
+    mu - NxQ
+    S - NxQ
+    gamma - NxQ
+    dL_dpsi2 - MxM
+    """
+    # here are the "statistics" for psi2
+    # Produced the derivatives w.r.t. psi2:
+    # _dL_dvariance      1
+    # _dL_dlengthscale   Q
+    # _dL_dZ             MxQ
+    # _dL_dgamma         NxQ
+    # _dL_dmu            NxQ
+    # _dL_dS             NxQ
+    
+    lengthscale2 = np.square(lengthscale)
+    
+    _psi2_Zhat = 0.5 * (Z[:, None, :] + Z[None, :, :]) # M,M,Q
+    _psi2_Zdist = 0.5 * (Z[:, None, :] - Z[None, :, :]) # M,M,Q
+    _psi2_Zdist_sq = np.square(_psi2_Zdist / lengthscale) # M,M,Q
+    _psi2_Z_sq_sum = (np.square(Z[:,None,:])+np.square(Z[None,:,:]))/lengthscale2 # MxMxQ
+
+    # psi2
+    _psi2_denom = 2.*S / lengthscale2 + 1. # NxQ
+    _psi2_denom_sqrt = np.sqrt(_psi2_denom)
+    _psi2_mudist = mu[:,None,None,:]-_psi2_Zhat #N,M,M,Q
+    _psi2_mudist_sq = np.square(_psi2_mudist)/(lengthscale2*_psi2_denom[:,None,None,:])
+    _psi2_common = gamma/(lengthscale2 * _psi2_denom * _psi2_denom_sqrt) # NxQ
+    _psi2_exponent1 = -_psi2_Zdist_sq -_psi2_mudist_sq -0.5*np.log(_psi2_denom[:,None,None,:])+np.log(gamma[:,None,None,:]) #N,M,M,Q
+    _psi2_exponent2 = np.log(1.-gamma[:,None,None,:]) - 0.5*(_psi2_Z_sq_sum) # NxMxMxQ
+    _psi2_exponent_max = np.maximum(_psi2_exponent1, _psi2_exponent2)
+    _psi2_exponent = _psi2_exponent_max+np.log(np.exp(_psi2_exponent1-_psi2_exponent_max) + np.exp(_psi2_exponent2-_psi2_exponent_max))
+    _psi2_exp_sum = _psi2_exponent.sum(axis=-1) #NxM
+    _psi2_q = variance*variance * np.exp(_psi2_exp_sum[:,:,:,None]-_psi2_exponent) # NxMxMxQ 
     _psi2_exp_dist_sq = np.exp(-_psi2_Zdist_sq -_psi2_mudist_sq) # NxMxMxQ
     _psi2_exp_Z = np.exp(-0.5*_psi2_Z_sq_sum) # MxMxQ
-    _psi2 = np.square(variance) * np.exp(_psi2_exp_sum) # N,M,M
+    _psi2 = variance*variance * (np.exp(_psi2_exp_sum).sum(axis=0)) # MxM
-    _dpsi2_dvariance = 2. * _psi2/variance # NxMxM
+    _dL_dvariance = np.einsum('mo,mo->',dL_dpsi2,_psi2)*2./variance
-    _dpsi2_dgamma = _psi2_q * (_psi2_exp_dist_sq/_psi2_denom_sqrt - _psi2_exp_Z) # NxMxMxQ
+    _dL_dgamma = np.einsum('mo,nmoq,nmoq->nq',dL_dpsi2,_psi2_q,(_psi2_exp_dist_sq/_psi2_denom_sqrt[:,None,None,:] - _psi2_exp_Z))
-    _dpsi2_dmu = _psi2_q * (-2.*_psi2_common*_psi2_mudist * _psi2_exp_dist_sq) # NxMxMxQ
+    _dL_dmu = -2.*np.einsum('mo,nmoq,nq,nmoq,nmoq->nq',dL_dpsi2,_psi2_q,_psi2_common,_psi2_mudist,_psi2_exp_dist_sq)
-    _dpsi2_dS = _psi2_q * (_psi2_common * (2.*_psi2_mudist_sq - 1.) * _psi2_exp_dist_sq) # NxMxMxQ
+    _dL_dS = np.einsum('mo,nmoq,nq,nmoq,nmoq->nq',dL_dpsi2,_psi2_q, _psi2_common, (2.*_psi2_mudist_sq-1.), _psi2_exp_dist_sq)
-    _dpsi2_dZ = 2.*_psi2_q * (_psi2_common*(-_psi2_Zdist*_psi2_denom+_psi2_mudist)*_psi2_exp_dist_sq - (1-gamma[:,None,None,:])*Z[:,None,:]/lengthscale2*_psi2_exp_Z) # NxMxMxQ
+    _dL_dZ = 2.*np.einsum('mo,nmoq,nmoq->mq',dL_dpsi2,_psi2_q,(_psi2_common[:,None,None,:]*(-_psi2_Zdist*_psi2_denom[:,None,None,:]+_psi2_mudist)*_psi2_exp_dist_sq - (1-gamma[:,None,None,:])*Z[:,None,:]/lengthscale2*_psi2_exp_Z))
-    _dpsi2_dlengthscale = 2.*lengthscale* _psi2_q * (_psi2_common*(S[:,None,None,:]/lengthscale2+_psi2_Zdist_sq*_psi2_denom+_psi2_mudist_sq)*_psi2_exp_dist_sq+(1-gamma[:,None,None,:])*_psi2_Z_sq_sum*0.5/lengthscale2*_psi2_exp_Z) # NxMxMxQ
+    _dL_dlengthscale = 2.*lengthscale* np.einsum('mo,nmoq,nmoq->q',dL_dpsi2,_psi2_q,(_psi2_common[:,None,None,:]*(S[:,None,None,:]/lengthscale2+_psi2_Zdist_sq*_psi2_denom[:,None,None,:]+_psi2_mudist_sq)*_psi2_exp_dist_sq+(1-gamma[:,None,None,:])*_psi2_Z_sq_sum*0.5/lengthscale2*_psi2_exp_Z))
 
-    return _psi2, _dpsi2_dvariance, _dpsi2_dgamma, _dpsi2_dmu, _dpsi2_dS, _dpsi2_dZ, _dpsi2_dlengthscale
+    return _dL_dvariance, _dL_dlengthscale, _dL_dZ, _dL_dmu, _dL_dS, _dL_dgamma

GPy/kern/_src/psi_comp/ssrbf_psi_gpucomp.py

@@ -1,535 +1,474 @@
-# Copyright (c) 2012, GPy authors (see AUTHORS.txt).
-# Licensed under the BSD 3-clause license (see LICENSE.txt)
 
 """
-The package for the psi statistics computation on GPU
+The module for psi-statistics for RBF kernel for Spike-and-Slab GPLVM
 """
 
 import numpy as np
-from GPy.util.caching import Cache_this
+from ....util.caching import Cache_this
-
+from . import PSICOMP_RBF
 from ....util import gpu_init
 
 try:
     import pycuda.gpuarray as gpuarray
-    from scikits.cuda import cublas
+    from pycuda.compiler import SourceModule
-    from pycuda.reduction import ReductionKernel    
+    from ....util.linalg_gpu import sum_axis
-    from pycuda.elementwise import ElementwiseKernel
-    from ....util import linalg_gpu
-    
-    
-    # The kernel form computing psi1 het_noise
-    comp_psi1 = ElementwiseKernel(
-        "double *psi1, double var, double *l, double *Z, double *mu, double *S, double *logGamma, double *log1Gamma, double *logpsi1denom, int N, int M, int Q",
-        "psi1[i] = comp_psi1_element(var, l, Z, mu, S, logGamma, log1Gamma, logpsi1denom, N, M, Q, i)",
-        "comp_psi1",
-        preamble="""
-        #define IDX_NMQ(n,m,q) ((q*M+m)*N+n)
-        #define IDX_NQ(n,q) (q*N+n)
-        #define IDX_MQ(m,q) (q*M+m)
-        #define LOGEXPSUM(a,b) (a>=b?a+log(1.0+exp(b-a)):b+log(1.0+exp(a-b)))
-        
-        __device__ double comp_psi1_element(double var, double *l, double *Z, double *mu, double *S, double *logGamma, double *log1Gamma, double *logpsi1denom, int N, int M, int Q, int idx)
-        {
-            int n = idx%N;
-            int m = idx/N;
-            double psi1_exp=0;
-            for(int q=0;q<Q;q++){
-                double muZ = mu[IDX_NQ(n,q)]-Z[IDX_MQ(m,q)];
-                double exp1 = logGamma[IDX_NQ(n,q)] - (logpsi1denom[IDX_NQ(n,q)] + muZ*muZ/(S[IDX_NQ(n,q)]+l[q]) )/2.0;
-                double exp2 = log1Gamma[IDX_NQ(n,q)] - Z[IDX_MQ(m,q)]*Z[IDX_MQ(m,q)]/(l[q]*2.0);
-                psi1_exp += LOGEXPSUM(exp1,exp2);
-            }
-            return var*exp(psi1_exp);
-        }
-        """)
-        
-    # The kernel form computing psi2 het_noise
-    comp_psi2 = ElementwiseKernel(
-        "double *psi2, double var, double *l, double *Z, double *mu, double *S, double *logGamma, double *log1Gamma, double *logpsi2denom, int N, int M, int Q",
-        "psi2[i] = comp_psi2_element(var, l, Z, mu, S, logGamma, log1Gamma, logpsi2denom, N, M, Q, i)",
-        "comp_psi2",
-        preamble="""
-        #define IDX_NMQ(n,m,q) ((q*M+m)*N+n)
-        #define IDX_NQ(n,q) (q*N+n)
-        #define IDX_MQ(m,q) (q*M+m)
-        #define LOGEXPSUM(a,b) (a>=b?a+log(1.0+exp(b-a)):b+log(1.0+exp(a-b)))
-        
-        __device__ double comp_psi2_element(double var, double *l, double *Z, double *mu, double *S, double *logGamma, double *log1Gamma, double *logpsi2denom, int N, int M, int Q, int idx)
-        {
-            // psi2 (n,m1,m2)
-            int m2 = idx/(M*N);
-            int m1 = (idx%(M*N))/N;
-            int n = idx%N;
-
-            double psi2_exp=0;
-            for(int q=0;q<Q;q++){ 
-                double dZ = Z[IDX_MQ(m1,q)]-Z[IDX_MQ(m2,q)];
-                double muZ = mu[IDX_NQ(n,q)] - (Z[IDX_MQ(m1,q)]+Z[IDX_MQ(m2,q)])/2.0;
-                double exp1 = logGamma[IDX_NQ(n,q)] - (logpsi2denom[IDX_NQ(n,q)])/2.0 - dZ*dZ/(l[q]*4.0) - muZ*muZ/(2*S[IDX_NQ(n,q)]+l[q]);
-                double exp2 = log1Gamma[IDX_NQ(n,q)] - (Z[IDX_MQ(m1,q)]*Z[IDX_MQ(m1,q)]+Z[IDX_MQ(m2,q)]*Z[IDX_MQ(m2,q)])/(l[q]*2.0);
-                psi2_exp += LOGEXPSUM(exp1,exp2);
-            }
-            return var*var*exp(psi2_exp);
-        }
-        """) 
-    
-    # compute psidenom
-    comp_logpsidenom = ElementwiseKernel(
-        "double *out, double *S, double *l, double scale, int N",
-        "out[i] = comp_logpsidenom_element(S, l, scale, N, i)",
-        "comp_logpsidenom",
-        preamble="""        
-        __device__ double comp_logpsidenom_element(double *S, double *l, double scale, int N, int idx)
-        {
-            int q = idx/N;
-            
-            return log(scale*S[idx]/l[q]+1.0);
-        }
-        """)
-    
-    # The kernel form computing psi1 het_noise
-    comp_dpsi1_dvar = ElementwiseKernel(
-        "double *dpsi1_dvar, double *psi1_neq, double *psi1exp1, double *psi1exp2, double *l, double *Z, double *mu, double *S, double *logGamma, double *log1Gamma, double *logpsi1denom, int N, int M, int Q",
-        "dpsi1_dvar[i] = comp_dpsi1_dvar_element(psi1_neq, psi1exp1, psi1exp2, l, Z, mu, S, logGamma, log1Gamma, logpsi1denom, N, M, Q, i)",
-        "comp_dpsi1_dvar",
-        preamble="""
-        #define IDX_NMQ(n,m,q) ((q*M+m)*N+n)
-        #define IDX_NQ(n,q) (q*N+n)
-        #define IDX_MQ(m,q) (q*M+m)
-        #define LOGEXPSUM(a,b) (a>=b?a+log(1.0+exp(b-a)):b+log(1.0+exp(a-b)))
-        
-        __device__ double comp_dpsi1_dvar_element(double *psi1_neq, double *psi1exp1, double *psi1exp2, double *l, double *Z, double *mu, double *S, double *logGamma, double *log1Gamma, double *logpsi1denom, int N, int M, int Q, int idx)
-        {
-            int n = idx%N;
-            int m = idx/N;
-            
-            double psi1_sum = 0;
-            for(int q=0;q<Q;q++){            
-                double muZ = mu[IDX_NQ(n,q)]-Z[IDX_MQ(m,q)];
-                double exp1_e = -(muZ*muZ/(S[IDX_NQ(n,q)]+l[q]) )/2.0;
-                double exp1 = logGamma[IDX_NQ(n,q)] - (logpsi1denom[IDX_NQ(n,q)])/2.0 + exp1_e;
-                double exp2_e = - Z[IDX_MQ(m,q)]*Z[IDX_MQ(m,q)]/(l[q]*2.0);
-                double exp2 = log1Gamma[IDX_NQ(n,q)] + exp2_e;
-                double psi1_q = LOGEXPSUM(exp1,exp2);
-                psi1_neq[IDX_NMQ(n,m,q)] = -psi1_q;
-                psi1exp1[IDX_NMQ(n,m,q)] = exp(exp1_e);
-                psi1exp2[IDX_MQ(m,q)] = exp(exp2_e);
-                psi1_sum += psi1_q;
-            }
-            for(int q=0;q<Q;q++) {
-                psi1_neq[IDX_NMQ(n,m,q)] = exp(psi1_neq[IDX_NMQ(n,m,q)]+psi1_sum);
-            }
-            return exp(psi1_sum);
-        }
-        """)
-    
-    # The kernel form computing psi1 het_noise
-    comp_psi1_der = ElementwiseKernel(
-        "double *dpsi1_dl, double *dpsi1_dmu, double *dpsi1_dS, double *dpsi1_dgamma, double *dpsi1_dZ, double *psi1_neq, double *psi1exp1, double *psi1exp2, double var, double *l, double *Z, double *mu, double *S, double *gamma, int N, int M, int Q",
-        "dpsi1_dl[i] = comp_psi1_der_element(dpsi1_dmu, dpsi1_dS, dpsi1_dgamma, dpsi1_dZ, psi1_neq, psi1exp1, psi1exp2, var, l, Z, mu, S, gamma, N, M, Q, i)",
-        "comp_psi1_der",
-        preamble="""
-        #define IDX_NMQ(n,m,q) ((q*M+m)*N+n)
-        #define IDX_NQ(n,q) (q*N+n)
-        #define IDX_MQ(m,q) (q*M+m)
-        
-        __device__ double comp_psi1_der_element(double *dpsi1_dmu, double *dpsi1_dS, double *dpsi1_dgamma, double *dpsi1_dZ, double *psi1_neq, double *psi1exp1, double *psi1exp2, double var, double *l, double *Z, double *mu, double *S, double *gamma, int N, int M, int Q, int idx)
-        {
-            int q = idx/(M*N);
-            int m = (idx%(M*N))/N;
-            int n = idx%N;
-            
-            double neq = psi1_neq[IDX_NMQ(n,m,q)];
-            double gamma_c = gamma[IDX_NQ(n,q)];
-            double Z_c = Z[IDX_MQ(m,q)];
-            double S_c = S[IDX_NQ(n,q)];
-            double l_c = l[q];
-            double l_sqrt_c = sqrt(l[q]);
-            double psi1exp1_c = psi1exp1[IDX_NMQ(n,m,q)];
-            double psi1exp2_c = psi1exp2[IDX_MQ(m,q)];
-
-            double denom = S_c/l_c+1.0;
-            double denom_sqrt = sqrt(denom);
-            double Zmu = Z_c-mu[IDX_NQ(n,q)];
-            double psi1_common = gamma_c/(denom_sqrt*denom*l_c);
-            double gamma1 = 1-gamma_c;
-            
-            dpsi1_dgamma[IDX_NMQ(n,m,q)] = var*neq*(psi1exp1_c/denom_sqrt - psi1exp2_c);
-            dpsi1_dmu[IDX_NMQ(n,m,q)] = var*neq*(psi1_common*Zmu*psi1exp1_c);
-            dpsi1_dS[IDX_NMQ(n,m,q)] = var*neq*(psi1_common*(Zmu*Zmu/(S_c+l_c)-1.0)*psi1exp1_c)/2.0;
-            dpsi1_dZ[IDX_NMQ(n,m,q)] = var*neq*(-psi1_common*Zmu*psi1exp1_c-gamma1*Z_c/l_c*psi1exp2_c);
-            return var*neq*(psi1_common*(S_c/l_c+Zmu*Zmu/(S_c+l_c))*psi1exp1_c+gamma1*Z_c*Z_c/l_c*psi1exp2_c)*l_sqrt_c;    
-        }
-        """)
-    
-    # The kernel form computing psi1 het_noise
-    comp_dpsi2_dvar = ElementwiseKernel(
-        "double *dpsi2_dvar, double *psi2_neq, double *psi2exp1, double *psi2exp2, double var, double *l, double *Z, double *mu, double *S, double *logGamma, double *log1Gamma, double *logpsi2denom, int N, int M, int Q",
-        "dpsi2_dvar[i] = comp_dpsi2_dvar_element(psi2_neq, psi2exp1, psi2exp2, var, l, Z, mu, S, logGamma, log1Gamma, logpsi2denom, N, M, Q, i)",
-        "comp_dpsi2_dvar",
-        preamble="""
-        #define IDX_NMMQ(n,m1,m2,q) (((q*M+m2)*M+m1)*N+n)
-        #define IDX_MMQ(m1,m2,q) ((q*M+m2)*M+m1)
-        #define IDX_NMQ(n,m,q) ((q*M+m)*N+n)
-        #define IDX_NQ(n,q) (q*N+n)
-        #define IDX_MQ(m,q) (q*M+m)
-        #define LOGEXPSUM(a,b) (a>=b?a+log(1.0+exp(b-a)):b+log(1.0+exp(a-b)))
-        
-        __device__ double comp_dpsi2_dvar_element(double *psi2_neq, double *psi2exp1, double *psi2exp2, double var, double *l, double *Z, double *mu, double *S, double *logGamma, double *log1Gamma, double *logpsi2denom, int N, int M, int Q, int idx)
-        {
-            // psi2 (n,m1,m2)
-            int m2 = idx/(M*N);
-            int m1 = (idx%(M*N))/N;
-            int n = idx%N;
-
-            double psi2_sum=0;
-            for(int q=0;q<Q;q++){ 
-                double dZ = Z[IDX_MQ(m1,q)]-Z[IDX_MQ(m2,q)];
-                double muZ = mu[IDX_NQ(n,q)] - (Z[IDX_MQ(m1,q)]+Z[IDX_MQ(m2,q)])/2.0;
-                double exp1_e = - dZ*dZ/(l[q]*4.0) - muZ*muZ/(2*S[IDX_NQ(n,q)]+l[q]);
-                double exp1 = logGamma[IDX_NQ(n,q)] - (logpsi2denom[IDX_NQ(n,q)])/2.0 +exp1_e;
-                double exp2_e = - (Z[IDX_MQ(m1,q)]*Z[IDX_MQ(m1,q)]+Z[IDX_MQ(m2,q)]*Z[IDX_MQ(m2,q)])/(l[q]*2.0);
-                double exp2 = log1Gamma[IDX_NQ(n,q)] + exp2_e;
-                double psi2_q = LOGEXPSUM(exp1,exp2);
-                psi2_neq[IDX_NMMQ(n,m1,m2,q)] = -psi2_q;
-                psi2exp1[IDX_NMMQ(n,m1,m2,q)] = exp(exp1_e);
-                psi2exp2[IDX_MMQ(m1,m2,q)] = exp(exp2_e);
-                psi2_sum += psi2_q;
-            }
-            for(int q=0;q<Q;q++) {
-                psi2_neq[IDX_NMMQ(n,m1,m2,q)] = exp(psi2_neq[IDX_NMMQ(n,m1,m2,q)]+psi2_sum);
-            }
-            return 2*var*exp(psi2_sum);            
-        }
-        """)
-    
-    # The kernel form computing psi1 het_noise
-    comp_psi2_der = ElementwiseKernel(
-        "double *dpsi2_dl, double *dpsi2_dmu, double *dpsi2_dS, double *dpsi2_dgamma, double *dpsi2_dZ, double *psi2_neq, double *psi2exp1, double *psi2exp2, double var, double *l, double *Z, double *mu, double *S, double *gamma, int N, int M, int Q",
-        "dpsi2_dl[i] = comp_psi2_der_element(dpsi2_dmu, dpsi2_dS, dpsi2_dgamma, dpsi2_dZ, psi2_neq, psi2exp1, psi2exp2, var, l, Z, mu, S, gamma, N, M, Q, i)",
-        "comp_psi2_der",
-        preamble="""
-        #define IDX_NMMQ(n,m1,m2,q) (((q*M+m2)*M+m1)*N+n)
-        #define IDX_MMQ(m1,m2,q) ((q*M+m2)*M+m1)
-        #define IDX_NMQ(n,m,q) ((q*M+m)*N+n)
-        #define IDX_NQ(n,q) (q*N+n)
-        #define IDX_MQ(m,q) (q*M+m)
-        
-        __device__ double comp_psi2_der_element(double *dpsi2_dmu, double *dpsi2_dS, double *dpsi2_dgamma, double *dpsi2_dZ, double *psi2_neq, double *psi2exp1, double *psi2exp2, double var, double *l, double *Z, double *mu, double *S, double *gamma, int N, int M, int Q, int idx)
-        {
-            // dpsi2 (n,m1,m2,q)
-            int q = idx/(M*M*N);
-            int m2 = (idx%(M*M*N))/(M*N);
-            int m1 = (idx%(M*N))/N;
-            int n = idx%N;
-            
-            double neq = psi2_neq[IDX_NMMQ(n,m1,m2,q)];
-            double gamma_c = gamma[IDX_NQ(n,q)];
-            double Z1_c = Z[IDX_MQ(m1,q)];
-            double Z2_c = Z[IDX_MQ(m2,q)];
-            double S_c = S[IDX_NQ(n,q)];
-            double l_c = l[q];
-            double l_sqrt_c = sqrt(l[q]);
-            double psi2exp1_c = psi2exp1[IDX_NMMQ(n,m1,m2,q)];
-            double psi2exp2_c = psi2exp2[IDX_MMQ(m1,m2,q)];
-
-            double dZ = Z2_c - Z1_c;
-            double muZ = mu[IDX_NQ(n,q)] - (Z1_c+Z2_c)/2.0;
-            double Z2 = Z1_c*Z1_c+Z2_c*Z2_c;
-            double denom = 2.0*S_c/l_c+1.0;
-            double denom_sqrt = sqrt(denom);
-            double psi2_common = gamma_c/(denom_sqrt*denom*l_c);
-            double gamma1 = 1-gamma_c;
-            double var2 = var*var;
-            
-            dpsi2_dgamma[IDX_NMMQ(n,m1,m2,q)] = var2*neq*(psi2exp1_c/denom_sqrt - psi2exp2_c);
-            dpsi2_dmu[IDX_NMMQ(n,m1,m2,q)] = var2*neq*(-2.0*psi2_common*muZ*psi2exp1_c);
-            dpsi2_dS[IDX_NMMQ(n,m1,m2,q)] = var2*neq*(psi2_common*(2.0*muZ*muZ/(2.0*S_c+l_c)-1.0)*psi2exp1_c);
-            dpsi2_dZ[IDX_NMMQ(n,m1,m2,q)] = var2*neq*(psi2_common*(dZ*denom/-2.0+muZ)*psi2exp1_c-gamma1*Z2_c/l_c*psi2exp2_c)*2.0;
-            return var2*neq*(psi2_common*(S_c/l_c+dZ*dZ*denom/(4.0*l_c)+muZ*muZ/(2.0*S_c+l_c))*psi2exp1_c+gamma1*Z2/(2.0*l_c)*psi2exp2_c)*l_sqrt_c*2.0;    
-        }
-        """)
-        
 except:
     pass    
 
-class PSICOMP_SSRBF(object):
+gpu_code = """
-    def __init__(self):
+    // define THREADNUM
-        assert gpu_init.initSuccess, "GPU initialization failed!"
+
-        self.cublas_handle = gpu_init.cublas_handle
+    #define IDX_NMQ(n,m,q) ((q*M+m)*N+n)
+    #define IDX_NMM(n,m1,m2) ((m2*M+m1)*N+n)
+    #define IDX_NQ(n,q) (q*N+n)
+    #define IDX_NM(n,m) (m*N+n)
+    #define IDX_MQ(m,q) (q*M+m)
+    #define IDX_MM(m1,m2) (m2*M+m1)
+    #define IDX_NQB(n,q,b) ((b*Q+q)*N+n)
+    #define IDX_QB(q,b) (b*Q+q)
+
+    // Divide data evenly
+    __device__ void divide_data(int total_data, int psize, int pidx, int *start, int *end) {
+        int residue = (total_data)%psize;
+        if(pidx<residue) {
+            int size = total_data/psize+1;
+            *start = size*pidx;
+            *end = *start+size;
+        } else {
+            int size = total_data/psize;
+            *start = size*pidx+residue;
+            *end = *start+size;
+        }
+    }
+    
+    __device__ void reduce_sum(double* array, int array_size) {
+        int s;
+        if(array_size >= blockDim.x) {
+            for(int i=blockDim.x+threadIdx.x; i<array_size; i+= blockDim.x) {
+                array[threadIdx.x] += array[i];
+            }
+            array_size = blockDim.x;
+        }
+        __syncthreads();
+        for(int i=1; i<=array_size;i*=2) {s=i;}
+        if(threadIdx.x < array_size-s) {array[threadIdx.x] += array[s+threadIdx.x];}
+        __syncthreads();
+        for(s=s/2;s>=1;s=s/2) {
+            if(threadIdx.x < s) {array[threadIdx.x] += array[s+threadIdx.x];}
+            __syncthreads();
+        }
+    }
+
+    __global__ void compDenom(double *log_denom1, double *log_denom2, double *log_gamma, double*log_gamma1, double *gamma, double *l, double *S, int N, int Q)
+    {
+        int n_start, n_end;
+        divide_data(N, gridDim.x, blockIdx.x, &n_start, &n_end);
+        
+        for(int i=n_start*Q+threadIdx.x; i<n_end*Q; i+=blockDim.x) {
+            int n=i/Q;
+            int q=i%Q;
+
+            double Snq = S[IDX_NQ(n,q)];
+            double lq = l[q]*l[q];
+            double gnq = gamma[IDX_NQ(n,q)];
+            log_denom1[IDX_NQ(n,q)] = log(Snq/lq+1.);
+            log_denom2[IDX_NQ(n,q)] = log(2.*Snq/lq+1.);
+            log_gamma[IDX_NQ(n,q)] = log(gnq);
+            log_gamma1[IDX_NQ(n,q)] = log(1.-gnq);
+        }
+    }
+
+    __global__ void psi1computations(double *psi1, double *log_denom1, double *log_gamma, double*log_gamma1, double var, double *l, double *Z, double *mu, double *S, int N, int M, int Q)
+    {
+        int m_start, m_end;
+        divide_data(M, gridDim.x, blockIdx.x, &m_start, &m_end);
+        
+        for(int m=m_start; m<m_end; m++) {
+            for(int n=threadIdx.x; n<N; n+= blockDim.x) {            
+                double log_psi1 = 0;
+                for(int q=0;q<Q;q++) {
+                    double Zmq = Z[IDX_MQ(m,q)];
+                    double muZ = mu[IDX_NQ(n,q)]-Zmq;
+                    double Snq = S[IDX_NQ(n,q)];
+                    double lq = l[q]*l[q];
+                    double exp1 = log_gamma[IDX_NQ(n,q)]-(muZ*muZ/(Snq+lq)+log_denom1[IDX_NQ(n,q)])/(2.);
+                    double exp2 = log_gamma1[IDX_NQ(n,q)]-Zmq*Zmq/(2.*lq);
+                    log_psi1 += (exp1>exp2)?exp1+log1p(exp(exp2-exp1)):exp2+log1p(exp(exp1-exp2));
+                }
+                psi1[IDX_NM(n,m)] = var*exp(log_psi1);
+            }
+        }
+    }
+    
+    __global__ void psi2computations(double *psi2, double *psi2n, double *log_denom2, double *log_gamma, double*log_gamma1, double var, double *l, double *Z, double *mu, double *S, int N, int M, int Q)
+    {
+        int psi2_idx_start, psi2_idx_end;
+        __shared__ double psi2_local[THREADNUM];
+        divide_data((M+1)*M/2, gridDim.x, blockIdx.x, &psi2_idx_start, &psi2_idx_end);
+        
+        for(int psi2_idx=psi2_idx_start; psi2_idx<psi2_idx_end; psi2_idx++) {
+            int m1 = int((sqrt(8.*psi2_idx+1.)-1.)/2.);
+            int m2 = psi2_idx - (m1+1)*m1/2;
+            
+            psi2_local[threadIdx.x] = 0;
+            for(int n=threadIdx.x;n<N;n+=blockDim.x) {
+                double log_psi2_n = 0;
+                for(int q=0;q<Q;q++) {
+                    double Zm1q = Z[IDX_MQ(m1,q)];
+                    double Zm2q = Z[IDX_MQ(m2,q)];
+                    double dZ = Zm1q - Zm2q;
+                    double muZhat = mu[IDX_NQ(n,q)]- (Zm1q+Zm2q)/2.;
+                    double Z2 = Zm1q*Zm1q+Zm2q*Zm2q;
+                    double Snq = S[IDX_NQ(n,q)];
+                    double lq = l[q]*l[q];
+                    double exp1 = dZ*dZ/(-4.*lq)-muZhat*muZhat/(2.*Snq+lq) - log_denom2[IDX_NQ(n,q)]/2. + log_gamma[IDX_NQ(n,q)];
+                    double exp2 = log_gamma1[IDX_NQ(n,q)] - Z2/(2.*lq);
+                    log_psi2_n += (exp1>exp2)?exp1+log1p(exp(exp2-exp1)):exp2+log1p(exp(exp1-exp2));
+                }
+                double exp_psi2_n = exp(log_psi2_n);
+                psi2n[IDX_NMM(n,m1,m2)] = var*var*exp_psi2_n;
+                if(m1!=m2) { psi2n[IDX_NMM(n,m2,m1)] = var*var*exp_psi2_n;}
+                psi2_local[threadIdx.x] += exp_psi2_n;
+            }
+            __syncthreads();
+            reduce_sum(psi2_local, THREADNUM);
+            if(threadIdx.x==0) {
+                psi2[IDX_MM(m1,m2)] = var*var*psi2_local[0];
+                if(m1!=m2) { psi2[IDX_MM(m2,m1)] = var*var*psi2_local[0]; }
+            }
+            __syncthreads();
+        }
+    }
+    
+    __global__ void psi1compDer(double *dvar, double *dl, double *dZ, double *dmu, double *dS, double *dgamma, double *dL_dpsi1, double *psi1, double *log_denom1, double *log_gamma, double*log_gamma1, double var, double *l, double *Z, double *mu, double *S, double *gamma, int N, int M, int Q)
+    {
+        int m_start, m_end;
+        __shared__ double g_local[THREADNUM];
+        divide_data(M, gridDim.x, blockIdx.x, &m_start, &m_end);
+        int P = int(ceil(double(N)/THREADNUM));
+
+        double dvar_local = 0;
+        for(int q=0;q<Q;q++) {
+            double lq_sqrt = l[q];
+            double lq = lq_sqrt*lq_sqrt;
+            double dl_local = 0;
+            for(int p=0;p<P;p++) {
+                int n = p*THREADNUM + threadIdx.x;
+                double dmu_local = 0;
+                double dS_local = 0;
+                double dgamma_local = 0;
+                double Snq,mu_nq,gnq,log_gnq,log_gnq1,log_de;
+                if(n<N) {Snq = S[IDX_NQ(n,q)]; mu_nq=mu[IDX_NQ(n,q)]; gnq = gamma[IDX_NQ(n,q)];
+                        log_gnq = log_gamma[IDX_NQ(n,q)]; log_gnq1 = log_gamma1[IDX_NQ(n,q)];
+                        log_de = log_denom1[IDX_NQ(n,q)];}
+                for(int m=m_start; m<m_end; m++) {
+                    if(n<N) {
+                        double lpsi1 = psi1[IDX_NM(n,m)]*dL_dpsi1[IDX_NM(n,m)];
+                        if(q==0) {dvar_local += lpsi1;}
+                        
+                        double Zmq = Z[IDX_MQ(m,q)];
+                        double Zmu = Zmq - mu_nq;
+                        double denom = Snq+lq;
+                        double Zmu2_denom = Zmu*Zmu/denom;
+                        
+                        double exp1 = log_gnq-(Zmu*Zmu/(Snq+lq)+log_de)/(2.);
+                        double exp2 = log_gnq1-Zmq*Zmq/(2.*lq);
+                        double d_exp1,d_exp2;
+                        if(exp1>exp2) {
+                            d_exp1 = 1.;
+                            d_exp2 = exp(exp2-exp1);
+                        } else {
+                            d_exp1 = exp(exp1-exp2);
+                            d_exp2 = 1.;
+                        }
+                        double exp_sum = d_exp1+d_exp2;
+                        
+                        dmu_local += lpsi1*Zmu*d_exp1/(denom*exp_sum);
+                        dS_local += lpsi1*(Zmu2_denom-1.)*d_exp1/(denom*exp_sum);
+                        dgamma_local += lpsi1*(d_exp1/gnq-d_exp2/(1.-gnq))/exp_sum;
+                        dl_local += lpsi1*((Zmu2_denom+Snq/lq)/denom*d_exp1+Zmq*Zmq/(lq*lq)*d_exp2)/(2.*exp_sum);
+                        g_local[threadIdx.x] = lpsi1*(-Zmu/denom*d_exp1-Zmq/lq*d_exp2)/exp_sum;
+                    }
+                    __syncthreads();
+                    reduce_sum(g_local, p<P-1?THREADNUM:N-(P-1)*THREADNUM);
+                    if(threadIdx.x==0) {dZ[IDX_MQ(m,q)] += g_local[0];}
+                }
+                if(n<N) {
+                    dmu[IDX_NQB(n,q,blockIdx.x)] += dmu_local;
+                    dS[IDX_NQB(n,q,blockIdx.x)] += dS_local/2.;
+                    dgamma[IDX_NQB(n,q,blockIdx.x)] += dgamma_local;
+                }
+                __threadfence_block();
+            }
+            g_local[threadIdx.x] = dl_local*2.*lq_sqrt;
+            __syncthreads();
+            reduce_sum(g_local, THREADNUM);
+            if(threadIdx.x==0) {dl[IDX_QB(q,blockIdx.x)] += g_local[0];}
+        }
+        g_local[threadIdx.x] = dvar_local;
+        __syncthreads();
+        reduce_sum(g_local, THREADNUM);
+        if(threadIdx.x==0) {dvar[blockIdx.x] += g_local[0]/var;}
+    }
+    
+    __global__ void psi2compDer(double *dvar, double *dl, double *dZ, double *dmu, double *dS, double *dgamma, double *dL_dpsi2, double *psi2n, double *log_denom2, double *log_gamma, double*log_gamma1, double var, double *l, double *Z, double *mu, double *S, double *gamma, int N, int M, int Q)
+    {
+        int m_start, m_end;
+        __shared__ double g_local[THREADNUM];
+        divide_data(M, gridDim.x, blockIdx.x, &m_start, &m_end);
+        int P = int(ceil(double(N)/THREADNUM));
+
+        double dvar_local = 0;
+        for(int q=0;q<Q;q++) {
+            double lq_sqrt = l[q];
+            double lq = lq_sqrt*lq_sqrt;
+            double dl_local = 0;
+            for(int p=0;p<P;p++) {
+                int n = p*THREADNUM + threadIdx.x;
+                double dmu_local = 0;
+                double dS_local = 0;
+                double dgamma_local = 0;
+                double Snq,mu_nq,gnq,log_gnq,log_gnq1,log_de;
+                if(n<N) {Snq = S[IDX_NQ(n,q)]; mu_nq=mu[IDX_NQ(n,q)]; gnq = gamma[IDX_NQ(n,q)];
+                        log_gnq = log_gamma[IDX_NQ(n,q)]; log_gnq1 = log_gamma1[IDX_NQ(n,q)];
+                        log_de = log_denom2[IDX_NQ(n,q)];}
+                for(int m1=m_start; m1<m_end; m1++) {
+                    g_local[threadIdx.x] = 0;
+                    for(int m2=0;m2<M;m2++) {
+                        if(n<N) {
+                            double lpsi2 = psi2n[IDX_NMM(n,m1,m2)]*dL_dpsi2[IDX_MM(m1,m2)];
+                            if(q==0) {dvar_local += lpsi2;}
+                            
+                            double Zm1q = Z[IDX_MQ(m1,q)];
+                            double Zm2q = Z[IDX_MQ(m2,q)];
+                            double dZ = Zm1q - Zm2q;
+                            double Z2 = Zm1q*Zm1q+Zm2q*Zm2q;
+                            double muZhat =  mu_nq - (Zm1q + Zm2q)/2.;
+                            double denom = 2.*Snq+lq;
+                            double muZhat2_denom = muZhat*muZhat/denom;
+                            
+                            double exp1 = dZ*dZ/(-4.*lq)-muZhat*muZhat/(2.*Snq+lq) - log_de/2. + log_gnq;
+                            double exp2 = log_gnq1 - Z2/(2.*lq);
+                            double d_exp1,d_exp2;
+                            if(exp1>exp2) {
+                                d_exp1 = 1.;
+                                d_exp2 = exp(exp2-exp1);
+                            } else {
+                                d_exp1 = exp(exp1-exp2);
+                                d_exp2 = 1.;
+                            }
+                            double exp_sum = d_exp1+d_exp2;
+                            
+                            dmu_local += lpsi2*muZhat/denom*d_exp1/exp_sum;
+                            dS_local += lpsi2*(2.*muZhat2_denom-1.)/denom*d_exp1/exp_sum;
+                            dgamma_local += lpsi2*(d_exp1/gnq-d_exp2/(1.-gnq))/exp_sum;
+                            dl_local += lpsi2*(((Snq/lq+muZhat2_denom)/denom+dZ*dZ/(4.*lq*lq))*d_exp1+Z2/(2.*lq*lq)*d_exp2)/exp_sum;
+                            g_local[threadIdx.x] += 2.*lpsi2*((muZhat/denom-dZ/(2*lq))*d_exp1-Zm1q/lq*d_exp2)/exp_sum;
+                        }
+                    }
+                    __syncthreads();
+                    reduce_sum(g_local, p<P-1?THREADNUM:N-(P-1)*THREADNUM);
+                    if(threadIdx.x==0) {dZ[IDX_MQ(m1,q)] += g_local[0];}
+                }
+                if(n<N) {
+                    dmu[IDX_NQB(n,q,blockIdx.x)] += -2.*dmu_local;
+                    dS[IDX_NQB(n,q,blockIdx.x)] += dS_local;
+                    dgamma[IDX_NQB(n,q,blockIdx.x)] += dgamma_local;
+                }
+                __threadfence_block();
+            }
+            g_local[threadIdx.x] = dl_local*2.*lq_sqrt;
+            __syncthreads();
+            reduce_sum(g_local, THREADNUM);
+            if(threadIdx.x==0) {dl[IDX_QB(q,blockIdx.x)] += g_local[0];}
+        }
+        g_local[threadIdx.x] = dvar_local;
+        __syncthreads();
+        reduce_sum(g_local, THREADNUM);
+        if(threadIdx.x==0) {dvar[blockIdx.x] += g_local[0]*2/var;}
+    }
+    """
+
+class PSICOMP_SSRBF_GPU(PSICOMP_RBF):
+
+    def __init__(self, threadnum=128, blocknum=15, GPU_direct=False):
+        self.GPU_direct = GPU_direct
         self.gpuCache = None
-        self.gpuCacheAll = None
+        
+        self.threadnum = threadnum
+        self.blocknum = blocknum
+        module = SourceModule("#define THREADNUM "+str(self.threadnum)+"\n"+gpu_code)
+        self.g_psi1computations = module.get_function('psi1computations')
+        self.g_psi1computations.prepare('PPPPdPPPPiii')
+        self.g_psi2computations = module.get_function('psi2computations')
+        self.g_psi2computations.prepare('PPPPPdPPPPiii')
+        self.g_psi1compDer = module.get_function('psi1compDer')
+        self.g_psi1compDer.prepare('PPPPPPPPPPPdPPPPPiii')
+        self.g_psi2compDer = module.get_function('psi2compDer')
+        self.g_psi2compDer.prepare('PPPPPPPPPPPdPPPPPiii')
+        self.g_compDenom = module.get_function('compDenom')
+        self.g_compDenom.prepare('PPPPPPPii')
+
+    def __deepcopy__(self, memo):
+        s = PSICOMP_SSRBF_GPU(threadnum=self.threadnum, blocknum=self.blocknum, GPU_direct=self.GPU_direct)
+        memo[id(self)] = s 
+        return s
 
     def _initGPUCache(self, N, M, Q):            
-        if self.gpuCache!=None and self.gpuCache['mu_gpu'].shape[0] == N:
+        if self.gpuCache == None:
-            return
+            self.gpuCache = {
-        
-        if self.gpuCacheAll!=None and self.gpuCacheAll['mu_gpu'].shape[0]<N: # Too small cache -> reallocate
-            self._releaseMemory()
-            
-        if self.gpuCacheAll == None:
-            self.gpuCacheAll = {
                              'l_gpu'                :gpuarray.empty((Q,),np.float64,order='F'),
                              'Z_gpu'                :gpuarray.empty((M,Q),np.float64,order='F'),
                              'mu_gpu'               :gpuarray.empty((N,Q),np.float64,order='F'),
                              'S_gpu'                :gpuarray.empty((N,Q),np.float64,order='F'),
                              'gamma_gpu'            :gpuarray.empty((N,Q),np.float64,order='F'),
-                             'logGamma_gpu'         :gpuarray.empty((N,Q),np.float64,order='F'),
-                             'log1Gamma_gpu'        :gpuarray.empty((N,Q),np.float64,order='F'),
-                             'logpsi1denom_gpu'      :gpuarray.empty((N,Q),np.float64,order='F'),
-                             'logpsi2denom_gpu'      :gpuarray.empty((N,Q),np.float64,order='F'),
-                             'psi0_gpu'             :gpuarray.empty((N,),np.float64,order='F'),
                              'psi1_gpu'             :gpuarray.empty((N,M),np.float64,order='F'),
-                             'psi2_gpu'             :gpuarray.empty((N,M,M),np.float64,order='F'),
+                             'psi2_gpu'             :gpuarray.empty((M,M),np.float64,order='F'),
-                             # derivatives psi1
+                             'psi2n_gpu'            :gpuarray.empty((N,M,M),np.float64,order='F'),
-                             'psi1_neq_gpu'         :gpuarray.empty((N,M,Q),np.float64, order='F'),
+                             'dL_dpsi1_gpu'         :gpuarray.empty((N,M),np.float64,order='F'),
-                             'psi1exp1_gpu'         :gpuarray.empty((N,M,Q),np.float64, order='F'),
+                             'dL_dpsi2_gpu'         :gpuarray.empty((M,M),np.float64,order='F'),
-                             'psi1exp2_gpu'         :gpuarray.empty((N,M,Q),np.float64, order='F'),
+                             'log_denom1_gpu'       :gpuarray.empty((N,Q),np.float64,order='F'),
-                             'dpsi1_dvar_gpu'       :gpuarray.empty((N,M),np.float64, order='F'),
+                             'log_denom2_gpu'       :gpuarray.empty((N,Q),np.float64,order='F'),
-                             'dpsi1_dl_gpu'         :gpuarray.empty((N,M,Q),np.float64, order='F'),
+                             'log_gamma_gpu'        :gpuarray.empty((N,Q),np.float64,order='F'),
-                             'dpsi1_dZ_gpu'         :gpuarray.empty((N,M,Q),np.float64, order='F'),
+                             'log_gamma1_gpu'       :gpuarray.empty((N,Q),np.float64,order='F'),
-                             'dpsi1_dgamma_gpu'     :gpuarray.empty((N,M,Q),np.float64, order='F'),
+                             # derivatives
-                             'dpsi1_dmu_gpu'        :gpuarray.empty((N,M,Q),np.float64, order='F'),
+                             'dvar_gpu'             :gpuarray.empty((self.blocknum,),np.float64, order='F'),
-                             'dpsi1_dS_gpu'         :gpuarray.empty((N,M,Q),np.float64, order='F'),
+                             'dl_gpu'               :gpuarray.empty((Q,self.blocknum),np.float64, order='F'),
-                             # derivatives psi2
+                             'dZ_gpu'               :gpuarray.empty((M,Q),np.float64, order='F'),
-                             'psi2_neq_gpu'         :gpuarray.empty((N,M,M,Q),np.float64, order='F'),
+                             'dmu_gpu'              :gpuarray.empty((N,Q,self.blocknum),np.float64, order='F'),
-                             'psi2exp1_gpu'         :gpuarray.empty((N,M,M,Q),np.float64, order='F'),
+                             'dS_gpu'               :gpuarray.empty((N,Q,self.blocknum),np.float64, order='F'),
-                             'psi2exp2_gpu'         :gpuarray.empty((M,M,Q),np.float64, order='F'),
+                             'dgamma_gpu'           :gpuarray.empty((N,Q,self.blocknum),np.float64, order='F'),
-                             'dpsi2_dvar_gpu'       :gpuarray.empty((N,M,M),np.float64, order='F'),
+                             # grad
-                             'dpsi2_dl_gpu'         :gpuarray.empty((N,M,M,Q),np.float64, order='F'),
+                             'grad_l_gpu'               :gpuarray.empty((Q,),np.float64, order='F'),
-                             'dpsi2_dZ_gpu'         :gpuarray.empty((N,M,M,Q),np.float64, order='F'),
+                             'grad_mu_gpu'              :gpuarray.empty((N,Q,),np.float64, order='F'),
-                             'dpsi2_dgamma_gpu'     :gpuarray.empty((N,M,M,Q),np.float64, order='F'),
+                             'grad_S_gpu'               :gpuarray.empty((N,Q,),np.float64, order='F'),
-                             'dpsi2_dmu_gpu'        :gpuarray.empty((N,M,M,Q),np.float64, order='F'),
+                             'grad_gamma_gpu'           :gpuarray.empty((N,Q,),np.float64, order='F'),
-                             'dpsi2_dS_gpu'         :gpuarray.empty((N,M,M,Q),np.float64, order='F'),
-                             # gradients
-                             'grad_l_gpu'           :gpuarray.empty((Q,),np.float64,order='F'),
-                             'grad_Z_gpu'           :gpuarray.empty((M,Q),np.float64,order='F'),
-                             'grad_mu_gpu'          :gpuarray.empty((N,Q),np.float64,order='F'),
-                             'grad_S_gpu'           :gpuarray.empty((N,Q),np.float64,order='F'),
-                             'grad_gamma_gpu'       :gpuarray.empty((N,Q),np.float64,order='F'),
                              }
-            self.gpuCache = self.gpuCacheAll
-        elif self.gpuCacheAll['mu_gpu'].shape[0]==N:
-            self.gpuCache = self.gpuCacheAll
         else:
-            # remap to a smaller cache
+            assert N==self.gpuCache['mu_gpu'].shape[0]
-            self.gpuCache = self.gpuCacheAll.copy()
+            assert M==self.gpuCache['Z_gpu'].shape[0]
-            Nlist=['mu_gpu','S_gpu','gamma_gpu','logGamma_gpu','log1Gamma_gpu','logpsi1denom_gpu','logpsi2denom_gpu','psi0_gpu','psi1_gpu','psi2_gpu',
+            assert Q==self.gpuCache['l_gpu'].shape[0]
-                   'psi1_neq_gpu','psi1exp1_gpu','psi1exp2_gpu','dpsi1_dvar_gpu','dpsi1_dl_gpu','dpsi1_dZ_gpu','dpsi1_dgamma_gpu','dpsi1_dmu_gpu',
-                   'dpsi1_dS_gpu','psi2_neq_gpu','psi2exp1_gpu','dpsi2_dvar_gpu','dpsi2_dl_gpu','dpsi2_dZ_gpu','dpsi2_dgamma_gpu','dpsi2_dmu_gpu','dpsi2_dS_gpu','grad_mu_gpu','grad_S_gpu','grad_gamma_gpu',]
-            oldN = self.gpuCacheAll['mu_gpu'].shape[0]
-            for v in Nlist:
-                u = self.gpuCacheAll[v]
-                self.gpuCache[v] = u.ravel()[:u.size/oldN*N].reshape(*((N,)+u.shape[1:]))
     
-    def _releaseMemory(self):
+    def sync_params(self, lengthscale, Z, mu, S, gamma):
-        if self.gpuCacheAll!=None:
+        if len(lengthscale)==1:
-            [v.gpudata.free() for v in self.gpuCacheAll.values()]
+            self.gpuCache['l_gpu'].fill(lengthscale)
-            self.gpuCacheAll = None
-            self.gpuCache = None
-    
-    def estimateMemoryOccupation(self, N, M, Q):
-        """
-        Estimate the best batch size.
-        N - the number of total datapoints
-        M - the number of inducing points
-        Q - the number of hidden (input) dimensions
-        return: the constant memory size, the memory occupation of batchsize=1
-        unit: GB
-        """
-        return (2.*Q+2.*M*Q+M*M*Q)*8./1024./1024./1024., (1.+2.*M+10.*Q+2.*M*M+8.*M*Q+7.*M*M*Q)*8./1024./1024./1024.
-
-    @Cache_this(limit=1,ignore_args=(0,))
-    def psicomputations(self, variance, lengthscale, Z, mu, S, gamma):
-        """Compute Psi statitsitcs"""
-        if isinstance(lengthscale, np.ndarray) and len(lengthscale)>1:
-            ARD = True
         else:
-            ARD = False
+            self.gpuCache['l_gpu'].set(np.asfortranarray(lengthscale))
+        self.gpuCache['Z_gpu'].set(np.asfortranarray(Z))
+        self.gpuCache['mu_gpu'].set(np.asfortranarray(mu))
+        self.gpuCache['S_gpu'].set(np.asfortranarray(S))
+        self.gpuCache['gamma_gpu'].set(np.asfortranarray(gamma))
+        N,Q = self.gpuCache['S_gpu'].shape
+        self.g_compDenom.prepared_call((self.blocknum,1),(self.threadnum,1,1), self.gpuCache['log_denom1_gpu'].gpudata,self.gpuCache['log_denom2_gpu'].gpudata,self.gpuCache['log_gamma_gpu'].gpudata,self.gpuCache['log_gamma1_gpu'].gpudata,self.gpuCache['gamma_gpu'].gpudata,self.gpuCache['l_gpu'].gpudata,self.gpuCache['S_gpu'].gpudata, np.int32(N), np.int32(Q))
         
-        N = mu.shape[0]
+    def reset_derivative(self):
-        M = Z.shape[0]
+        self.gpuCache['dvar_gpu'].fill(0.)
-        Q = mu.shape[1]
+        self.gpuCache['dl_gpu'].fill(0.)
+        self.gpuCache['dZ_gpu'].fill(0.)
+        self.gpuCache['dmu_gpu'].fill(0.)
+        self.gpuCache['dS_gpu'].fill(0.)
+        self.gpuCache['dgamma_gpu'].fill(0.)
+        self.gpuCache['grad_l_gpu'].fill(0.)
+        self.gpuCache['grad_mu_gpu'].fill(0.)
+        self.gpuCache['grad_S_gpu'].fill(0.)
+        self.gpuCache['grad_gamma_gpu'].fill(0.)
     
+    def get_dimensions(self, Z, variational_posterior):
+        return variational_posterior.mean.shape[0], Z.shape[0], Z.shape[1]
+
+    @Cache_this(limit=1, ignore_args=(0,))
+    def psicomputations(self, variance, lengthscale, Z, variational_posterior):
+        """
+        Z - MxQ
+        mu - NxQ
+        S - NxQ
+        """
+        N,M,Q = self.get_dimensions(Z, variational_posterior)
         self._initGPUCache(N,M,Q)
-        l_gpu = self.gpuCache['l_gpu']
+        self.sync_params(lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
-        Z_gpu = self.gpuCache['Z_gpu']
+        
-        mu_gpu = self.gpuCache['mu_gpu']
-        S_gpu = self.gpuCache['S_gpu']
-        gamma_gpu = self.gpuCache['gamma_gpu']
-        logGamma_gpu = self.gpuCache['logGamma_gpu']
-        log1Gamma_gpu = self.gpuCache['log1Gamma_gpu']
-        logpsi1denom_gpu = self.gpuCache['logpsi1denom_gpu']
-        logpsi2denom_gpu = self.gpuCache['logpsi2denom_gpu']
-        psi0_gpu = self.gpuCache['psi0_gpu']
         psi1_gpu = self.gpuCache['psi1_gpu']
         psi2_gpu = self.gpuCache['psi2_gpu']
+        psi2n_gpu = self.gpuCache['psi2n_gpu']
+        l_gpu = self.gpuCache['l_gpu']
+        Z_gpu = self.gpuCache['Z_gpu']
+        mu_gpu = self.gpuCache['mu_gpu']
+        S_gpu = self.gpuCache['S_gpu']
+        log_denom1_gpu = self.gpuCache['log_denom1_gpu']
+        log_denom2_gpu = self.gpuCache['log_denom2_gpu']
+        log_gamma_gpu = self.gpuCache['log_gamma_gpu']
+        log_gamma1_gpu = self.gpuCache['log_gamma1_gpu']
 
-        if ARD:
+        psi0 = np.empty((N,))
-            l_gpu.set(np.asfortranarray(lengthscale**2))
+        psi0[:] = variance
+        self.g_psi1computations.prepared_call((self.blocknum,1),(self.threadnum,1,1),psi1_gpu.gpudata, log_denom1_gpu.gpudata, log_gamma_gpu.gpudata, log_gamma1_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata, np.int32(N), np.int32(M), np.int32(Q))
+        self.g_psi2computations.prepared_call((self.blocknum,1),(self.threadnum,1,1),psi2_gpu.gpudata, psi2n_gpu.gpudata, log_denom2_gpu.gpudata, log_gamma_gpu.gpudata, log_gamma1_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata, np.int32(N), np.int32(M), np.int32(Q))
+        
+        if self.GPU_direct:
+            return psi0, psi1_gpu, psi2_gpu
         else:
-            l_gpu.fill(lengthscale*lengthscale)
+            return psi0, psi1_gpu.get(), psi2_gpu.get()
-        Z_gpu.set(np.asfortranarray(Z))
-        mu_gpu.set(np.asfortranarray(mu))
-        S_gpu.set(np.asfortranarray(S))
-        gamma_gpu.set(np.asfortranarray(gamma))
-        linalg_gpu.log(gamma_gpu,logGamma_gpu)
-        linalg_gpu.logOne(gamma_gpu,log1Gamma_gpu)
-        comp_logpsidenom(logpsi1denom_gpu, S_gpu,l_gpu,1.0,N)
-        comp_logpsidenom(logpsi2denom_gpu, S_gpu,l_gpu,2.0,N)
 
-        psi0_gpu.fill(variance)
+    @Cache_this(limit=1, ignore_args=(0,1,2,3))
-        comp_psi1(psi1_gpu, variance, l_gpu, Z_gpu, mu_gpu, S_gpu, logGamma_gpu, log1Gamma_gpu, logpsi1denom_gpu, N, M, Q)
+    def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior):
-        comp_psi2(psi2_gpu, variance, l_gpu, Z_gpu, mu_gpu, S_gpu, logGamma_gpu, log1Gamma_gpu, logpsi2denom_gpu, N, M, Q)
+        ARD = (len(lengthscale)!=1)
         
-        return psi0_gpu, psi1_gpu, psi2_gpu
+        N,M,Q = self.get_dimensions(Z, variational_posterior)
-    
+        psi1_gpu = self.gpuCache['psi1_gpu']
-    @Cache_this(limit=1,ignore_args=(0,))
+        psi2n_gpu = self.gpuCache['psi2n_gpu']
-    def _psiDercomputations(self, variance, lengthscale, Z, mu, S, gamma):
-        """Compute the derivatives w.r.t. Psi statistics"""        
-        N, M, Q = mu.shape[0],Z.shape[0], mu.shape[1]
-        
-        self._initGPUCache(N,M,Q)
         l_gpu = self.gpuCache['l_gpu']
         Z_gpu = self.gpuCache['Z_gpu']
         mu_gpu = self.gpuCache['mu_gpu']
         S_gpu = self.gpuCache['S_gpu']
         gamma_gpu = self.gpuCache['gamma_gpu']
-        logGamma_gpu = self.gpuCache['logGamma_gpu']
+        dvar_gpu = self.gpuCache['dvar_gpu']
-        log1Gamma_gpu = self.gpuCache['log1Gamma_gpu']
+        dl_gpu = self.gpuCache['dl_gpu']
-        logpsi1denom_gpu = self.gpuCache['logpsi1denom_gpu']
+        dZ_gpu = self.gpuCache['dZ_gpu']
-        logpsi2denom_gpu = self.gpuCache['logpsi2denom_gpu']
+        dmu_gpu = self.gpuCache['dmu_gpu']
-
+        dS_gpu = self.gpuCache['dS_gpu']
-        psi1_neq_gpu = self.gpuCache['psi1_neq_gpu']
+        dgamma_gpu = self.gpuCache['dgamma_gpu']
-        psi1exp1_gpu = self.gpuCache['psi1exp1_gpu']
-        psi1exp2_gpu = self.gpuCache['psi1exp2_gpu']
-        dpsi1_dvar_gpu = self.gpuCache['dpsi1_dvar_gpu']
-        dpsi1_dl_gpu = self.gpuCache['dpsi1_dl_gpu']
-        dpsi1_dZ_gpu = self.gpuCache['dpsi1_dZ_gpu']
-        dpsi1_dgamma_gpu = self.gpuCache['dpsi1_dgamma_gpu']
-        dpsi1_dmu_gpu = self.gpuCache['dpsi1_dmu_gpu']
-        dpsi1_dS_gpu = self.gpuCache['dpsi1_dS_gpu']
-
-        psi2_neq_gpu = self.gpuCache['psi2_neq_gpu']
-        psi2exp1_gpu = self.gpuCache['psi2exp1_gpu']
-        psi2exp2_gpu = self.gpuCache['psi2exp2_gpu']
-        dpsi2_dvar_gpu = self.gpuCache['dpsi2_dvar_gpu']
-        dpsi2_dl_gpu = self.gpuCache['dpsi2_dl_gpu']
-        dpsi2_dZ_gpu = self.gpuCache['dpsi2_dZ_gpu']
-        dpsi2_dgamma_gpu = self.gpuCache['dpsi2_dgamma_gpu']
-        dpsi2_dmu_gpu = self.gpuCache['dpsi2_dmu_gpu']
-        dpsi2_dS_gpu = self.gpuCache['dpsi2_dS_gpu']
-
-        #==========================================================================================================
-        # Assuming the l_gpu, Z_gpu, mu_gpu, S_gpu, gamma_gpu, logGamma_gpu, log1Gamma_gpu, 
-        # logpsi1denom_gpu, logpsi2denom_gpu has been synchonized.
-        #==========================================================================================================
-        
-        # psi1 derivatives
-        comp_dpsi1_dvar(dpsi1_dvar_gpu, psi1_neq_gpu, psi1exp1_gpu,psi1exp2_gpu, l_gpu, Z_gpu, mu_gpu, S_gpu, logGamma_gpu, log1Gamma_gpu, logpsi1denom_gpu, N, M, Q)
-        comp_psi1_der(dpsi1_dl_gpu,dpsi1_dmu_gpu,dpsi1_dS_gpu,dpsi1_dgamma_gpu, dpsi1_dZ_gpu, psi1_neq_gpu,psi1exp1_gpu,psi1exp2_gpu, variance, l_gpu, Z_gpu, mu_gpu, S_gpu, gamma_gpu, N, M, Q)
-
-        # psi2 derivatives
-        comp_dpsi2_dvar(dpsi2_dvar_gpu, psi2_neq_gpu, psi2exp1_gpu,psi2exp2_gpu, variance, l_gpu, Z_gpu, mu_gpu, S_gpu, logGamma_gpu, log1Gamma_gpu, logpsi2denom_gpu, N, M, Q)
-        comp_psi2_der(dpsi2_dl_gpu,dpsi2_dmu_gpu,dpsi2_dS_gpu,dpsi2_dgamma_gpu, dpsi2_dZ_gpu, psi2_neq_gpu,psi2exp1_gpu,psi2exp2_gpu, variance, l_gpu, Z_gpu, mu_gpu, S_gpu, gamma_gpu, N, M, Q)
-
-    def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior):
-        mu = variational_posterior.mean
-        S = variational_posterior.variance
-        gamma = variational_posterior.binary_prob
-        self._psiDercomputations(variance, lengthscale, Z, mu, S, gamma)
-        N, M, Q = mu.shape[0],Z.shape[0], mu.shape[1]
-        
-        if isinstance(lengthscale, np.ndarray) and len(lengthscale)>1:
-            ARD = True
-        else:
-            ARD = False
-            
-        dpsi1_dvar_gpu = self.gpuCache['dpsi1_dvar_gpu']
-        dpsi2_dvar_gpu = self.gpuCache['dpsi2_dvar_gpu']
-        dpsi1_dl_gpu = self.gpuCache['dpsi1_dl_gpu']
-        dpsi2_dl_gpu = self.gpuCache['dpsi2_dl_gpu']
-        psi1_comb_gpu = self.gpuCache['psi1_neq_gpu']
-        psi2_comb_gpu = self.gpuCache['psi2_neq_gpu']
         grad_l_gpu = self.gpuCache['grad_l_gpu']
-        
-        # variance
-        variance.gradient = gpuarray.sum(dL_dpsi0).get() \
-                            + cublas.cublasDdot(self.cublas_handle, dL_dpsi1.size, dL_dpsi1.gpudata, 1, dpsi1_dvar_gpu.gpudata, 1) \
-                            + cublas.cublasDdot(self.cublas_handle, dL_dpsi2.size, dL_dpsi2.gpudata, 1, dpsi2_dvar_gpu.gpudata, 1)
-
-        # lengscale
-        if ARD:
-            grad_l_gpu.fill(0.)
-            linalg_gpu.mul_bcast(psi1_comb_gpu, dL_dpsi1, dpsi1_dl_gpu, dL_dpsi1.size)
-            linalg_gpu.sum_axis(grad_l_gpu, psi1_comb_gpu, 1, N*M)
-            linalg_gpu.mul_bcast(psi2_comb_gpu, dL_dpsi2, dpsi2_dl_gpu, dL_dpsi2.size)
-            linalg_gpu.sum_axis(grad_l_gpu, psi2_comb_gpu, 1, N*M*M)            
-            lengthscale.gradient = grad_l_gpu.get()
-        else:
-            linalg_gpu.mul_bcast(psi1_comb_gpu, dL_dpsi1, dpsi1_dl_gpu, dL_dpsi1.size)
-            linalg_gpu.mul_bcast(psi2_comb_gpu, dL_dpsi2, dpsi2_dl_gpu, dL_dpsi2.size)
-            lengthscale.gradient = gpuarray.sum(psi1_comb_gpu).get() + gpuarray.sum(psi2_comb_gpu).get()
-                
-    def gradients_Z_expectations(self, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior):
-        mu = variational_posterior.mean
-        S = variational_posterior.variance
-        gamma = variational_posterior.binary_prob
-        self._psiDercomputations(variance, lengthscale, Z, mu, S, gamma)
-        N, M, Q = mu.shape[0],Z.shape[0], mu.shape[1]
-
-        dpsi1_dZ_gpu = self.gpuCache['dpsi1_dZ_gpu']
-        dpsi2_dZ_gpu = self.gpuCache['dpsi2_dZ_gpu']
-        psi1_comb_gpu = self.gpuCache['psi1_neq_gpu']
-        psi2_comb_gpu = self.gpuCache['psi2_neq_gpu']
-        grad_Z_gpu = self.gpuCache['grad_Z_gpu']
-
-        grad_Z_gpu.fill(0.)
-        linalg_gpu.mul_bcast(psi1_comb_gpu, dL_dpsi1, dpsi1_dZ_gpu, dL_dpsi1.size)
-        linalg_gpu.sum_axis(grad_Z_gpu, psi1_comb_gpu, 1, N)
-        linalg_gpu.mul_bcast(psi2_comb_gpu, dL_dpsi2, dpsi2_dZ_gpu, dL_dpsi2.size)
-        linalg_gpu.sum_axis(grad_Z_gpu, psi2_comb_gpu, 1, N*M)
-        return grad_Z_gpu.get()
-        
-    def gradients_qX_expectations(self, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior):
-        mu = variational_posterior.mean
-        S = variational_posterior.variance
-        gamma = variational_posterior.binary_prob
-        self._psiDercomputations(variance, lengthscale, Z, mu, S, gamma)
-        N, M, Q = mu.shape[0],Z.shape[0], mu.shape[1]
-
-        dpsi1_dmu_gpu = self.gpuCache['dpsi1_dmu_gpu']
-        dpsi2_dmu_gpu = self.gpuCache['dpsi2_dmu_gpu']
-        dpsi1_dS_gpu = self.gpuCache['dpsi1_dS_gpu']
-        dpsi2_dS_gpu = self.gpuCache['dpsi2_dS_gpu']
-        dpsi1_dgamma_gpu = self.gpuCache['dpsi1_dgamma_gpu']
-        dpsi2_dgamma_gpu = self.gpuCache['dpsi2_dgamma_gpu']
-        psi1_comb_gpu = self.gpuCache['psi1_neq_gpu']
-        psi2_comb_gpu = self.gpuCache['psi2_neq_gpu']
         grad_mu_gpu = self.gpuCache['grad_mu_gpu']
         grad_S_gpu = self.gpuCache['grad_S_gpu']
         grad_gamma_gpu = self.gpuCache['grad_gamma_gpu']
+        log_denom1_gpu = self.gpuCache['log_denom1_gpu']
+        log_denom2_gpu = self.gpuCache['log_denom2_gpu']
+        log_gamma_gpu = self.gpuCache['log_gamma_gpu']
+        log_gamma1_gpu = self.gpuCache['log_gamma1_gpu']
         
-        # mu gradients
+        if self.GPU_direct:
-        grad_mu_gpu.fill(0.)
+            dL_dpsi1_gpu = dL_dpsi1
-        linalg_gpu.mul_bcast(psi1_comb_gpu, dL_dpsi1, dpsi1_dmu_gpu, dL_dpsi1.size)
+            dL_dpsi2_gpu = dL_dpsi2
-        linalg_gpu.sum_axis(grad_mu_gpu, psi1_comb_gpu, N, M)
+            dL_dpsi0_sum = gpuarray.sum(dL_dpsi0).get()
-        linalg_gpu.mul_bcast(psi2_comb_gpu, dL_dpsi2, dpsi2_dmu_gpu, dL_dpsi2.size)
+        else:
-        linalg_gpu.sum_axis(grad_mu_gpu, psi2_comb_gpu, N, M*M)
+            dL_dpsi1_gpu = self.gpuCache['dL_dpsi1_gpu']
+            dL_dpsi2_gpu = self.gpuCache['dL_dpsi2_gpu']
+            dL_dpsi1_gpu.set(np.asfortranarray(dL_dpsi1))
+            dL_dpsi2_gpu.set(np.asfortranarray(dL_dpsi2))
+            dL_dpsi0_sum = dL_dpsi0.sum()
 
-        # S gradients
+        self.reset_derivative()
-        grad_S_gpu.fill(0.)
+        # t=self.g_psi1compDer(dvar_gpu,dl_gpu,dZ_gpu,dmu_gpu,dS_gpu,dL_dpsi1_gpu,psi1_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True)
-        linalg_gpu.mul_bcast(psi1_comb_gpu, dL_dpsi1, dpsi1_dS_gpu, dL_dpsi1.size)
+        # print 'g_psi1compDer '+str(t)
-        linalg_gpu.sum_axis(grad_S_gpu, psi1_comb_gpu, N, M)
+        # t=self.g_psi2compDer(dvar_gpu,dl_gpu,dZ_gpu,dmu_gpu,dS_gpu,dL_dpsi2_gpu,psi2n_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True)
-        linalg_gpu.mul_bcast(psi2_comb_gpu, dL_dpsi2, dpsi2_dS_gpu, dL_dpsi2.size)
+        # print 'g_psi2compDer '+str(t)
-        linalg_gpu.sum_axis(grad_S_gpu, psi2_comb_gpu, N, M*M)
+        self.g_psi1compDer.prepared_call((self.blocknum,1),(self.threadnum,1,1),dvar_gpu.gpudata,dl_gpu.gpudata,dZ_gpu.gpudata,dmu_gpu.gpudata,dS_gpu.gpudata,dgamma_gpu.gpudata,dL_dpsi1_gpu.gpudata,psi1_gpu.gpudata, log_denom1_gpu.gpudata, log_gamma_gpu.gpudata, log_gamma1_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata,gamma_gpu.gpudata,np.int32(N), np.int32(M), np.int32(Q))
+        self.g_psi2compDer.prepared_call((self.blocknum,1),(self.threadnum,1,1),dvar_gpu.gpudata,dl_gpu.gpudata,dZ_gpu.gpudata,dmu_gpu.gpudata,dS_gpu.gpudata,dgamma_gpu.gpudata,dL_dpsi2_gpu.gpudata,psi2n_gpu.gpudata, log_denom2_gpu.gpudata, log_gamma_gpu.gpudata, log_gamma1_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata,gamma_gpu.gpudata,np.int32(N), np.int32(M), np.int32(Q))
+
+        dL_dvar = dL_dpsi0_sum + gpuarray.sum(dvar_gpu).get()
+        sum_axis(grad_mu_gpu,dmu_gpu,N*Q,self.blocknum)
+        dL_dmu = grad_mu_gpu.get()
+        sum_axis(grad_S_gpu,dS_gpu,N*Q,self.blocknum)
+        dL_dS = grad_S_gpu.get()
+        sum_axis(grad_gamma_gpu,dgamma_gpu,N*Q,self.blocknum)
+        dL_dgamma = grad_gamma_gpu.get()
+        dL_dZ = dZ_gpu.get()
+        if ARD:
+            sum_axis(grad_l_gpu,dl_gpu,Q,self.blocknum)
+            dL_dlengscale = grad_l_gpu.get()
+        else:
+            dL_dlengscale = gpuarray.sum(dl_gpu).get()
+            
+        return dL_dvar, dL_dlengscale, dL_dZ, dL_dmu, dL_dS, dL_dgamma
     
-        # gamma gradients
-        grad_gamma_gpu.fill(0.)
-        linalg_gpu.mul_bcast(psi1_comb_gpu, dL_dpsi1, dpsi1_dgamma_gpu, dL_dpsi1.size)
-        linalg_gpu.sum_axis(grad_gamma_gpu, psi1_comb_gpu, N, M)
-        linalg_gpu.mul_bcast(psi2_comb_gpu, dL_dpsi2, dpsi2_dgamma_gpu, dL_dpsi2.size)
-        linalg_gpu.sum_axis(grad_gamma_gpu, psi2_comb_gpu, N, M*M)
 
-        return grad_mu_gpu.get(), grad_S_gpu.get(), grad_gamma_gpu.get()

GPy/kern/_src/rbf.py

@@ -3,13 +3,10 @@
 
 
 import numpy as np
-from scipy import weave
-from ...util.misc import param_to_array
 from stationary import Stationary
-from GPy.util.caching import Cache_this
+from psi_comp import PSICOMP_RBF
-from ...core.parameterization import variational
+from psi_comp.rbf_psi_gpucomp import PSICOMP_RBF_GPU
-from psi_comp import ssrbf_psi_comp
+from ...util.config import *
-from psi_comp.ssrbf_psi_gpucomp import PSICOMP_SSRBF
 
 class RBF(Stationary):
     """
@@ -25,10 +22,11 @@ class RBF(Stationary):
         super(RBF, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name, useGPU=useGPU)
         self.weave_options = {}
         self.group_spike_prob = False
-        
+        self.psicomp = PSICOMP_RBF()
         if self.useGPU:
-            self.psicomp = PSICOMP_SSRBF()
+            self.psicomp = PSICOMP_RBF_GPU()
-            
+        else:
+            self.psicomp = PSICOMP_RBF()
 
     def K_of_r(self, r):
         return self.variance * np.exp(-0.5 * r**2)
@@ -36,307 +34,40 @@ class RBF(Stationary):
     def dK_dr(self, r):
         return -r*self.K_of_r(r)
 
+    def __getstate__(self):
+        dc = super(RBF, self).__getstate__()
+        if self.useGPU:
+            dc['psicomp'] = PSICOMP_RBF()
+        return dc
+ 
+    def __setstate__(self, state):
+        return super(RBF, self).__setstate__(state)
+
+    def spectrum(self, omega):
+        assert self.input_dim == 1 #TODO: higher dim spectra?
+        return self.variance*np.sqrt(2*np.pi)*self.lengthscale*np.exp(-self.lengthscale*2*omega**2/2)
+
     #---------------------------------------#
     #             PSI statistics            #
     #---------------------------------------#
 
     def psi0(self, Z, variational_posterior):
-        if self.useGPU:
+        return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[0]
-            if isinstance(variational_posterior, variational.SpikeAndSlabPosterior):
-                return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[0]
-        else:
-            return self.Kdiag(variational_posterior.mean)
 
     def psi1(self, Z, variational_posterior):
-        if isinstance(variational_posterior, variational.SpikeAndSlabPosterior):
+        return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[1]
-            if self.useGPU:
-                return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[1]
-            else:
-                psi1, _, _, _, _, _, _ = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
-        else:
-            _, _, _, psi1 = self._psi1computations(Z, variational_posterior)
-        return psi1
 
     def psi2(self, Z, variational_posterior):
-        if isinstance(variational_posterior, variational.SpikeAndSlabPosterior):
+        return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[2]
-            if self.useGPU:
-                return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)[2]
-            else:
-                psi2, _, _, _, _, _, _ = ssrbf_psi_comp._psi2computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
-        else:
-            _, _, _, _, psi2 = self._psi2computations(Z, variational_posterior)
-        return psi2
 
     def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
-        # Spike-and-Slab GPLVM
+        dL_dvar, dL_dlengscale = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[:2]
-        if isinstance(variational_posterior, variational.SpikeAndSlabPosterior):
+        self.variance.gradient = dL_dvar
-            if self.useGPU:
+        self.lengthscale.gradient = dL_dlengscale
-                self.psicomp.update_gradients_expectations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)
-            else:
 
-                _, _dpsi1_dvariance, _, _, _, _, _dpsi1_dlengthscale = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
+    def gradients_Z_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
-                _, _dpsi2_dvariance, _, _, _, _, _dpsi2_dlengthscale = ssrbf_psi_comp._psi2computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
+        return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[2]
-    
-                #contributions from psi0:
-                self.variance.gradient = np.sum(dL_dpsi0)
-    
-                #from psi1
-                self.variance.gradient += np.sum(dL_dpsi1 * _dpsi1_dvariance)
-                if self.ARD:
-                    self.lengthscale.gradient = (dL_dpsi1[:,:,None]*_dpsi1_dlengthscale).reshape(-1,self.input_dim).sum(axis=0)
-                else:
-                    self.lengthscale.gradient = (dL_dpsi1[:,:,None]*_dpsi1_dlengthscale).sum()  
-    
-                #from psi2
-                self.variance.gradient += (dL_dpsi2 * _dpsi2_dvariance).sum()
-                if self.ARD:
-                    self.lengthscale.gradient += (dL_dpsi2[:,:,:,None] * _dpsi2_dlengthscale).reshape(-1,self.input_dim).sum(axis=0)
-                else:
-                    self.lengthscale.gradient += (dL_dpsi2[:,:,:,None] * _dpsi2_dlengthscale).sum()
-                    
-        elif isinstance(variational_posterior, variational.NormalPosterior):
-            l2 = self.lengthscale**2
-            if l2.size != self.input_dim:
-                l2 = l2*np.ones(self.input_dim)
-
-            #contributions from psi0:
-            self.variance.gradient = np.sum(dL_dpsi0)
-            self.lengthscale.gradient = 0.
-
-            #from psi1
-            denom, _, dist_sq, psi1 = self._psi1computations(Z, variational_posterior)
-            d_length = psi1[:,:,None] * ((dist_sq - 1.)/(self.lengthscale*denom) +1./self.lengthscale)
-            dpsi1_dlength = d_length * dL_dpsi1[:, :, None]
-            if self.ARD:
-                self.lengthscale.gradient += dpsi1_dlength.sum(0).sum(0)
-            else:
-                self.lengthscale.gradient += dpsi1_dlength.sum()
-            self.variance.gradient += np.sum(dL_dpsi1 * psi1) / self.variance
-            #from psi2
-            S = variational_posterior.variance
-            _, Zdist_sq, _, mudist_sq, psi2 = self._psi2computations(Z, variational_posterior)
-            if not self.ARD:
-                self.lengthscale.gradient += self._weave_psi2_lengthscale_grads(dL_dpsi2, psi2, Zdist_sq, S, mudist_sq, l2).sum()
-            else:
-                self.lengthscale.gradient += self._weave_psi2_lengthscale_grads(dL_dpsi2, psi2, Zdist_sq, S, mudist_sq, l2)
-
-            self.variance.gradient += 2.*np.sum(dL_dpsi2 * psi2)/self.variance
-
-        else:
-            raise ValueError, "unknown distriubtion received for psi-statistics"
-
-    def gradients_Z_expectations(self, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
-        # Spike-and-Slab GPLVM
-        if isinstance(variational_posterior, variational.SpikeAndSlabPosterior):
-            if self.useGPU:
-                return self.psicomp.gradients_Z_expectations(dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)
-            else:
-                _, _, _, _, _, _dpsi1_dZ, _ = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
-                _, _, _, _, _, _dpsi2_dZ, _ = ssrbf_psi_comp._psi2computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
-    
-                #psi1
-                grad = (dL_dpsi1[:, :, None] * _dpsi1_dZ).sum(axis=0)
-    
-                #psi2
-                grad += (dL_dpsi2[:, :, :, None] * _dpsi2_dZ).sum(axis=0).sum(axis=1)
-    
-                return grad
-
-        elif isinstance(variational_posterior, variational.NormalPosterior):
-            l2 = self.lengthscale **2
-
-            #psi1
-            denom, dist, dist_sq, psi1 = self._psi1computations(Z, variational_posterior)
-            grad = np.einsum('ij,ij,ijk,ijk->jk', dL_dpsi1, psi1, dist, -1./(denom*l2))
-
-            #psi2
-            Zdist, Zdist_sq, mudist, mudist_sq, psi2 = self._psi2computations(Z, variational_posterior)
-            term1 = Zdist / l2 # M, M, Q
-            S = variational_posterior.variance
-            term2 = mudist / (2.*S[:,None,None,:] + l2) # N, M, M, Q
-
-            grad += 2.*np.einsum('ijk,ijk,ijkl->kl', dL_dpsi2, psi2, term1[None,:,:,:] + term2)
-
-            return grad
-        else:
-            raise ValueError, "unknown distriubtion received for psi-statistics"
 
     def gradients_qX_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
-        # Spike-and-Slab GPLVM
+        return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[3:]
-        if isinstance(variational_posterior, variational.SpikeAndSlabPosterior):
-            if self.useGPU:
-                return self.psicomp.gradients_qX_expectations(dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)
-            else:   
-                ndata = variational_posterior.mean.shape[0]
 
-                _, _, _dpsi1_dgamma, _dpsi1_dmu, _dpsi1_dS, _, _ = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
-                _, _, _dpsi2_dgamma, _dpsi2_dmu, _dpsi2_dS, _, _ = ssrbf_psi_comp._psi2computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
-    
-                #psi1
-                grad_mu = (dL_dpsi1[:, :, None] * _dpsi1_dmu).sum(axis=1)
-                grad_S = (dL_dpsi1[:, :, None] * _dpsi1_dS).sum(axis=1)
-                grad_gamma = (dL_dpsi1[:,:,None] * _dpsi1_dgamma).sum(axis=1)
-    
-                #psi2
-                grad_mu += (dL_dpsi2[:, :, :, None] * _dpsi2_dmu).reshape(ndata,-1,self.input_dim).sum(axis=1)
-                grad_S += (dL_dpsi2[:, :, :, None] * _dpsi2_dS).reshape(ndata,-1,self.input_dim).sum(axis=1)
-                grad_gamma += (dL_dpsi2[:,:,:, None] * _dpsi2_dgamma).reshape(ndata,-1,self.input_dim).sum(axis=1)
-                
-                if self.group_spike_prob:
-                    grad_gamma[:] = grad_gamma.mean(axis=0)
-    
-                return grad_mu, grad_S, grad_gamma
-
-        elif isinstance(variational_posterior, variational.NormalPosterior):
-
-            l2 = self.lengthscale **2
-            #psi1
-            denom, dist, dist_sq, psi1 = self._psi1computations(Z, variational_posterior)
-            tmp = psi1[:, :, None] / l2 / denom
-            grad_mu = np.sum(dL_dpsi1[:, :, None] * tmp * dist, 1)
-            grad_S = np.sum(dL_dpsi1[:, :, None] * 0.5 * tmp * (dist_sq - 1), 1)
-            #psi2
-            _, _, mudist, mudist_sq, psi2 = self._psi2computations(Z, variational_posterior)
-            S = variational_posterior.variance
-            tmp = psi2[:, :, :, None] / (2.*S[:,None,None,:] + l2)
-            grad_mu += -2.*np.einsum('ijk,ijkl,ijkl->il', dL_dpsi2, tmp , mudist)
-            grad_S += np.einsum('ijk,ijkl,ijkl->il', dL_dpsi2 , tmp , (2.*mudist_sq - 1))
-
-        else:
-            raise ValueError, "unknown distriubtion received for psi-statistics"
-
-        return grad_mu, grad_S
-
-    #---------------------------------------#
-    #            Precomputations            #
-    #---------------------------------------#
-
-    @Cache_this(limit=1)
-    def _psi1computations(self, Z, vp):
-        mu, S = vp.mean, vp.variance
-        l2 = self.lengthscale **2
-        denom = S[:, None, :] / l2 + 1. # N,1,Q
-        dist = Z[None, :, :] - mu[:, None, :] # N,M,Q
-        dist_sq = np.square(dist) / l2 / denom # N,M,Q
-        exponent = -0.5 * np.sum(dist_sq + np.log(denom), -1)#N,M
-        psi1 = self.variance * np.exp(exponent) # N,M
-        return denom, dist, dist_sq, psi1
-
-
-    @Cache_this(limit=1, ignore_args=(0,))
-    def _Z_distances(self, Z):
-        Zhat = 0.5 * (Z[:, None, :] + Z[None, :, :]) # M,M,Q
-        Zdist = 0.5 * (Z[:, None, :] - Z[None, :, :]) # M,M,Q
-        return Zhat, Zdist
-
-    @Cache_this(limit=1)
-    def _psi2computations(self, Z, vp):
-        mu, S = vp.mean, vp.variance
-
-        N, Q = mu.shape
-        M = Z.shape[0]
-
-        #compute required distances
-        Zhat, Zdist = self._Z_distances(Z)
-        Zdist_sq = np.square(Zdist / self.lengthscale) # M,M,Q
-
-        #allocate memory for the things we want to compute
-        mudist = np.empty((N, M, M, Q))
-        mudist_sq = np.empty((N, M, M, Q))
-        psi2 = np.empty((N, M, M))
-
-        l2 = self.lengthscale **2
-        denom = (2.*S[:,None,None,:] / l2) + 1. # N,Q
-        half_log_denom = 0.5 * np.log(denom[:,0,0,:])
-        denom_l2 = denom[:,0,0,:]*l2
-
-        variance_sq = float(np.square(self.variance))
-        code = """
-        double tmp, exponent_tmp;
-
-        #pragma omp parallel for private(tmp, exponent_tmp)
-        for (int n=0; n<N; n++)
-        {
-            for (int m=0; m<M; m++)
-            {
-                for (int mm=0; mm<(m+1); mm++)
-                {
-                    exponent_tmp = 0.0;
-                    for (int q=0; q<Q; q++)
-                    {
-                        //compute mudist
-                        tmp = mu(n,q) - Zhat(m,mm,q);
-                        mudist(n,m,mm,q) = tmp;
-                        mudist(n,mm,m,q) = tmp;
-
-                        //now mudist_sq
-                        tmp = tmp*tmp/denom_l2(n,q);
-                        mudist_sq(n,m,mm,q) = tmp;
-                        mudist_sq(n,mm,m,q) = tmp;
-
-                        //now exponent
-                        tmp = -Zdist_sq(m,mm,q) - tmp - half_log_denom(n,q);
-                        exponent_tmp += tmp;
-                    }
-                    //compute psi2 by exponontiating
-                    psi2(n,m,mm) = variance_sq * exp(exponent_tmp);
-                    psi2(n,mm,m) = psi2(n,m,mm);
-                }
-            }
-        }
-        """
-
-        support_code = """
-        #include <omp.h>
-        #include <math.h>
-        """
-        mu = param_to_array(mu)
-        weave.inline(code, support_code=support_code, libraries=['gomp'],
-                     arg_names=['N', 'M', 'Q', 'mu', 'Zhat', 'mudist_sq', 'mudist', 'denom_l2', 'Zdist_sq', 'half_log_denom', 'psi2', 'variance_sq'],
-                     type_converters=weave.converters.blitz, **self.weave_options)
-
-        return Zdist, Zdist_sq, mudist, mudist_sq, psi2
-
-    def _weave_psi2_lengthscale_grads(self, dL_dpsi2, psi2, Zdist_sq, S, mudist_sq, l2):
-
-        #here's the einsum equivalent, it's ~3 times slower
-        #return 2.*np.einsum( 'ijk,ijk,ijkl,il->l', dL_dpsi2, psi2, Zdist_sq * (2.*S[:,None,None,:]/l2 + 1.) + mudist_sq + S[:, None, None, :] / l2, 1./(2.*S + l2))*self.lengthscale
-
-        result = np.zeros(self.input_dim)
-        code = """
-        double tmp;
-        for(int q=0; q<Q; q++)
-        {
-            tmp = 0.0;
-            #pragma omp parallel for reduction(+:tmp)
-            for(int n=0; n<N; n++)
-            {
-                for(int m=0; m<M; m++)
-                {
-                    //diag terms
-                    tmp += dL_dpsi2(n,m,m) * psi2(n,m,m) * (Zdist_sq(m,m,q) * (2.0*S(n,q)/l2(q) + 1.0) + mudist_sq(n,m,m,q) + S(n,q)/l2(q)) / (2.0*S(n,q) + l2(q)) ;
-
-                    //off-diag terms
-                    for(int mm=0; mm<m; mm++)
-                    {
-                        tmp += 2.0 * dL_dpsi2(n,m,mm) * psi2(n,m,mm) * (Zdist_sq(m,mm,q) * (2.0*S(n,q)/l2(q) + 1.0) + mudist_sq(n,m,mm,q) + S(n,q)/l2(q)) / (2.0*S(n,q) + l2(q)) ;
-                    }
-                }
-            }
-            result(q) = tmp;
-        }
-
-        """
-        support_code = """
-        #include <omp.h>
-        #include <math.h>
-        """
-        N,Q = S.shape
-        M = psi2.shape[-1]
-
-        S = param_to_array(S)
-        weave.inline(code, support_code=support_code, libraries=['gomp'],
-                     arg_names=['psi2', 'dL_dpsi2', 'N', 'M', 'Q', 'mudist_sq', 'l2', 'Zdist_sq', 'S', 'result'],
-                     type_converters=weave.converters.blitz, **self.weave_options)
-
-        return 2.*result*self.lengthscale

GPy/kern/_src/splitKern.py

@@ -0,0 +1,204 @@
+"""
+A new kernel
+"""
+
+import numpy as np
+from kern import Kern,CombinationKernel
+from .independent_outputs import index_to_slices
+import itertools
+
+class DiffGenomeKern(Kern):
+
+    def __init__(self, kernel, idx_p, Xp, index_dim=-1, name='DiffGenomeKern'):
+        self.idx_p = idx_p
+        self.index_dim=index_dim
+        self.kern = SplitKern(kernel,Xp, index_dim=index_dim)
+        super(DiffGenomeKern, self).__init__(input_dim=kernel.input_dim+1, active_dims=None, name=name)
+        self.add_parameter(self.kern)
+    
+    def K(self, X, X2=None):
+        assert X2==None
+        K = self.kern.K(X,X2)
+        
+        if self.idx_p<=0 or self.idx_p>X.shape[0]/2:
+            return K
+        
+        slices = index_to_slices(X[:,self.index_dim])
+        idx_start = slices[1][0].start
+        idx_end = idx_start+self.idx_p
+        K_c = K[idx_start:idx_end,idx_start:idx_end].copy()
+        K[idx_start:idx_end,:] = K[:self.idx_p,:]
+        K[:,idx_start:idx_end] = K[:,:self.idx_p]
+        K[idx_start:idx_end,idx_start:idx_end] = K_c
+        
+        return K
+    
+    def Kdiag(self,X):
+        Kdiag = self.kern.Kdiag(X)
+
+        if self.idx_p<=0 or self.idx_p>X.shape[0]/2:
+            return Kdiag
+
+        slices = index_to_slices(X[:,self.index_dim])
+        idx_start = slices[1][0].start
+        idx_end = idx_start+self.idx_p
+        Kdiag[idx_start:idx_end] = Kdiag[:self.idx_p]
+        
+        return Kdiag
+    
+    def update_gradients_full(self,dL_dK,X,X2=None):
+        assert X2==None
+        if self.idx_p<=0 or self.idx_p>X.shape[0]/2:
+            self.kern.update_gradients_full(dL_dK, X)
+            return
+        
+        slices = index_to_slices(X[:,self.index_dim])
+        idx_start = slices[1][0].start
+        idx_end = idx_start+self.idx_p
+        
+        self.kern.update_gradients_full(dL_dK[idx_start:idx_end,:], X[:self.idx_p],X)
+        grad_p1 = self.kern.gradient.copy()
+        self.kern.update_gradients_full(dL_dK[:,idx_start:idx_end], X, X[:self.idx_p])
+        grad_p2 = self.kern.gradient.copy()
+        self.kern.update_gradients_full(dL_dK[idx_start:idx_end,idx_start:idx_end], X[:self.idx_p],X[idx_start:idx_end])
+        grad_p3 = self.kern.gradient.copy()
+        self.kern.update_gradients_full(dL_dK[idx_start:idx_end,idx_start:idx_end], X[idx_start:idx_end], X[:self.idx_p])
+        grad_p4 = self.kern.gradient.copy()
+
+        self.kern.update_gradients_full(dL_dK[idx_start:idx_end,:], X[idx_start:idx_end],X)
+        grad_n1 = self.kern.gradient.copy()
+        self.kern.update_gradients_full(dL_dK[:,idx_start:idx_end], X, X[idx_start:idx_end])
+        grad_n2 = self.kern.gradient.copy()
+        self.kern.update_gradients_full(dL_dK[idx_start:idx_end,idx_start:idx_end], X[idx_start:idx_end], X[idx_start:idx_end])
+        grad_n3 = self.kern.gradient.copy()
+
+        self.kern.update_gradients_full(dL_dK, X)
+        self.kern.gradient += grad_p1+grad_p2-grad_p3-grad_p4-grad_n1-grad_n2+2*grad_n3
+
+    def update_gradients_diag(self, dL_dKdiag, X):
+        pass
+
+class SplitKern(CombinationKernel):
+
+    def __init__(self, kernel, Xp, index_dim=-1, name='SplitKern'):
+        assert isinstance(index_dim, int), "The index dimension must be an integer!"
+        self.kern = kernel
+        self.kern_cross = SplitKern_cross(kernel,Xp)
+        super(SplitKern, self).__init__(kernels=[self.kern, self.kern_cross], extra_dims=[index_dim], name=name)
+        self.index_dim = index_dim
+
+    def K(self,X ,X2=None):
+        slices = index_to_slices(X[:,self.index_dim])
+        assert len(slices)<=2, 'The Split kernel only support two different indices'
+        if X2 is None:
+            target = np.zeros((X.shape[0], X.shape[0]))
+            # diagonal blocks
+            [[target.__setitem__((s,ss), self.kern.K(X[s,:], X[ss,:])) for s,ss in itertools.product(slices_i, slices_i)] for slices_i in slices]
+            if len(slices)>1:
+                # cross blocks
+                [target.__setitem__((s,ss), self.kern_cross.K(X[s,:], X[ss,:])) for s,ss in itertools.product(slices[0], slices[1])]
+                # cross blocks
+                [target.__setitem__((s,ss), self.kern_cross.K(X[s,:], X[ss,:])) for s,ss in itertools.product(slices[1], slices[0])]
+        else:
+            slices2 = index_to_slices(X2[:,self.index_dim])
+            assert len(slices2)<=2, 'The Split kernel only support two different indices'
+            target = np.zeros((X.shape[0], X2.shape[0]))
+            # diagonal blocks
+            [[target.__setitem__((s,s2), self.kern.K(X[s,:],X2[s2,:])) for s,s2 in itertools.product(slices[i], slices2[i])] for i in xrange(min(len(slices),len(slices2)))]
+            if len(slices)>1:
+                [target.__setitem__((s,s2), self.kern_cross.K(X[s,:],X2[s2,:])) for s,s2 in itertools.product(slices[1], slices2[0])]
+            if len(slices2)>1:
+                [target.__setitem__((s,s2), self.kern_cross.K(X[s,:],X2[s2,:])) for s,s2 in itertools.product(slices[0], slices2[1])]                
+        return target
+
+    def Kdiag(self,X):
+        return self.kern.Kdiag(X)
+
+    def update_gradients_full(self,dL_dK,X,X2=None):
+        slices = index_to_slices(X[:,self.index_dim])
+        target = np.zeros(self.kern.size)
+
+        def collate_grads(dL, X, X2, cross=False):
+            if cross:
+                self.kern_cross.update_gradients_full(dL,X,X2)
+                target[:] += self.kern_cross.kern.gradient
+            else:
+                self.kern.update_gradients_full(dL,X,X2)
+                target[:] += self.kern.gradient
+    
+        if X2 is None:
+            assert dL_dK.shape==(X.shape[0],X.shape[0])
+            [[collate_grads(dL_dK[s,ss], X[s], X[ss]) for s,ss in itertools.product(slices_i, slices_i)] for slices_i in slices]
+            if len(slices)>1:
+                [collate_grads(dL_dK[s,ss], X[s], X[ss], True) for s,ss in itertools.product(slices[0], slices[1])]
+                [collate_grads(dL_dK[s,ss], X[s], X[ss], True) for s,ss in itertools.product(slices[1], slices[0])]
+        else:
+            assert dL_dK.shape==(X.shape[0],X2.shape[0])
+            slices2 = index_to_slices(X2[:,self.index_dim])
+            [[collate_grads(dL_dK[s,s2],X[s],X2[s2]) for s,s2 in itertools.product(slices[i], slices2[i])] for i in xrange(min(len(slices),len(slices2)))]
+            if len(slices)>1:
+                [collate_grads(dL_dK[s,s2], X[s], X2[s2], True) for s,s2 in itertools.product(slices[1], slices2[0])]
+            if len(slices2)>1:
+                [collate_grads(dL_dK[s,s2], X[s], X2[s2], True) for s,s2 in itertools.product(slices[0], slices2[1])]
+        self.kern.gradient = target
+
+    def update_gradients_diag(self, dL_dKdiag, X):
+        self.kern.update_gradients_diag(self, dL_dKdiag, X)
+
+class SplitKern_cross(Kern):
+
+    def __init__(self, kernel, Xp, name='SplitKern_cross'):
+        assert isinstance(kernel, Kern)
+        self.kern = kernel
+        if not isinstance(Xp,np.ndarray):
+            Xp = np.array([[Xp]])
+        self.Xp = Xp
+        super(SplitKern_cross, self).__init__(input_dim=kernel.input_dim, active_dims=None, name=name)
+        
+    def K(self, X, X2=None):
+        if X2 is None:
+            return np.dot(self.kern.K(X,self.Xp),self.kern.K(self.Xp,X))/self.kern.K(self.Xp,self.Xp)
+        else:
+            return np.dot(self.kern.K(X,self.Xp),self.kern.K(self.Xp,X2))/self.kern.K(self.Xp,self.Xp)
+        
+    def Kdiag(self, X):
+        return np.inner(self.kern.K(X,self.Xp),self.kern.K(self.Xp,X).T)/self.kern.K(self.Xp,self.Xp)
+
+    def update_gradients_full(self, dL_dK, X, X2=None):
+        if X2 is None:
+            X2 = X
+                        
+        k1 = self.kern.K(X,self.Xp)
+        k2 = self.kern.K(self.Xp,X2)
+        k3 = self.kern.K(self.Xp,self.Xp)
+        dL_dk1 = np.einsum('ij,j->i',dL_dK,k2[0])/k3[0,0]
+        dL_dk2 = np.einsum('ij,i->j',dL_dK,k1[:,0])/k3[0,0]
+        dL_dk3 = np.einsum('ij,ij->',dL_dK,-np.dot(k1,k2)/(k3[0,0]*k3[0,0]))
+
+        self.kern.update_gradients_full(dL_dk1[:,None],X,self.Xp)
+        grad = self.kern.gradient.copy()
+        self.kern.update_gradients_full(dL_dk2[None,:],self.Xp,X2)
+        grad += self.kern.gradient.copy()
+        self.kern.update_gradients_full(np.array([[dL_dk3]]),self.Xp,self.Xp)
+        grad += self.kern.gradient.copy()
+        
+        self.kern.gradient = grad
+
+    def update_gradients_diag(self, dL_dKdiag, X):
+        k1 = self.kern.K(X,self.Xp)
+        k2 = self.kern.K(self.Xp,X)
+        k3 = self.kern.K(self.Xp,self.Xp)
+        dL_dk1 = dL_dKdiag*k2[0]/k3
+        dL_dk2 = dL_dKdiag*k1[:,0]/k3
+        dL_dk3 = -dL_dKdiag*(k1[:,0]*k2[0]).sum()/(k3*k3)
+        
+        self.kern.update_gradients_full(dL_dk1[:,None],X,self.Xp)
+        grad1 = self.kern.gradient.copy()
+        self.kern.update_gradients_full(dL_dk2[None,:],self.Xp,X)
+        grad2 = self.kern.gradient.copy()
+        self.kern.update_gradients_full(np.array([[dL_dk3]]),self.Xp,self.Xp)
+        grad3 = self.kern.gradient.copy()
+        
+        self.kern.gradient = grad1+grad2+grad3
+        
+

GPy/kern/_src/ssrbf.py

@@ -1,139 +0,0 @@
-# Copyright (c) 2012, GPy authors (see AUTHORS.txt).
-# Licensed under the BSD 3-clause license (see LICENSE.txt)
-
-
-from kern import Kern
-import numpy as np
-from ...util.linalg import tdot
-from ...util.config import *
-from stationary import Stationary
-from psi_comp import ssrbf_psi_comp
-
-class SSRBF(Stationary):
-    """
-    Radial Basis Function kernel, aka squared-exponential, exponentiated quadratic or Gaussian kernel
-    for Spike-and-Slab GPLVM
-
-    .. math::
-
-       k(r) = \sigma^2 \exp \\bigg(- \\frac{1}{2} r^2 \\bigg) \ \ \ \ \  \\text{ where  } r^2 = \sum_{i=1}^d \\frac{ (x_i-x^\prime_i)^2}{\ell_i^2}
-
-    where \ell_i is the lengthscale, \sigma^2 the variance and d the dimensionality of the input.
-
-    :param input_dim: the number of input dimensions
-    :type input_dim: int
-    :param variance: the variance of the kernel
-    :type variance: float
-    :param lengthscale: the vector of lengthscale of the kernel
-    :type lengthscale: array or list of the appropriate size (or float if there is only one lengthscale parameter)
-    :param ARD: Auto Relevance Determination. If equal to "False", the kernel is isotropic (ie. one single lengthscale parameter \ell), otherwise there is one lengthscale parameter per dimension.
-    :type ARD: Boolean
-    :rtype: kernel object
-
-    .. Note: this object implements both the ARD and 'spherical' version of the function
-    """
-
-    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=True, active_dims=None, name='SSRBF'):
-        assert ARD==True, "Not Implemented!"
-        super(SSRBF, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name)
-        
-    def K_of_r(self, r):
-        return self.variance * np.exp(-0.5 * r**2)
-
-    def dK_dr(self, r):
-        return -r*self.K_of_r(r)
-
-    def parameters_changed(self):
-        pass
-
-    def Kdiag(self, X):
-        ret = np.empty(X.shape[0])
-        ret[:] = self.variance
-        return ret
-        
-    #---------------------------------------#
-    #             PSI statistics            #
-    #---------------------------------------#
-    
-    def psi0(self, Z, variational_posterior):
-        ret = np.empty(variational_posterior.mean.shape[0])
-        ret[:] = self.variance
-        return ret
-
-    def psi1(self, Z, variational_posterior):
-        _psi1, _, _, _, _, _, _ = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
-        return _psi1
-
-    def psi2(self, Z, variational_posterior):
-        _psi2, _, _, _, _, _, _ = ssrbf_psi_comp._psi2computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
-        return _psi2
-
-    def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
-        _, _dpsi1_dvariance, _, _, _, _, _dpsi1_dlengthscale = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
-        _, _dpsi2_dvariance, _, _, _, _, _dpsi2_dlengthscale = ssrbf_psi_comp._psi2computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
-
-        #contributions from psi0:
-        self.variance.gradient = np.sum(dL_dpsi0)
-
-        #from psi1
-        self.variance.gradient += np.sum(dL_dpsi1 * _dpsi1_dvariance)
-        self.lengthscale.gradient = (dL_dpsi1[:,:,None]*_dpsi1_dlengthscale).reshape(-1,self.input_dim).sum(axis=0) 
-    
-
-        #from psi2
-        self.variance.gradient += (dL_dpsi2 * _dpsi2_dvariance).sum()
-        self.lengthscale.gradient += (dL_dpsi2[:,:,:,None] * _dpsi2_dlengthscale).reshape(-1,self.input_dim).sum(axis=0)        
-        
-    def gradients_Z_expectations(self, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
-        _, _, _, _, _, _dpsi1_dZ, _ = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
-        _, _, _, _, _, _dpsi2_dZ, _ = ssrbf_psi_comp._psi2computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
-
-        #psi1
-        grad = (dL_dpsi1[:, :, None] * _dpsi1_dZ).sum(axis=0)
-
-        #psi2
-        grad += (dL_dpsi2[:, :, :, None] * _dpsi2_dZ).sum(axis=0).sum(axis=1)
-
-        return grad
-
-    def gradients_qX_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
-        ndata = variational_posterior.mean.shape[0]
-        
-        _, _, _dpsi1_dgamma, _dpsi1_dmu, _dpsi1_dS, _, _ = ssrbf_psi_comp._psi1computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
-        _, _, _dpsi2_dgamma, _dpsi2_dmu, _dpsi2_dS, _, _ = ssrbf_psi_comp._psi2computations(self.variance, self.lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
-
-        #psi1
-        grad_mu = (dL_dpsi1[:, :, None] * _dpsi1_dmu).sum(axis=1)
-        grad_S = (dL_dpsi1[:, :, None] * _dpsi1_dS).sum(axis=1)
-        grad_gamma = (dL_dpsi1[:,:,None] * _dpsi1_dgamma).sum(axis=1)
-        #psi2
-        grad_mu += (dL_dpsi2[:, :, :, None] * _dpsi2_dmu).reshape(ndata,-1,self.input_dim).sum(axis=1)
-        grad_S += (dL_dpsi2[:, :, :, None] * _dpsi2_dS).reshape(ndata,-1,self.input_dim).sum(axis=1)
-        grad_gamma += (dL_dpsi2[:,:,:, None] * _dpsi2_dgamma).reshape(ndata,-1,self.input_dim).sum(axis=1)
-        
-        return grad_mu, grad_S, grad_gamma
-        
-    #---------------------------------------#
-    #            Precomputations            #
-    #---------------------------------------#
-
-    #@cache_this(1)
-    def _K_computations(self, X, X2):
-        """
-        K(X,X2) - X is NxQ 
-        Q -> input dimension (self.input_dim)
-        """
-        if X2 is None:
-            self._X2 = None
-                            
-            X = X / self.lengthscale
-            Xsquare = np.sum(np.square(X), axis=1)
-            self._K_dist2 = -2.*tdot(X) + (Xsquare[:, None] + Xsquare[None, :])
-        else:
-            self._X2 = X2.copy()
-            
-            X = X / self.lengthscale
-            X2 = X2 / self.lengthscale
-            self._K_dist2 = -2.*np.dot(X, X2.T) + (np.sum(np.square(X), axis=1)[:, None] + np.sum(np.square(X2), axis=1)[None, :])
-        self._K_dvar = np.exp(-0.5 * self._K_dist2)
-

GPy/kern/_src/static.py

@@ -6,7 +6,6 @@ from kern import Kern
 import numpy as np
 from ...core.parameterization import Param
 from ...core.parameterization.transformations import Logexp
-import numpy as np
 
 class Static(Kern):
     def __init__(self, input_dim, variance, active_dims, name):
@@ -25,7 +24,7 @@ class Static(Kern):
     def gradients_X_diag(self, dL_dKdiag, X):
         return np.zeros(X.shape)
 
-    def gradients_Z_expectations(self, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
+    def gradients_Z_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
         return np.zeros(Z.shape)
 
     def gradients_qX_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
@@ -39,8 +38,13 @@ class Static(Kern):
 
     def psi2(self, Z, variational_posterior):
         K = self.K(variational_posterior.mean, Z)
-        return K[:,:,None]*K[:,None,:] # NB. more efficient implementations on inherriting classes
+        return np.einsum('ij,ik->jk',K,K) #K[:,:,None]*K[:,None,:] # NB. more efficient implementations on inherriting classes
 
+    def input_sensitivity(self, summarize=True):
+        if summarize:
+            return super(Static, self).input_sensitivity(summarize=summarize)
+        else:
+            return np.ones(self.input_dim) * self.variance
 
 class White(Static):
     def __init__(self, input_dim, variance=1., active_dims=None, name='white'):
@@ -53,7 +57,7 @@ class White(Static):
             return np.zeros((X.shape[0], X2.shape[0]))
 
     def psi2(self, Z, variational_posterior):
-        return np.zeros((variational_posterior.shape[0], Z.shape[0], Z.shape[0]), dtype=np.float64)
+        return np.zeros((Z.shape[0], Z.shape[0]), dtype=np.float64)
 
     def update_gradients_full(self, dL_dK, X, X2=None):
         self.variance.gradient = np.trace(dL_dK)
@@ -64,7 +68,6 @@ class White(Static):
     def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
         self.variance.gradient = dL_dpsi0.sum()
 
-
 class Bias(Static):
     def __init__(self, input_dim, variance=1., active_dims=None, name='bias'):
         super(Bias, self).__init__(input_dim, variance, active_dims, name)
@@ -82,12 +85,12 @@ class Bias(Static):
         self.variance.gradient = dL_dKdiag.sum()
 
     def psi2(self, Z, variational_posterior):
-        ret = np.empty((variational_posterior.shape[0], Z.shape[0], Z.shape[0]), dtype=np.float64)
+        ret = np.empty((Z.shape[0], Z.shape[0]), dtype=np.float64)
-        ret[:] = self.variance**2
+        ret[:] = self.variance*self.variance*variational_posterior.shape[0]
         return ret
 
     def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
-        self.variance.gradient = dL_dpsi0.sum() + dL_dpsi1.sum() + 2.*self.variance*dL_dpsi2.sum()
+        self.variance.gradient = dL_dpsi0.sum() + dL_dpsi1.sum() + 2.*self.variance*dL_dpsi2.sum()*variational_posterior.shape[0]
 
 class Fixed(Static):
     def __init__(self, input_dim, covariance_matrix, variance=1., active_dims=None, name='fixed'):
@@ -97,7 +100,7 @@ class Fixed(Static):
         :param variance: the variance of the kernel
         :type variance: float
         """
-        super(Bias, self).__init__(input_dim, variance, active_dims, name)
+        super(Fixed, self).__init__(input_dim, variance, active_dims, name)
         self.fixed_K = covariance_matrix
     def K(self, X, X2):
         return self.variance * self.fixed_K
@@ -112,7 +115,7 @@ class Fixed(Static):
         self.variance.gradient = np.einsum('i,i', dL_dKdiag, self.fixed_K)
 
     def psi2(self, Z, variational_posterior):
-        return np.zeros((variational_posterior.shape[0], Z.shape[0], Z.shape[0]), dtype=np.float64)
+        return np.zeros((Z.shape[0], Z.shape[0]), dtype=np.float64)
 
     def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
         self.variance.gradient = dL_dpsi0.sum()

GPy/kern/_src/stationary.py

@@ -79,7 +79,6 @@ class Stationary(Kern):
         #a convenience function, so we can cache dK_dr
         return self.dK_dr(self._scaled_dist(X, X2))
 
-    @Cache_this(limit=5, ignore_args=(0,))
     def _unscaled_dist(self, X, X2=None):
         """
         Compute the Euclidean distance between each row of X and X2, or between
@@ -90,13 +89,14 @@ class Stationary(Kern):
             Xsq = np.sum(np.square(X),1)
             r2 = -2.*tdot(X) + (Xsq[:,None] + Xsq[None,:])
             util.diag.view(r2)[:,]= 0. # force diagnoal to be zero: sometime numerically a little negative
+            r2 = np.clip(r2, 0, np.inf)
             return np.sqrt(r2)
         else:
             #X2, = self._slice_X(X2)
             X1sq = np.sum(np.square(X),1)
             X2sq = np.sum(np.square(X2),1)
             r2 = -2.*np.dot(X, X2.T) + X1sq[:,None] + X2sq[None,:]
-            r2[r2<0] = 0. # A bit hacky
+            r2 = np.clip(r2, 0, np.inf)
             return np.sqrt(r2)
 
     @Cache_this(limit=5, ignore_args=())
@@ -160,27 +160,27 @@ class Stationary(Kern):
         """
         invdist = self._inv_dist(X, X2)
         dL_dr = self.dK_dr_via_X(X, X2) * dL_dK
-        #The high-memory numpy way:
-        #d =  X[:, None, :] - X2[None, :, :]
-        #ret = np.sum((invdist*dL_dr)[:,:,None]*d,1)/self.lengthscale**2
-        #if X2 is None:
-            #ret *= 2.
-
-        #the lower memory way with a loop
         tmp = invdist*dL_dr
         if X2 is None:
-            tmp *= 2.
+            tmp = tmp + tmp.T
             X2 = X
+
+        #The high-memory numpy way:
+        #d =  X[:, None, :] - X2[None, :, :]
+        #ret = np.sum(tmp[:,:,None]*d,1)/self.lengthscale**2
+
+        #the lower memory way with a loop
         ret = np.empty(X.shape, dtype=np.float64)
-        [np.einsum('ij,ij->i', tmp, X[:,q][:,None]-X2[:,q][None,:], out=ret[:,q]) for q in xrange(self.input_dim)]
+        [np.sum(tmp*(X[:,q][:,None]-X2[:,q][None,:]), axis=1, out=ret[:,q]) for q in xrange(self.input_dim)]
         ret /= self.lengthscale**2
+
         return ret
 
     def gradients_X_diag(self, dL_dKdiag, X):
         return np.zeros(X.shape)
 
-    def input_sensitivity(self):
+    def input_sensitivity(self, summarize=True):
-        return np.ones(self.input_dim)/self.lengthscale
+        return np.ones(self.input_dim)/self.lengthscale**2
 
 class Exponential(Stationary):
     def __init__(self, input_dim, variance=1., lengthscale=None, ARD=False, active_dims=None, name='Exponential'):
@@ -192,6 +192,27 @@ class Exponential(Stationary):
     def dK_dr(self, r):
         return -0.5*self.K_of_r(r)
 
+
+class OU(Stationary):
+    """
+    OU kernel:
+
+    .. math::
+
+       k(r) = \\sigma^2 \exp(- r) \\ \\ \\ \\  \\text{ where  } r = \sqrt{\sum_{i=1}^input_dim \\frac{(x_i-y_i)^2}{\ell_i^2} }
+
+    """
+
+    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=False, active_dims=None, name='OU'):
+        super(OU, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name)
+
+    def K_of_r(self, r):
+        return self.variance * np.exp(-r)
+
+    def dK_dr(self,r):
+        return -1.*self.variance*np.exp(-r)
+
+
 class Matern32(Stationary):
     """
     Matern 3/2 kernel:
@@ -319,7 +340,7 @@ class RatQuad(Stationary):
     """
 
 
-    def __init__(self, input_dim, variance=1., lengthscale=None, power=2., ARD=False, active_dims=None, name='ExpQuad'):
+    def __init__(self, input_dim, variance=1., lengthscale=None, power=2., ARD=False, active_dims=None, name='RatQuad'):
         super(RatQuad, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name)
         self.power = Param('power', power, Logexp())
         self.add_parameters(self.power)

GPy/kern/_src/trunclinear.py

@@ -0,0 +1,208 @@
+# Copyright (c) 2012, GPy authors (see AUTHORS.txt).
+# Licensed under the BSD 3-clause license (see LICENSE.txt)
+
+
+import numpy as np
+from scipy import weave
+from kern import Kern
+from ...util.linalg import tdot
+from ...util.misc import param_to_array
+from ...core.parameterization import Param
+from ...core.parameterization.transformations import Logexp
+from ...util.caching import Cache_this
+from ...core.parameterization import variational
+from ...util.config import *
+
+class TruncLinear(Kern):
+    """
+    Truncated Linear kernel
+
+    .. math::
+
+       k(x,y) = \sum_{i=1}^input_dim \sigma^2_i \max(0, x_iy_i - \simga_q)
+
+    :param input_dim: the number of input dimensions
+    :type input_dim: int
+    :param variances: the vector of variances :math:`\sigma^2_i`
+    :type variances: array or list of the appropriate size (or float if there
+                     is only one variance parameter)
+    :param ARD: Auto Relevance Determination. If False, the kernel has only one
+                variance parameter \sigma^2, otherwise there is one variance
+                parameter per dimension.
+    :type ARD: Boolean
+    :rtype: kernel object
+
+    """
+
+    def __init__(self, input_dim, variances=None, delta=None, ARD=False, active_dims=None, name='linear'):
+        super(TruncLinear, self).__init__(input_dim, active_dims, name)
+        self.ARD = ARD
+        if not ARD:
+            if variances is not None:
+                variances = np.asarray(variances)
+                delta = np.asarray(delta)
+                assert variances.size == 1, "Only one variance needed for non-ARD kernel"
+            else:
+                variances = np.ones(1)
+                delta = np.zeros(1)
+        else:
+            if variances is not None:
+                variances = np.asarray(variances)
+                delta = np.asarray(delta)
+                assert variances.size == self.input_dim, "bad number of variances, need one ARD variance per input_dim"
+            else:
+                variances = np.ones(self.input_dim)
+                delta = np.zeros(self.input_dim)
+
+        self.variances = Param('variances', variances, Logexp())
+        self.delta = Param('delta', delta)
+        self.add_parameter(self.variances)
+        self.add_parameter(self.delta)
+            
+    @Cache_this(limit=2)
+    def K(self, X, X2=None):
+        XX = self.variances*self._product(X, X2)
+        return XX.sum(axis=-1)
+
+    @Cache_this(limit=2)
+    def _product(self, X, X2=None):
+        if X2 is None:
+            X2 = X
+        XX = np.einsum('nq,mq->nmq',X-self.delta,X2-self.delta)
+        XX[XX<0] = 0
+        return XX
+
+    def Kdiag(self, X):
+        return (self.variances*np.square(X-self.delta)).sum(axis=-1)
+
+    def update_gradients_full(self, dL_dK, X, X2=None):
+        dK_dvar = self._product(X, X2)
+        if X2 is None:
+            X2=X
+        dK_ddelta = self.variances*(2*self.delta-X[:,None,:]-X2[None,:,:])*(dK_dvar>0)
+        if self.ARD:
+            self.variances.gradient[:] = np.einsum('nmq,nm->q',dK_dvar,dL_dK)
+            self.delta.gradient[:] = np.einsum('nmq,nm->q',dK_ddelta,dL_dK)
+        else:
+            self.variances.gradient[:] = np.einsum('nmq,nm->',dK_dvar,dL_dK)
+            self.delta.gradient[:] = np.einsum('nmq,nm->',dK_ddelta,dL_dK)
+
+    def update_gradients_diag(self, dL_dKdiag, X):
+        if self.ARD:
+            self.variances.gradient[:] = np.einsum('nq,n->q',np.square(X-self.delta),dL_dKdiag)
+            self.delta.gradient[:] = np.einsum('nq,n->q',2*self.variances*(self.delta-X),dL_dKdiag)
+        else:
+            self.variances.gradient[:] = np.einsum('nq,n->',np.square(X-self.delta),dL_dKdiag)
+            self.delta.gradient[:] = np.einsum('nq,n->',2*self.variances*(self.delta-X),dL_dKdiag)
+
+    def gradients_X(self, dL_dK, X, X2=None):
+        XX = self._product(X, X2)
+        if X2 is None:
+            Xp = (self.variances*(X-self.delta))*(XX>0)
+        else:
+            Xp = (self.variances*(X2-self.delta))*(XX>0)
+        if X2 is None:
+            return np.einsum('nmq,nm->nq',Xp,dL_dK)+np.einsum('mnq,nm->mq',Xp,dL_dK)
+        else:
+            return np.einsum('nmq,nm->nq',Xp,dL_dK)
+
+    def gradients_X_diag(self, dL_dKdiag, X):
+        return 2.*self.variances*dL_dKdiag[:,None]*(X-self.delta)
+
+    def input_sensitivity(self):
+        return np.ones(self.input_dim) * self.variances
+
+class TruncLinear_inf(Kern):
+    """
+    Truncated Linear kernel
+
+    .. math::
+
+       k(x,y) = \sum_{i=1}^input_dim \sigma^2_i \max(0, x_iy_i - \simga_q)
+
+    :param input_dim: the number of input dimensions
+    :type input_dim: int
+    :param variances: the vector of variances :math:`\sigma^2_i`
+    :type variances: array or list of the appropriate size (or float if there
+                     is only one variance parameter)
+    :param ARD: Auto Relevance Determination. If False, the kernel has only one
+                variance parameter \sigma^2, otherwise there is one variance
+                parameter per dimension.
+    :type ARD: Boolean
+    :rtype: kernel object
+
+    """
+
+    def __init__(self, input_dim, interval, variances=None, ARD=False, active_dims=None, name='linear'):
+        super(TruncLinear_inf, self).__init__(input_dim, active_dims, name)
+        self.interval = interval
+        self.ARD = ARD
+        if not ARD:
+            if variances is not None:
+                variances = np.asarray(variances)
+                assert variances.size == 1, "Only one variance needed for non-ARD kernel"
+            else:
+                variances = np.ones(1)
+        else:
+            if variances is not None:
+                variances = np.asarray(variances)
+                assert variances.size == self.input_dim, "bad number of variances, need one ARD variance per input_dim"
+            else:
+                variances = np.ones(self.input_dim)
+
+        self.variances = Param('variances', variances, Logexp())
+        self.add_parameter(self.variances)
+        
+    
+#     @Cache_this(limit=2)
+    def K(self, X, X2=None):
+        tmp = self._product(X, X2)
+        return (self.variances*tmp).sum(axis=-1)
+
+#     @Cache_this(limit=2)
+    def _product(self, X, X2=None):
+        if X2 is None:
+            X2 = X
+        X_X2 = X[:,None,:]-X2[None,:,:]
+        tmp = np.abs(X_X2**3)/6+np.einsum('nq,mq->nmq',X,X2)*(self.interval[1]-self.interval[0]) \
+              -(X[:,None,:]+X2[None,:,:])*(self.interval[1]*self.interval[1]-self.interval[0]*self.interval[0])/2+(self.interval[1]**3-self.interval[0]**3)/3.
+        return tmp
+
+    def Kdiag(self, X):
+        tmp = np.square(X)*(self.interval[1]-self.interval[0]) \
+              -X*(self.interval[1]*self.interval[1]-self.interval[0]*self.interval[0])+(self.interval[1]**3-self.interval[0]**3)/3
+        return (self.variances*tmp).sum(axis=-1)
+
+    def update_gradients_full(self, dL_dK, X, X2=None):
+        dK_dvar = self._product(X, X2)
+        if self.ARD:
+            self.variances.gradient[:] = np.einsum('nmq,nm->q',dK_dvar,dL_dK)
+        else:
+            self.variances.gradient[:] = np.einsum('nmq,nm->',dK_dvar,dL_dK)
+
+    def update_gradients_diag(self, dL_dKdiag, X):
+        tmp = np.square(X)*(self.interval[1]-self.interval[0]) \
+              -X*(self.interval[1]*self.interval[1]-self.interval[0]*self.interval[0])+(self.interval[1]**3-self.interval[0]**3)/3
+        if self.ARD:
+            self.variances.gradient[:] = np.einsum('nq,n->q',tmp,dL_dKdiag)
+        else:
+            self.variances.gradient[:] = np.einsum('nq,n->',tmp,dL_dKdiag)
+
+    def gradients_X(self, dL_dK, X, X2=None):
+        XX = self._product(X, X2)
+        if X2 is None:
+            Xp = (self.variances*(X-self.delta))*(XX>0)
+        else:
+            Xp = (self.variances*(X2-self.delta))*(XX>0)
+        if X2 is None:
+            return np.einsum('nmq,nm->nq',Xp,dL_dK)+np.einsum('mnq,nm->mq',Xp,dL_dK)
+        else:
+            return np.einsum('nmq,nm->nq',Xp,dL_dK)
+
+    def gradients_X_diag(self, dL_dKdiag, X):
+        return 2.*self.variances*dL_dKdiag[:,None]*(X-self.delta)
+
+    def input_sensitivity(self):
+        return np.ones(self.input_dim) * self.variances
+
+

GPy/likelihoods/bernoulli.py

@@ -227,3 +227,6 @@ class Bernoulli(Likelihood):
         ns = np.ones_like(gp, dtype=int)
         Ysim = np.random.binomial(ns, self.gp_link.transf(gp))
         return Ysim.reshape(orig_shape)
+
+    def exact_inference_gradients(self, dL_dKdiag,Y_metadata=None):
+        pass

GPy/likelihoods/ordinal.py

@@ -0,0 +1,48 @@
+# Copyright (c) 2014 The GPy authors (see AUTHORS.txt)
+# Licensed under the BSD 3-clause license (see LICENSE.txt)
+
+
+import sympy as sym
+from GPy.util.symbolic import gammaln, normcdfln, normcdf, IndMatrix, create_matrix
+import numpy as np
+from ..util.univariate_Gaussian import std_norm_pdf, std_norm_cdf
+import link_functions
+from symbolic import Symbolic
+from scipy import stats
+
+class Ordinal(Symbolic):
+    """
+    Ordinal
+
+    .. math::
+        p(y_{i}|\pi(f_{i})) = \left(\frac{r}{r+f_i}\right)^r \frac{\Gamma(r+y_i)}{y!\Gamma(r)}\left(\frac{f_i}{r+f_i}\right)^{y_i}
+
+    .. Note::
+        Y takes non zero integer values..
+        link function should have a positive domain, e.g. log (default).
+
+    .. See also::
+        symbolic.py, for the parent class
+    """
+    def __init__(self, categories=3, gp_link=None):
+        if gp_link is None:
+            gp_link = link_functions.Identity()
+
+        dispersion = sym.Symbol('width', positive=True, real=True)
+        y_0 = sym.Symbol('y_0', nonnegative=True, integer=True)
+        f_0 = sym.Symbol('f_0', positive=True, real=True) 
+        log_pdf = create_matrix('log_pdf', 1, categories)
+        log_pdf[0] = normcdfln(-f_0)
+        if categories>2:
+            w = create_matrix('w', 1, categories)
+            log_pdf[categories-1] = normcdfln(w.sum() + f_0)
+            for i in range(1, categories-1):
+                log_pdf[i] = sym.log(normcdf(w[0, 0:i-1].sum() + f_0) - normcdf(w[0, 0:i].sum()-f_0) )
+        else:
+            log_pdf[1] = normcdfln(f_0)
+        log_pdf.index_var = y_0
+        super(Ordinal, self).__init__(log_pdf=log_pdf, gp_link=gp_link, name='Ordinal')
+
+        # TODO: Check this.
+        self.log_concave = True
+

GPy/mappings/additive.py

@@ -0,0 +1,61 @@
+# Copyright (c) 2013, GPy authors (see AUTHORS.txt).
+# Licensed under the BSD 3-clause license (see LICENSE.txt)
+
+import numpy as np
+from ..core.mapping import Mapping
+import GPy
+
+class Additive(Mapping):
+    """
+    Mapping based on adding two existing mappings together.
+
+    .. math::
+
+       f(\mathbf{x}*) = f_1(\mathbf{x}*) + f_2(\mathbf(x)*)
+
+    :param mapping1: first mapping to add together.
+    :type mapping1: GPy.mappings.Mapping
+    :param mapping2: second mapping to add together.
+    :type mapping2: GPy.mappings.Mapping
+    :param tensor: whether or not to use the tensor product of input spaces
+    :type tensor: bool
+
+    """
+
+    def __init__(self, mapping1, mapping2, tensor=False):
+        if tensor:
+            input_dim = mapping1.input_dim + mapping2.input_dim
+        else:
+            input_dim = mapping1.input_dim
+            assert(mapping1.input_dim==mapping2.input_dim)
+        assert(mapping1.output_dim==mapping2.output_dim)
+        output_dim = mapping1.output_dim
+        Mapping.__init__(self, input_dim=input_dim, output_dim=output_dim)
+        self.mapping1 = mapping1
+        self.mapping2 = mapping2
+        self.num_params = self.mapping1.num_params + self.mapping2.num_params
+        self.name = self.mapping1.name + '+' + self.mapping2.name
+    def _get_param_names(self):
+        return self.mapping1._get_param_names + self.mapping2._get_param_names
+
+    def _get_params(self):
+        return np.hstack((self.mapping1._get_params() self.mapping2._get_params()))
+
+    def _set_params(self, x):
+        self.mapping1._set_params(x[:self.mapping1.num_params])
+        self.mapping2._set_params(x[self.mapping1.num_params:])
+        
+    def randomize(self):
+        self.mapping1._randomize()
+        self.mapping2._randomize()
+
+    def f(self, X):
+        return self.mapping1.f(X) + self.mapping2.f(X)
+
+    def df_dtheta(self, dL_df, X):
+        self._df_dA = (dL_df[:, :, None]*self.kern.K(X, self.X)[:, None, :]).sum(0).T
+        self._df_dbias = (dL_df.sum(0))
+        return np.hstack((self._df_dA.flatten(), self._df_dbias))
+
+    def df_dX(self, dL_df, X):
+        return self.kern.dK_dX((dL_df[:, None, :]*self.A[None, :, :]).sum(2), X, self.X) 

GPy/models/__init__.py

@@ -17,3 +17,6 @@ from ss_gplvm import SSGPLVM
 from gp_coregionalized_regression import GPCoregionalizedRegression
 from sparse_gp_coregionalized_regression import SparseGPCoregionalizedRegression
 from gp_heteroscedastic_regression import GPHeteroscedasticRegression
+from ss_mrd import SSMRD
+from gp_kronecker_gaussian_regression import GPKroneckerGaussianRegression
+from gp_var_gauss import GPVariationalGaussianApproximation

GPy/models/bayesian_gplvm.py

@@ -8,8 +8,9 @@ from ..likelihoods import Gaussian
 from ..inference.optimization import SCG
 from ..util import linalg
 from ..core.parameterization.variational import NormalPosterior, NormalPrior, VariationalPosterior
-from ..inference.latent_function_inference.var_dtc_parallel import update_gradients
+from ..inference.latent_function_inference.var_dtc_parallel import update_gradients, VarDTC_minibatch
 from ..inference.latent_function_inference.var_dtc_gpu import VarDTC_GPU
+import logging
 
 class BayesianGPLVM(SparseGP):
     """
@@ -24,9 +25,14 @@ class BayesianGPLVM(SparseGP):
 
     """
     def __init__(self, Y, input_dim, X=None, X_variance=None, init='PCA', num_inducing=10,
-                 Z=None, kernel=None, inference_method=None, likelihood=None, name='bayesian gplvm', **kwargs):
+                 Z=None, kernel=None, inference_method=None, likelihood=None, name='bayesian gplvm', mpi_comm=None, normalizer=None):
+        self.mpi_comm = mpi_comm
+        self.__IN_OPTIMIZATION__ = False
+
+        self.logger = logging.getLogger(self.__class__.__name__)
         if X == None:
             from ..util.initialization import initialize_latent
+            self.logger.info("initializing latent space X with method {}".format(init))
             X, fracs = initialize_latent(init, input_dim, Y)
         else:
             fracs = np.ones(input_dim)
@@ -34,41 +40,66 @@ class BayesianGPLVM(SparseGP):
         self.init = init
 
         if X_variance is None:
+            self.logger.info("initializing latent space variance ~ uniform(0,.1)")
             X_variance = np.random.uniform(0,.1,X.shape)
 
-
         if Z is None:
+            self.logger.info("initializing inducing inputs")
             Z = np.random.permutation(X.copy())[:num_inducing]
         assert Z.shape[1] == X.shape[1]
 
         if kernel is None:
-            kernel = kern.RBF(input_dim, lengthscale=fracs, ARD=True) # + kern.white(input_dim)
+            self.logger.info("initializing kernel RBF")
+            kernel = kern.RBF(input_dim, lengthscale=1./fracs, ARD=True) #+ kern.Bias(input_dim) + kern.White(input_dim)
 
         if likelihood is None:
             likelihood = Gaussian()
 
-
         self.variational_prior = NormalPrior()
         X = NormalPosterior(X, X_variance)
 
         if inference_method is None:
-            if np.any(np.isnan(Y)):
+            inan = np.isnan(Y)
+            if np.any(inan):
                 from ..inference.latent_function_inference.var_dtc import VarDTCMissingData
-                inference_method = VarDTCMissingData()
+                self.logger.debug("creating inference_method with var_dtc missing data")
+                inference_method = VarDTCMissingData(inan=inan)
+            elif mpi_comm is not None:
+                inference_method = VarDTC_minibatch(mpi_comm=mpi_comm)
             else:
                 from ..inference.latent_function_inference.var_dtc import VarDTC
+                self.logger.debug("creating inference_method var_dtc")
                 inference_method = VarDTC()
+        if isinstance(inference_method,VarDTC_minibatch):
+            inference_method.mpi_comm = mpi_comm
 
-        SparseGP.__init__(self, X, Y, Z, kernel, likelihood, inference_method, name, **kwargs)
+        if kernel.useGPU and isinstance(inference_method, VarDTC_GPU):
+            kernel.psicomp.GPU_direct = True
+
+        SparseGP.__init__(self, X, Y, Z, kernel, likelihood, inference_method, name, normalizer=normalizer)
+        self.logger.info("Adding X as parameter")
         self.add_parameter(self.X, index=0)
 
+        if mpi_comm != None:
+            from ..util.mpi import divide_data
+            N_start, N_end, N_list = divide_data(Y.shape[0], mpi_comm)
+            self.N_range = (N_start, N_end)
+            self.N_list = np.array(N_list)
+            self.Y_local = self.Y[N_start:N_end]
+            print 'MPI RANK: '+str(self.mpi_comm.rank)+' with datasize: '+str(self.N_range)
+            mpi_comm.Bcast(self.param_array, root=0)
+
     def set_X_gradients(self, X, X_grad):
         """Set the gradients of the posterior distribution of X in its specific form."""
         X.mean.gradient, X.variance.gradient = X_grad
 
+    def get_X_gradients(self, X):
+        """Get the gradients of the posterior distribution of X in its specific form."""
+        return X.mean.gradient, X.variance.gradient
+
     def parameters_changed(self):
-        if isinstance(self.inference_method, VarDTC_GPU):
+        if isinstance(self.inference_method, VarDTC_GPU) or isinstance(self.inference_method, VarDTC_minibatch):
-            update_gradients(self)
+            update_gradients(self, mpi_comm=self.mpi_comm)
             return
 
         super(BayesianGPLVM, self).parameters_changed()
@@ -76,6 +107,22 @@ class BayesianGPLVM(SparseGP):
 
         self.X.mean.gradient, self.X.variance.gradient = self.kern.gradients_qX_expectations(variational_posterior=self.X, Z=self.Z, dL_dpsi0=self.grad_dict['dL_dpsi0'], dL_dpsi1=self.grad_dict['dL_dpsi1'], dL_dpsi2=self.grad_dict['dL_dpsi2'])
 
+        # This is testing code -------------------------
+#         i = np.random.randint(self.X.shape[0])
+#         X_ = self.X.mean
+#         which = np.sqrt(((X_ - X_[i:i+1])**2).sum(1)).argsort()>(max(0, self.X.shape[0]-51))
+#         _, _, grad_dict = self.inference_method.inference(self.kern, self.X[which], self.Z, self.likelihood, self.Y[which], self.Y_metadata)
+#         grad = self.kern.gradients_qX_expectations(variational_posterior=self.X[which], Z=self.Z, dL_dpsi0=grad_dict['dL_dpsi0'], dL_dpsi1=grad_dict['dL_dpsi1'], dL_dpsi2=grad_dict['dL_dpsi2'])
+#
+#         self.X.mean.gradient[:] = 0
+#         self.X.variance.gradient[:] = 0
+#         self.X.mean.gradient[which] = grad[0]
+#         self.X.variance.gradient[which] = grad[1]
+
+        # update for the KL divergence
+#         self.variational_prior.update_gradients_KL(self.X, which)
+        # -----------------------------------------------
+
         # update for the KL divergence
         self.variational_prior.update_gradients_KL(self.X)
 
@@ -83,7 +130,7 @@ class BayesianGPLVM(SparseGP):
                 resolution=50, ax=None, marker='o', s=40,
                 fignum=None, plot_inducing=True, legend=True,
                 plot_limits=None,
-                aspect='auto', updates=False, **kwargs):
+                aspect='auto', updates=False, predict_kwargs={}, imshow_kwargs={}):
         import sys
         assert "matplotlib" in sys.modules, "matplotlib package has not been imported."
         from ..plotting.matplot_dep import dim_reduction_plots
@@ -91,7 +138,7 @@ class BayesianGPLVM(SparseGP):
         return dim_reduction_plots.plot_latent(self, labels, which_indices,
                 resolution, ax, marker, s,
                 fignum, plot_inducing, legend,
-                plot_limits, aspect, updates, **kwargs)
+                plot_limits, aspect, updates, predict_kwargs, imshow_kwargs)
 
     def do_test_latents(self, Y):
         """
@@ -100,36 +147,41 @@ class BayesianGPLVM(SparseGP):
         Notes:
         This will only work with a univariate Gaussian likelihood (for now)
         """
-        assert not self.likelihood.is_heteroscedastic
         N_test = Y.shape[0]
         input_dim = self.Z.shape[1]
+
         means = np.zeros((N_test, input_dim))
         covars = np.zeros((N_test, input_dim))
 
-        dpsi0 = -0.5 * self.input_dim * self.likelihood.precision
+        dpsi0 = -0.5 * self.input_dim / self.likelihood.variance
-        dpsi2 = self.dL_dpsi2[0][None, :, :] # TODO: this may change if we ignore het. likelihoods
+        dpsi2 = self.grad_dict['dL_dpsi2'][0][None, :, :] # TODO: this may change if we ignore het. likelihoods
-        V = self.likelihood.precision * Y
+        V = Y/self.likelihood.variance
 
         #compute CPsi1V
-        if self.Cpsi1V is None:
+        #if self.Cpsi1V is None:
-            psi1V = np.dot(self.psi1.T, self.likelihood.V)
+        #    psi1V = np.dot(self.psi1.T, self.likelihood.V)
-            tmp, _ = linalg.dtrtrs(self._Lm, np.asfortranarray(psi1V), lower=1, trans=0)
+        #    tmp, _ = linalg.dtrtrs(self._Lm, np.asfortranarray(psi1V), lower=1, trans=0)
-            tmp, _ = linalg.dpotrs(self.LB, tmp, lower=1)
+        #    tmp, _ = linalg.dpotrs(self.LB, tmp, lower=1)
-            self.Cpsi1V, _ = linalg.dtrtrs(self._Lm, tmp, lower=1, trans=1)
+        #    self.Cpsi1V, _ = linalg.dtrtrs(self._Lm, tmp, lower=1, trans=1)
 
-        dpsi1 = np.dot(self.Cpsi1V, V.T)
+        dpsi1 = np.dot(self.posterior.woodbury_vector, V.T)
 
-        start = np.zeros(self.input_dim * 2)
+        #start = np.zeros(self.input_dim * 2)
+
+
+        from scipy.optimize import minimize
 
         for n, dpsi1_n in enumerate(dpsi1.T[:, :, None]):
-            args = (self.kern, self.Z, dpsi0, dpsi1_n.T, dpsi2)
+            args = (input_dim, self.kern.copy(), self.Z, dpsi0, dpsi1_n.T, dpsi2)
-            xopt, fopt, neval, status = SCG(f=latent_cost, gradf=latent_grad, x=start, optargs=args, display=False)
+            res = minimize(latent_cost_and_grad, jac=True, x0=np.hstack((means[n], covars[n])), args=args, method='BFGS')
-
+            xopt = res.x
             mu, log_S = xopt.reshape(2, 1, -1)
             means[n] = mu[0].copy()
             covars[n] = np.exp(log_S[0]).copy()
 
-        return means, covars
+        X = NormalPosterior(means, covars)
+
+        return X
 
     def dmu_dX(self, Xnew):
         """
@@ -159,59 +211,72 @@ class BayesianGPLVM(SparseGP):
         from ..plotting.matplot_dep import dim_reduction_plots
 
         return dim_reduction_plots.plot_steepest_gradient_map(self,*args,**kwargs)
+    def __getstate__(self):
+        dc = super(BayesianGPLVM, self).__getstate__()
+        dc['mpi_comm'] = None
+        if self.mpi_comm != None:
+            del dc['N_range']
+            del dc['N_list']
+            del dc['Y_local']
+        return dc
+
+    def __setstate__(self, state):
+        return super(BayesianGPLVM, self).__setstate__(state)
+
+    #=====================================================
+    # The MPI parallelization
+    #     - can move to model at some point
+    #=====================================================
+
+    def _set_params_transformed(self, p):
+        if self.mpi_comm != None:
+            if self.__IN_OPTIMIZATION__ and self.mpi_comm.rank==0:
+                self.mpi_comm.Bcast(np.int32(1),root=0)
+            self.mpi_comm.Bcast(p, root=0)
+        super(BayesianGPLVM, self)._set_params_transformed(p)
+
+    def optimize(self, optimizer=None, start=None, **kwargs):
+        self.__IN_OPTIMIZATION__ = True
+        if self.mpi_comm==None:
+            super(BayesianGPLVM, self).optimize(optimizer,start,**kwargs)
+        elif self.mpi_comm.rank==0:
+            super(BayesianGPLVM, self).optimize(optimizer,start,**kwargs)
+            self.mpi_comm.Bcast(np.int32(-1),root=0)
+        elif self.mpi_comm.rank>0:
+            x = self._get_params_transformed().copy()
+            flag = np.empty(1,dtype=np.int32)
+            while True:
+                self.mpi_comm.Bcast(flag,root=0)
+                if flag==1:
+                    self._set_params_transformed(x)
+                elif flag==-1:
+                    break
+                else:
+                    self.__IN_OPTIMIZATION__ = False
+                    raise Exception("Unrecognizable flag for synchronization!")
+        self.__IN_OPTIMIZATION__ = False
 
 
-def latent_cost_and_grad(mu_S, kern, Z, dL_dpsi0, dL_dpsi1, dL_dpsi2):
+def latent_cost_and_grad(mu_S, input_dim, kern, Z, dL_dpsi0, dL_dpsi1, dL_dpsi2):
     """
     objective function for fitting the latent variables for test points
     (negative log-likelihood: should be minimised!)
     """
-    mu, log_S = mu_S.reshape(2, 1, -1)
+    mu = mu_S[:input_dim][None]
+    log_S = mu_S[input_dim:][None]
     S = np.exp(log_S)
 
-    psi0 = kern.psi0(Z, mu, S)
+    X = NormalPosterior(mu, S)
-    psi1 = kern.psi1(Z, mu, S)
-    psi2 = kern.psi2(Z, mu, S)
 
-    lik = dL_dpsi0 * psi0 + np.dot(dL_dpsi1.flatten(), psi1.flatten()) + np.dot(dL_dpsi2.flatten(), psi2.flatten()) - 0.5 * np.sum(np.square(mu) + S) + 0.5 * np.sum(log_S)
+    psi0 = kern.psi0(Z, X)
+    psi1 = kern.psi1(Z, X)
+    psi2 = kern.psi2(Z, X)
 
-    mu0, S0 = kern.dpsi0_dmuS(dL_dpsi0, Z, mu, S)
+    lik = dL_dpsi0 * psi0.sum() + np.einsum('ij,kj->...', dL_dpsi1, psi1) + np.einsum('ijk,lkj->...', dL_dpsi2, psi2) - 0.5 * np.sum(np.square(mu) + S) + 0.5 * np.sum(log_S)
-    mu1, S1 = kern.dpsi1_dmuS(dL_dpsi1, Z, mu, S)
-    mu2, S2 = kern.dpsi2_dmuS(dL_dpsi2, Z, mu, S)
 
-    dmu = mu0 + mu1 + mu2 - mu
+    dLdmu, dLdS = kern.gradients_qX_expectations(dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, X)
+    dmu = dLdmu - mu
     # dS = S0 + S1 + S2 -0.5 + .5/S
-    dlnS = S * (S0 + S1 + S2 - 0.5) + .5
+    dlnS = S * (dLdS - 0.5) + .5
+
     return -lik, -np.hstack((dmu.flatten(), dlnS.flatten()))
-
-def latent_cost(mu_S, kern, Z, dL_dpsi0, dL_dpsi1, dL_dpsi2):
-    """
-    objective function for fitting the latent variables (negative log-likelihood: should be minimised!)
-    This is the same as latent_cost_and_grad but only for the objective
-    """
-    mu, log_S = mu_S.reshape(2, 1, -1)
-    S = np.exp(log_S)
-
-    psi0 = kern.psi0(Z, mu, S)
-    psi1 = kern.psi1(Z, mu, S)
-    psi2 = kern.psi2(Z, mu, S)
-
-    lik = dL_dpsi0 * psi0 + np.dot(dL_dpsi1.flatten(), psi1.flatten()) + np.dot(dL_dpsi2.flatten(), psi2.flatten()) - 0.5 * np.sum(np.square(mu) + S) + 0.5 * np.sum(log_S)
-    return -float(lik)
-
-def latent_grad(mu_S, kern, Z, dL_dpsi0, dL_dpsi1, dL_dpsi2):
-    """
-    This is the same as latent_cost_and_grad but only for the grad
-    """
-    mu, log_S = mu_S.reshape(2, 1, -1)
-    S = np.exp(log_S)
-
-    mu0, S0 = kern.dpsi0_dmuS(dL_dpsi0, Z, mu, S)
-    mu1, S1 = kern.dpsi1_dmuS(dL_dpsi1, Z, mu, S)
-    mu2, S2 = kern.dpsi2_dmuS(dL_dpsi2, Z, mu, S)
-
-    dmu = mu0 + mu1 + mu2 - mu
-    # dS = S0 + S1 + S2 -0.5 + .5/S
-    dlnS = S * (S0 + S1 + S2 - 0.5) + .5
-
-    return -np.hstack((dmu.flatten(), dlnS.flatten()))

GPy/models/gp_heteroscedastic_regression.py

@@ -30,7 +30,9 @@ class GPHeteroscedasticRegression(GP):
             kernel = kern.RBF(X.shape[1])
 
         #Likelihood
-        likelihoods_list = [likelihoods.Gaussian(name="Gaussian_noise_%s" %j) for j in range(Ny)]
+        #likelihoods_list = [likelihoods.Gaussian(name="Gaussian_noise_%s" %j) for j in range(Ny)]
+        noise_terms = np.unique(Y_metadata['output_index'].flatten())
+        likelihoods_list = [likelihoods.Gaussian(name="Gaussian_noise_%s" %j) for j in noise_terms]
         likelihood = likelihoods.MixedNoise(likelihoods_list=likelihoods_list)
 
         super(GPHeteroscedasticRegression, self).__init__(X,Y,kernel,likelihood, Y_metadata=Y_metadata)

GPy/models/gp_kronecker_gaussian_regression.py

@@ -0,0 +1,119 @@
+# Copyright (c) 2014, James Hensman, Alan Saul
+# Distributed under the terms of the GNU General public License, see LICENSE.txt
+
+import numpy as np
+from ..core.model import Model
+from ..core.parameterization import ObsAr
+from .. import likelihoods
+
+class GPKroneckerGaussianRegression(Model):
+    """
+    Kronecker GP regression
+
+    Take two kernels computed on separate spaces K1(X1), K2(X2), and a data
+    matrix Y which is f size (N1, N2).
+
+    The effective covaraince is np.kron(K2, K1)
+    The effective data is vec(Y) = Y.flatten(order='F')
+
+    The noise must be iid Gaussian.
+
+    See Stegle et al.
+    @inproceedings{stegle2011efficient,
+      title={Efficient inference in matrix-variate gaussian models with $\\backslash$ iid observation noise},
+      author={Stegle, Oliver and Lippert, Christoph and Mooij, Joris M and Lawrence, Neil D and Borgwardt, Karsten M},
+      booktitle={Advances in Neural Information Processing Systems},
+      pages={630--638},
+      year={2011}
+    }
+
+    """
+    def __init__(self, X1, X2, Y, kern1, kern2, noise_var=1., name='KGPR'):
+        Model.__init__(self, name=name)
+        # accept the construction arguments
+        self.X1 = ObsAr(X1)
+        self.X2 = ObsAr(X2)
+        self.Y = Y
+        self.kern1, self.kern2 = kern1, kern2
+        self.add_parameter(self.kern1)
+        self.add_parameter(self.kern2)
+
+        self.likelihood = likelihoods.Gaussian()
+        self.likelihood.variance = noise_var
+        self.add_parameter(self.likelihood)
+
+        self.num_data1, self.input_dim1 = self.X1.shape
+        self.num_data2, self.input_dim2 = self.X2.shape
+
+        assert kern1.input_dim == self.input_dim1
+        assert kern2.input_dim == self.input_dim2
+        assert Y.shape == (self.num_data1, self.num_data2)
+
+    def log_likelihood(self):
+        return self._log_marginal_likelihood
+
+    def parameters_changed(self):
+        (N1, D1), (N2, D2) = self.X1.shape, self.X2.shape
+        K1, K2 = self.kern1.K(self.X1), self.kern2.K(self.X2)
+
+        # eigendecompositon
+        S1, U1 = np.linalg.eigh(K1)
+        S2, U2 = np.linalg.eigh(K2)
+        W = np.kron(S2, S1) + self.likelihood.variance
+
+        Y_ = U1.T.dot(self.Y).dot(U2)
+
+        # store these quantities: needed for prediction
+        Wi = 1./W
+        Ytilde = Y_.flatten(order='F')*Wi
+
+        self._log_marginal_likelihood = -0.5*self.num_data1*self.num_data2*np.log(2*np.pi)\
+                                        -0.5*np.sum(np.log(W))\
+                                        -0.5*np.dot(Y_.flatten(order='F'), Ytilde)
+
+        # gradients for data fit part
+        Yt_reshaped = Ytilde.reshape(N1, N2, order='F')
+        tmp = U1.dot(Yt_reshaped)
+        dL_dK1 = .5*(tmp*S2).dot(tmp.T)
+        tmp = U2.dot(Yt_reshaped.T)
+        dL_dK2 = .5*(tmp*S1).dot(tmp.T)
+
+        # gradients for logdet
+        Wi_reshaped = Wi.reshape(N1, N2, order='F')
+        tmp = np.dot(Wi_reshaped, S2)
+        dL_dK1 += -0.5*(U1*tmp).dot(U1.T)
+        tmp = np.dot(Wi_reshaped.T, S1)
+        dL_dK2 += -0.5*(U2*tmp).dot(U2.T)
+
+        self.kern1.update_gradients_full(dL_dK1, self.X1)
+        self.kern2.update_gradients_full(dL_dK2, self.X2)
+
+        # gradients for noise variance
+        dL_dsigma2 = -0.5*Wi.sum() + 0.5*np.sum(np.square(Ytilde))
+        self.likelihood.variance.gradient = dL_dsigma2
+
+        # store these quantities for prediction:
+        self.Wi, self.Ytilde, self.U1, self.U2 = Wi, Ytilde, U1, U2
+
+    def predict(self, X1new, X2new):
+        """
+        Return the predictive mean and variance at a series of new points X1new, X2new
+        Only returns the diagonal of the predictive variance, for now.
+
+        :param X1new: The points at which to make a prediction
+        :type X1new: np.ndarray, Nnew x self.input_dim1
+        :param X2new: The points at which to make a prediction
+        :type X2new: np.ndarray, Nnew x self.input_dim2
+
+        """
+        k1xf = self.kern1.K(X1new, self.X1)
+        k2xf = self.kern2.K(X2new, self.X2)
+        A = k1xf.dot(self.U1)
+        B = k2xf.dot(self.U2)
+        mu = A.dot(self.Ytilde.reshape(self.num_data1, self.num_data2, order='F')).dot(B.T).flatten(order='F')
+        k1xx = self.kern1.Kdiag(X1new)
+        k2xx = self.kern2.Kdiag(X2new)
+        BA = np.kron(B, A)
+        var = np.kron(k2xx, k1xx) - np.sum(BA**2*self.Wi, 1) + self.likelihood.variance
+
+        return mu[:, None], var[:, None]

GPy/models/gp_regression.py

@@ -15,17 +15,22 @@ class GPRegression(GP):
     :param X: input observations
     :param Y: observed values
     :param kernel: a GPy kernel, defaults to rbf
+    :param Norm normalizer: [False]
+
+        Normalize Y with the norm given.
+        If normalizer is False, no normalization will be done
+        If it is None, we use GaussianNorm(alization)
 
     .. Note:: Multiple independent outputs are allowed using columns of Y
 
     """
 
-    def __init__(self, X, Y, kernel=None, Y_metadata=None):
+    def __init__(self, X, Y, kernel=None, Y_metadata=None, normalizer=None):
 
         if kernel is None:
             kernel = kern.RBF(X.shape[1])
 
         likelihood = likelihoods.Gaussian()
 
-        super(GPRegression, self).__init__(X, Y, kernel, likelihood, name='GP regression', Y_metadata=Y_metadata)
+        super(GPRegression, self).__init__(X, Y, kernel, likelihood, name='GP regression', Y_metadata=Y_metadata, normalizer=normalizer)
 

GPy/models/gp_var_gauss.py

@@ -0,0 +1,108 @@
+# Copyright (c) 2014, James Hensman, Alan Saul
+# Distributed under the terms of the GNU General public License, see LICENSE.txt
+
+import numpy as np
+from scipy import stats
+from scipy.special import erf
+from ..core.model import Model
+from ..core.parameterization import ObsAr
+from .. import kern
+from ..core.parameterization.param import Param
+from ..util.linalg import pdinv
+
+log_2_pi = np.log(2*np.pi)
+
+
+class GPVariationalGaussianApproximation(Model):
+    """
+    The Variational Gaussian Approximation revisited implementation for regression
+
+    @article{Opper:2009,
+        title = {The Variational Gaussian Approximation Revisited},
+        author = {Opper, Manfred and Archambeau, C{\'e}dric},
+        journal = {Neural Comput.},
+        year = {2009},
+        pages = {786--792},
+    }
+    """
+    def __init__(self, X, Y, kernel=None):
+        Model.__init__(self,'Variational GP classification')
+        # accept the construction arguments
+        self.X = ObsAr(X)
+        if kernel is None:
+            kernel = kern.RBF(X.shape[1]) + kern.White(X.shape[1], 0.01)
+        self.kern = kernel
+        self.add_parameter(self.kern)
+        self.num_data, self.input_dim = self.X.shape
+
+        self.alpha = Param('alpha', np.zeros(self.num_data))
+        self.beta = Param('beta', np.ones(self.num_data))
+        self.add_parameter(self.alpha)
+        self.add_parameter(self.beta)
+
+        self.gh_x, self.gh_w = np.polynomial.hermite.hermgauss(20)
+        self.Ysign = np.where(Y==1, 1, -1).flatten()
+
+    def log_likelihood(self):
+        """
+        Marginal log likelihood evaluation
+        """
+        return self._log_lik
+
+    def likelihood_quadrature(self, m, v):
+        """
+        Perform Gauss-Hermite quadrature over the log of the likelihood, with a fixed weight
+        """
+        # assume probit for now.
+        X = self.gh_x[None, :]*np.sqrt(2.*v[:, None]) + (m*self.Ysign)[:, None]
+        p = stats.norm.cdf(X)
+        N = stats.norm.pdf(X)
+        F = np.log(p).dot(self.gh_w)
+        NoverP = N/p
+        dF_dm = (NoverP*self.Ysign[:,None]).dot(self.gh_w)
+        dF_dv = -0.5*(NoverP**2 + NoverP*X).dot(self.gh_w)
+        return F, dF_dm, dF_dv
+
+    def parameters_changed(self):
+        K = self.kern.K(self.X)
+        m = K.dot(self.alpha)
+        KB = K*self.beta[:, None]
+        BKB = KB*self.beta[None, :]
+        A = np.eye(self.num_data) + BKB
+        Ai, LA, _, Alogdet = pdinv(A)
+        Sigma = np.diag(self.beta**-2) - Ai/self.beta[:, None]/self.beta[None, :]  # posterior coavairance: need full matrix for gradients
+        var = np.diag(Sigma)
+
+        F, dF_dm, dF_dv = self.likelihood_quadrature(m, var)
+        dF_da = np.dot(K, dF_dm)
+        SigmaB = Sigma*self.beta
+        dF_db = -np.diag(Sigma.dot(np.diag(dF_dv)).dot(SigmaB))*2
+        KL = 0.5*(Alogdet + np.trace(Ai) - self.num_data + m.dot(self.alpha))
+        dKL_da = m
+        A_A2 = Ai - Ai.dot(Ai)
+        dKL_db = np.diag(np.dot(KB.T, A_A2))
+        self._log_lik = F.sum() - KL
+        self.alpha.gradient = dF_da - dKL_da
+        self.beta.gradient = dF_db - dKL_db
+
+        # K-gradients
+        dKL_dK = 0.5*(self.alpha[None, :]*self.alpha[:, None] + self.beta[:, None]*self.beta[None, :]*A_A2)
+        tmp = Ai*self.beta[:, None]/self.beta[None, :]
+        dF_dK = self.alpha[:, None]*dF_dm[None, :] + np.dot(tmp*dF_dv, tmp.T)
+        self.kern.update_gradients_full(dF_dK - dKL_dK, self.X)
+
+    def predict(self, Xnew):
+        """
+        Predict the function(s) at the new point(s) Xnew.
+
+        :param Xnew: The points at which to make a prediction
+        :type Xnew: np.ndarray, Nnew x self.input_dim
+        """
+        Wi, _, _, _ = pdinv(self.kern.K(self.X) + np.diag(self.beta**-2))
+        Kux = self.kern.K(self.X, Xnew)
+        mu = np.dot(Kux.T, self.alpha)
+        WiKux = np.dot(Wi, Kux)
+        Kxx = self.kern.Kdiag(Xnew)
+        var = Kxx - np.sum(WiKux*Kux, 0)
+
+        return 0.5*(1+erf(mu/np.sqrt(2.*(var+1))))

GPy/models/gplvm.py

@@ -45,10 +45,10 @@ class GPLVM(GP):
         self.X.gradient = self.kern.gradients_X(self.grad_dict['dL_dK'], self.X, None)
 
     def jacobian(self,X):
-        target = np.zeros((X.shape[0],X.shape[1],self.output_dim))
+        J = np.zeros((X.shape[0],X.shape[1],self.output_dim))
         for i in range(self.output_dim):
-            target[:,:,i]=self.kern.gradients_X(np.dot(self.Ki,self.likelihood.Y[:,i])[None, :],X,self.X)
+            J[:,:,i] = self.kern.gradients_X(self.posterior.woodbury_vector[:,i:i+1], X, self.X)
-        return target
+        return J
 
     def magnification(self,X):
         target=np.zeros(X.shape[0])

GPy/models/mrd.py

@@ -2,23 +2,30 @@
 # Licensed under the BSD 3-clause license (see LICENSE.txt)
 
 import numpy as np
-import itertools
+import itertools, logging
-import pylab
 
-from ..core import Model
 from ..kern import Kern
 from ..core.parameterization.variational import NormalPosterior, NormalPrior
 from ..core.parameterization import Param, Parameterized
+from ..core.parameterization.observable_array import ObsAr
 from ..inference.latent_function_inference.var_dtc import VarDTCMissingData, VarDTC
+from ..inference.latent_function_inference import InferenceMethodList
 from ..likelihoods import Gaussian
-from GPy.util.initialization import initialize_latent
+from ..util.initialization import initialize_latent
+from ..core.sparse_gp import SparseGP, GP
 
-class MRD(Model):
+class MRD(SparseGP):
     """
+    !WARNING: This is bleeding edge code and still in development. 
+    Functionality may change fundamentally during development!
+
     Apply MRD to all given datasets Y in Ylist.
 
     Y_i in [n x p_i]
 
+    If Ylist is a dictionary, the keys of the dictionary are the names, and the
+    values are the different datasets to compare. 
+
     The samples n in the datasets need
     to match up, whereas the dimensionality p_d can differ.
 
@@ -39,40 +46,77 @@ class MRD(Model):
     :param num_inducing: number of inducing inputs to use
     :param Z: initial inducing inputs
     :param kernel: list of kernels or kernel to copy for each output
-    :type kernel: [GPy.kern.kern] | GPy.kern.kern | None (default)
+    :type kernel: [GPy.kernels.kernels] | GPy.kernels.kernels | None (default)
-    :param :class:`~GPy.inference.latent_function_inference inference_method: the inference method to use
+    :param :class:`~GPy.inference.latent_function_inference inference_method: 
-    :param :class:`~GPy.likelihoods.likelihood.Likelihood` likelihood: the likelihood to use
+        InferenceMethodList of inferences, or one inference method for all
+    :param :class:`~GPy.likelihoodss.likelihoods.likelihoods` likelihoods: the likelihoods to use
     :param str name: the name of this model
     :param [str] Ynames: the names for the datasets given, must be of equal length as Ylist or None
     """
     def __init__(self, Ylist, input_dim, X=None, X_variance=None,
                  initx = 'PCA', initz = 'permute',
                  num_inducing=10, Z=None, kernel=None,
-                 inference_method=None, likelihood=None, name='mrd', Ynames=None):
+                 inference_method=None, likelihoods=None, name='mrd', Ynames=None):
-        super(MRD, self).__init__(name)
+        super(GP, self).__init__(name)
 
+        self.logger = logging.getLogger(self.__class__.__name__)
         self.input_dim = input_dim
         self.num_inducing = num_inducing
 
-        self.Ylist = Ylist
+        if isinstance(Ylist, dict):
+            Ynames, Ylist = zip(*Ylist.items())
+
+        self.logger.debug("creating observable arrays")
+        self.Ylist = [ObsAr(Y) for Y in Ylist]
+
+        if Ynames is None:
+            self.logger.debug("creating Ynames")
+            Ynames = ['Y{}'.format(i) for i in range(len(Ylist))]
+        self.names = Ynames
+        assert len(self.names) == len(self.Ylist), "one name per dataset, or None if Ylist is a dict"
+
+        if inference_method is None:
+            self.inference_method= InferenceMethodList()
+            warned = False
+            for y in Ylist:
+                inan = np.isnan(y)
+                if np.any(inan):
+                    if not warned:
+                        self.logger.warn("WARNING: NaN values detected, make sure initx method can cope with NaN values or provide starting latent space X")
+                        warned = True
+                    self.inference_method.append(VarDTCMissingData(limit=1, inan=inan))
+                else:
+                    self.inference_method.append(VarDTC(limit=1))
+                self.logger.debug("created inference method <{}>".format(hex(id(self.inference_method[-1]))))
+        else:
+            if not isinstance(inference_method, InferenceMethodList):
+                self.logger.debug("making inference_method an InferenceMethodList")
+                inference_method = InferenceMethodList(inference_method)
+            self.inference_method = inference_method
+
+
         self._in_init_ = True
+        if X is None:
             X, fracs = self._init_X(initx, Ylist)
+        else:
+            fracs = [X.var(0)]*len(Ylist)
         self.Z = Param('inducing inputs', self._init_Z(initz, X))
         self.num_inducing = self.Z.shape[0] # ensure M==N if M>N
 
         # sort out the kernels
+        self.logger.info("building kernels")
         if kernel is None:
             from ..kern import RBF
-            self.kern = [RBF(input_dim, ARD=1, lengthscale=fracs[i], name='rbf'.format(i)) for i in range(len(Ylist))]
+            kernels = [RBF(input_dim, ARD=1, lengthscale=fracs[i]) for i in range(len(Ylist))]
         elif isinstance(kernel, Kern):
-            self.kern = []
+            kernels = []
             for i in range(len(Ylist)):
                 k = kernel.copy()
-                self.kern.append(k)
+                kernels.append(k)
         else:
             assert len(kernel) == len(Ylist), "need one kernel per output"
             assert all([isinstance(k, Kern) for k in kernel]), "invalid kernel object detected!"
-            self.kern = kernel
+            kernels = kernel
 
         if X_variance is None:
             X_variance = np.random.uniform(0.1, 0.2, X.shape)
@@ -80,32 +124,28 @@ class MRD(Model):
         self.variational_prior = NormalPrior()
         self.X = NormalPosterior(X, X_variance)
 
-        if likelihood is None:
+        if likelihoods is None:
-            self.likelihood = [Gaussian(name='Gaussian_noise'.format(i)) for i in range(len(Ylist))]
+            likelihoods = [Gaussian(name='Gaussian_noise'.format(i)) for i in range(len(Ylist))]
-        else: self.likelihood = likelihood
+        else: likelihoods = likelihoods
-
-        if inference_method is None:
-            self.inference_method= []
-            for y in Ylist:
-                if np.any(np.isnan(y)):
-                    self.inference_method.append(VarDTCMissingData(limit=1))
-                else:
-                    self.inference_method.append(VarDTC(limit=1))
-        else:
-            self.inference_method = inference_method
-            self.inference_method.set_limit(len(Ylist))
 
+        self.logger.info("adding X and Z")
         self.add_parameters(self.X, self.Z)
 
-        if Ynames is None:
+        self.bgplvms = []
-            Ynames = ['Y{}'.format(i) for i in range(len(Ylist))]
+        self.num_data = Ylist[0].shape[0]
 
-        for i, n, k, l in itertools.izip(itertools.count(), Ynames, self.kern, self.likelihood):
+        for i, n, k, l, Y in itertools.izip(itertools.count(), Ynames, kernels, likelihoods, Ylist):
+            assert Y.shape[0] == self.num_data, "All datasets need to share the number of datapoints, and those have to correspond to one another"
             p = Parameterized(name=n)
             p.add_parameter(k)
+            p.kern = k
             p.add_parameter(l)
-            setattr(self, 'Y{}'.format(i), p)
+            p.likelihood = l
             self.add_parameter(p)
+            self.bgplvms.append(p)
+
+        self.posterior = None
+        self.logger.info("init done")
         self._in_init_ = False
 
     def parameters_changed(self):
@@ -113,14 +153,15 @@ class MRD(Model):
         self.posteriors = []
         self.Z.gradient[:] = 0.
         self.X.gradient[:] = 0.
-
+        for y, b, i in itertools.izip(self.Ylist, self.bgplvms, self.inference_method):
-        for y, k, l, i in itertools.izip(self.Ylist, self.kern, self.likelihood, self.inference_method):
+            self.logger.info('working on im <{}>'.format(hex(id(i))))
+            k, l = b.kern, b.likelihood
             posterior, lml, grad_dict = i.inference(k, self.X, self.Z, l, y)
 
             self.posteriors.append(posterior)
             self._log_marginal_likelihood += lml
 
-            # likelihood gradients
+            # likelihoods gradients
             l.update_gradients(grad_dict.pop('dL_dthetaL'))
 
             #gradients wrt kernel
@@ -133,12 +174,19 @@ class MRD(Model):
             #gradients wrt Z
             self.Z.gradient += k.gradients_X(dL_dKmm, self.Z)
             self.Z.gradient += k.gradients_Z_expectations(
-                               grad_dict['dL_dpsi1'], grad_dict['dL_dpsi2'], Z=self.Z, variational_posterior=self.X)
+                               grad_dict['dL_dpsi0'], 
+                               grad_dict['dL_dpsi1'], 
+                               grad_dict['dL_dpsi2'], 
+                               Z=self.Z, variational_posterior=self.X)
 
             dL_dmean, dL_dS = k.gradients_qX_expectations(variational_posterior=self.X, Z=self.Z, **grad_dict)
             self.X.mean.gradient += dL_dmean
             self.X.variance.gradient += dL_dS
 
+        self.posterior = self.posteriors[0]
+        self.kern = self.bgplvms[0].kern
+        self.likelihood = self.bgplvms[0].likelihood
+
         # update for the KL divergence
         self.variational_prior.update_gradients_KL(self.X)
         self._log_marginal_likelihood -= self.variational_prior.KL_divergence(self.X)
@@ -151,7 +199,7 @@ class MRD(Model):
             Ylist = self.Ylist
         if init in "PCA_concat":
             X, fracs = initialize_latent('PCA', self.input_dim, np.hstack(Ylist))
-            fracs = [fracs]*self.input_dim
+            fracs = [fracs]*len(Ylist)
         elif init in "PCA_single":
             X = np.zeros((Ylist[0].shape[0], self.input_dim))
             fracs = []
@@ -162,7 +210,7 @@ class MRD(Model):
         else: # init == 'random':
             X = np.random.randn(Ylist[0].shape[0], self.input_dim)
             fracs = X.var(0)
-            fracs = [fracs]*self.input_dim
+            fracs = [fracs]*len(Ylist)
         X -= X.mean()
         X /= X.std()
         return X, fracs
@@ -177,10 +225,12 @@ class MRD(Model):
         return Z
 
     def _handle_plotting(self, fignum, axes, plotf, sharex=False, sharey=False):
+        import matplotlib.pyplot as plt
         if axes is None:
-            fig = pylab.figure(num=fignum)
+            fig = plt.figure(num=fignum)
         sharex_ax = None
         sharey_ax = None
+        plots = []
         for i, g in enumerate(self.bgplvms):
             try:
                 if sharex:
@@ -197,26 +247,36 @@ class MRD(Model):
                 ax = axes[i]
             else:
                 raise ValueError("Need one axes per latent dimension input_dim")
-            plotf(i, g, ax)
+            plots.append(plotf(i, g, ax))
             if sharey_ax is not None:
-                pylab.setp(ax.get_yticklabels(), visible=False)
+                plt.setp(ax.get_yticklabels(), visible=False)
-        pylab.draw()
+        plt.draw()
         if axes is None:
+            try:
                 fig.tight_layout()
-            return fig
+            except:
-        else:
+                pass
-            return pylab.gcf()
+        return plots
 
-    def plot_X(self, fignum=None, ax=None):
+    def predict(self, Xnew, full_cov=False, Y_metadata=None, kern=None, Yindex=0):
-        fig = self._handle_plotting(fignum, ax, lambda i, g, ax: ax.imshow(g.X))
+        """
-        return fig
+        Prediction for data set Yindex[default=0].
+        This predicts the output mean and variance for the dataset given in Ylist[Yindex]
+        """
+        self.posterior = self.posteriors[Yindex]
+        self.kern = self.bgplvms[0].kern
+        self.likelihood = self.bgplvms[0].likelihood
+        return super(MRD, self).predict(Xnew, full_cov, Y_metadata, kern)
 
-    def plot_predict(self, fignum=None, ax=None, sharex=False, sharey=False, **kwargs):
+    #===============================================================================
-        fig = self._handle_plotting(fignum,
+    # TODO: Predict! Maybe even change to several bgplvms, which share an X?
-                                    ax,
+    #===============================================================================
-                                    lambda i, g, ax: ax.imshow(g. predict(g.X)[0], **kwargs),
+    #     def plot_predict(self, fignum=None, ax=None, sharex=False, sharey=False, **kwargs):
-                                    sharex=sharex, sharey=sharey)
+    #         fig = self._handle_plotting(fignum,
-        return fig
+    #                                     ax,
+    #                                     lambda i, g, ax: ax.imshow(g.predict(g.X)[0], **kwargs),
+    #                                     sharex=sharex, sharey=sharey)
+    #         return fig
 
     def plot_scales(self, fignum=None, ax=None, titles=None, sharex=False, sharey=True, *args, **kwargs):
         """
@@ -228,28 +288,55 @@ class MRD(Model):
         """
         if titles is None:
             titles = [r'${}$'.format(name) for name in self.names]
-        ymax = reduce(max, [np.ceil(max(g.input_sensitivity())) for g in self.bgplvms])
+        ymax = reduce(max, [np.ceil(max(g.kern.input_sensitivity())) for g in self.bgplvms])
         def plotf(i, g, ax):
             ax.set_ylim([0,ymax])
-            g.kern.plot_ARD(ax=ax, title=titles[i], *args, **kwargs)
+            return g.kern.plot_ARD(ax=ax, title=titles[i], *args, **kwargs)
         fig = self._handle_plotting(fignum, ax, plotf, sharex=sharex, sharey=sharey)
         return fig
 
-    def plot_latent(self, fignum=None, ax=None, *args, **kwargs):
+    def plot_latent(self, labels=None, which_indices=None,
-        fig = self.gref.plot_latent(fignum=fignum, ax=ax, *args, **kwargs) # self._handle_plotting(fignum, ax, lambda i, g, ax: g.plot_latent(ax=ax, *args, **kwargs))
+                resolution=50, ax=None, marker='o', s=40,
-        return fig
+                fignum=None, plot_inducing=True, legend=True,
-
+                plot_limits=None, 
-    def _debug_plot(self):
+                aspect='auto', updates=False, predict_kwargs={}, imshow_kwargs={}):
-        self.plot_X_1d()
+        """
-        fig = pylab.figure("MRD DEBUG PLOT", figsize=(4 * len(self.bgplvms), 9))
+        see plotting.matplot_dep.dim_reduction_plots.plot_latent
-        fig.clf()
+        if predict_kwargs is None, will plot latent spaces for 0th dataset (and kernel), otherwise give
-        axes = [fig.add_subplot(3, len(self.bgplvms), i + 1) for i in range(len(self.bgplvms))]
+        predict_kwargs=dict(Yindex='index') for plotting only the latent space of dataset with 'index'.
-        self.plot_X(ax=axes)
+        """
-        axes = [fig.add_subplot(3, len(self.bgplvms), i + len(self.bgplvms) + 1) for i in range(len(self.bgplvms))]
+        import sys
-        self.plot_latent(ax=axes)
+        assert "matplotlib" in sys.modules, "matplotlib package has not been imported."
-        axes = [fig.add_subplot(3, len(self.bgplvms), i + 2 * len(self.bgplvms) + 1) for i in range(len(self.bgplvms))]
+        from matplotlib import pyplot as plt
-        self.plot_scales(ax=axes)
+        from ..plotting.matplot_dep import dim_reduction_plots
-        pylab.draw()
+        if "Yindex" not in predict_kwargs:
+            predict_kwargs['Yindex'] = 0
+        if ax is None:
+            fig = plt.figure(num=fignum)
+            ax = fig.add_subplot(111)
+        else:
+            fig = ax.figure
+        plot = dim_reduction_plots.plot_latent(self, labels, which_indices,
+                                        resolution, ax, marker, s,
+                                        fignum, plot_inducing, legend,
+                                        plot_limits, aspect, updates, predict_kwargs, imshow_kwargs)
+        ax.set_title(self.bgplvms[predict_kwargs['Yindex']].name)
+        try:
             fig.tight_layout()
+        except:
+            pass
 
+        return plot
 
+    def __getstate__(self):
+        state = super(MRD, self).__getstate__()
+        del state['kern']
+        del state['likelihood']
+        return state
+
+    def __setstate__(self, state):
+        # TODO:
+        super(MRD, self).__setstate__(state)
+        self.kern = self.bgplvms[0].kern
+        self.likelihood = self.bgplvms[0].likelihood
+        self.parameters_changed()

GPy/models/sparse_gp_regression.py

@@ -30,7 +30,7 @@ class SparseGPRegression(SparseGP):
 
     """
 
-    def __init__(self, X, Y, kernel=None, Z=None, num_inducing=10, X_variance=None):
+    def __init__(self, X, Y, kernel=None, Z=None, num_inducing=10, X_variance=None, normalizer=None):
         num_data, input_dim = X.shape
 
         # kern defaults to rbf (plus white for stability)
@@ -49,7 +49,7 @@ class SparseGPRegression(SparseGP):
         if not (X_variance is None):
             X = NormalPosterior(X,X_variance)
 
-        SparseGP.__init__(self, X, Y, Z, kernel, likelihood, inference_method=VarDTC())
+        SparseGP.__init__(self, X, Y, Z, kernel, likelihood, inference_method=VarDTC(), normalizer=normalizer)
 
 class SparseGPRegressionUncertainInput(SparseGP):
     """
@@ -59,7 +59,7 @@ class SparseGPRegressionUncertainInput(SparseGP):
 
     """
 
-    def __init__(self, X, X_variance, Y, kernel=None, Z=None, num_inducing=10):
+    def __init__(self, X, X_variance, Y, kernel=None, Z=None, num_inducing=10, normalizer=None):
         """
         :param X: input observations
         :type X: np.ndarray (num_data x input_dim)
@@ -91,5 +91,5 @@ class SparseGPRegressionUncertainInput(SparseGP):
 
         likelihood = likelihoods.Gaussian()
 
-        SparseGP.__init__(self, X, Y, Z, kernel, likelihood, X_variance=X_variance, inference_method=VarDTC())
+        SparseGP.__init__(self, X, Y, Z, kernel, likelihood, X_variance=X_variance, inference_method=VarDTC(), normalizer=normalizer)
         self.ensure_default_constraints()

GPy/models/sparse_gplvm.py

@@ -11,7 +11,7 @@ from GPy.models.gplvm import GPLVM
 # from ..core import model
 # from ..util.linalg import pdinv, PCA
 
-class SparseGPLVM(SparseGPRegression, GPLVM):
+class SparseGPLVM(SparseGPRegression):
     """
     Sparse Gaussian Process Latent Variable Model
 
@@ -23,40 +23,25 @@ class SparseGPLVM(SparseGPRegression, GPLVM):
     :type init: 'PCA'|'random'
 
     """
-    def __init__(self, Y, input_dim, kernel=None, init='PCA', num_inducing=10):
+    def __init__(self, Y, input_dim, X=None, kernel=None, init='PCA', num_inducing=10):
-        X = self.initialise_latent(init, input_dim, Y)
+        if X is None:
+            from ..util.initialization import initialize_latent
+            X, fracs = initialize_latent(init, input_dim, Y)
         SparseGPRegression.__init__(self, X, Y, kernel=kernel, num_inducing=num_inducing)
-        self.ensure_default_constraints()
 
-    def _get_param_names(self):
+    def parameters_changed(self):
-        return (sum([['X_%i_%i' % (n, q) for q in range(self.input_dim)] for n in range(self.num_data)], [])
+        super(SparseGPLVM, self).parameters_changed()
-                + SparseGPRegression._get_param_names(self))
+        self.X.gradient = self.kern.gradients_X(self.grad_dict['dL_dKnm'], self.X, self.Z)
 
-    def _get_params(self):
+    def plot_latent(self, labels=None, which_indices=None,
-        return np.hstack((self.X.flatten(), SparseGPRegression._get_params(self)))
+                resolution=50, ax=None, marker='o', s=40,
+                fignum=None, plot_inducing=True, legend=True,
+                plot_limits=None, 
+                aspect='auto', updates=False, predict_kwargs={}, imshow_kwargs={}):
+        assert "matplotlib" in sys.modules, "matplotlib package has not been imported."
+        from ..plotting.matplot_dep import dim_reduction_plots
 
-    def _set_params(self, x):
+        return dim_reduction_plots.plot_latent(self, labels, which_indices,
-        self.X = x[:self.X.size].reshape(self.num_data, self.input_dim).copy()
+                resolution, ax, marker, s,
-        SparseGPRegression._set_params(self, x[self.X.size:])
+                fignum, plot_inducing, legend,
-
+                plot_limits, aspect, updates, predict_kwargs, imshow_kwargs)
-    def log_likelihood(self):
-        return SparseGPRegression.log_likelihood(self)
-
-    def dL_dX(self):
-        dL_dX = self.kern.dKdiag_dX(self.dL_dpsi0, self.X)
-        dL_dX += self.kern.gradients_X(self.dL_dpsi1, self.X, self.Z)
-
-        return dL_dX
-
-    def _log_likelihood_gradients(self):
-        return np.hstack((self.dL_dX().flatten(), SparseGPRegression._log_likelihood_gradients(self)))
-
-    def plot(self):
-        GPLVM.plot(self)
-        # passing Z without a small amout of jitter will induce the white kernel where we don;t want it!
-        mu, var, upper, lower = SparseGPRegression.predict(self, self.Z + np.random.randn(*self.Z.shape) * 0.0001)
-        pb.plot(mu[:, 0] , mu[:, 1], 'ko')
-
-    def plot_latent(self, *args, **kwargs):
-        input_1, input_2 = GPLVM.plot_latent(*args, **kwargs)
-        pb.plot(m.Z[:, input_1], m.Z[:, input_2], '^w')

GPy/models/ss_gplvm.py

@@ -2,20 +2,16 @@
 # Licensed under the BSD 3-clause license (see LICENSE.txt)
 
 import numpy as np
-import itertools
-from matplotlib import pyplot
 
-from ..core.sparse_gp import SparseGP
+from ..core.sparse_gp_mpi import SparseGP_MPI
 from .. import kern
 from ..likelihoods import Gaussian
-from ..inference.optimization import SCG
-from ..util import linalg
 from ..core.parameterization.variational import SpikeAndSlabPrior, SpikeAndSlabPosterior
-from ..inference.latent_function_inference.var_dtc_parallel import update_gradients
+from ..inference.latent_function_inference.var_dtc_parallel import update_gradients, VarDTC_minibatch
 from ..inference.latent_function_inference.var_dtc_gpu import VarDTC_GPU
+from ..kern._src.psi_comp.ssrbf_psi_gpucomp import PSICOMP_SSRBF_GPU
 
-
+class SSGPLVM(SparseGP_MPI):
-class SSGPLVM(SparseGP):
     """
     Spike-and-Slab Gaussian Process Latent Variable Model
 
@@ -27,8 +23,10 @@ class SSGPLVM(SparseGP):
     :type init: 'PCA'|'random'
 
     """
-    def __init__(self, Y, input_dim, X=None, X_variance=None, init='PCA', num_inducing=10,
+    def __init__(self, Y, input_dim, X=None, X_variance=None, Gamma=None, init='PCA', num_inducing=10,
-                 Z=None, kernel=None, inference_method=None, likelihood=None, name='Spike-and-Slab GPLVM', group_spike=False, **kwargs):
+                 Z=None, kernel=None, inference_method=None, likelihood=None, name='Spike_and_Slab GPLVM', group_spike=False, mpi_comm=None, pi=None, learnPi=True,normalizer=False, **kwargs):
+
+        self.group_spike = group_spike
         
         if X == None:
             from ..util.initialization import initialize_latent
@@ -41,11 +39,13 @@ class SSGPLVM(SparseGP):
         if X_variance is None: # The variance of the variational approximation (S)
             X_variance = np.random.uniform(0,.1,X.shape)
             
+        if Gamma is None:
             gamma = np.empty_like(X) # The posterior probabilities of the binary variable in the variational approximation
-        gamma[:] = 0.5 + 0.01 * np.random.randn(X.shape[0], input_dim)
+            gamma[:] = 0.5 + 0.1 * np.random.randn(X.shape[0], input_dim)
-        
+            gamma[gamma>1.-1e-9] = 1.-1e-9
-        if group_spike:
+            gamma[gamma<1e-9] = 1e-9
-            gamma[:] = gamma.mean(axis=0)
+        else:
+            gamma = Gamma.copy()
                 
         if Z is None:
             Z = np.random.permutation(X.copy())[:num_inducing]
@@ -56,31 +56,39 @@ class SSGPLVM(SparseGP):
 
         if kernel is None:
             kernel = kern.RBF(input_dim, lengthscale=fracs, ARD=True) # + kern.white(input_dim)
+        if kernel.useGPU:
+            kernel.psicomp = PSICOMP_SSRBF_GPU()
         
+        if inference_method is None:
+            inference_method = VarDTC_minibatch(mpi_comm=mpi_comm)
+
+        if pi is None:
             pi = np.empty((input_dim))
             pi[:] = 0.5
-        self.variational_prior = SpikeAndSlabPrior(pi=pi) # the prior probability of the latent binary variable b
+        self.variational_prior = SpikeAndSlabPrior(pi=pi,learnPi=learnPi) # the prior probability of the latent binary variable b
+        
         X = SpikeAndSlabPosterior(X, X_variance, gamma)
                 
-        if group_spike:
+        super(SSGPLVM,self).__init__(X, Y, Z, kernel, likelihood, variational_prior=self.variational_prior, inference_method=inference_method, name=name, mpi_comm=mpi_comm, normalizer=normalizer, **kwargs)
-            kernel.group_spike_prob = True
+        self.X.unfix()
-            self.variational_prior.group_spike_prob = True
+        self.X.variance.constrain_positive()
                 
-
+        if self.group_spike:
-        SparseGP.__init__(self, X, Y, Z, kernel, likelihood, inference_method, name, **kwargs)
+            [self.X.gamma[:,i].tie('tieGamma'+str(i)) for i in xrange(self.X.gamma.shape[1])] # Tie columns together
-        self.add_parameter(self.X, index=0)
-        self.add_parameter(self.variational_prior)
         
     def set_X_gradients(self, X, X_grad):
         """Set the gradients of the posterior distribution of X in its specific form."""
         X.mean.gradient, X.variance.gradient, X.binary_prob.gradient = X_grad
     
+    def get_X_gradients(self, X):
+        """Get the gradients of the posterior distribution of X in its specific form."""
+        return X.mean.gradient, X.variance.gradient, X.binary_prob.gradient
+
     def parameters_changed(self):
-        if isinstance(self.inference_method, VarDTC_GPU):
+        super(SSGPLVM,self).parameters_changed()
-            update_gradients(self)
+        if isinstance(self.inference_method, VarDTC_minibatch):
             return
         
-        super(SSGPLVM, self).parameters_changed()
         self._log_marginal_likelihood -= self.variational_prior.KL_divergence(self.X)
 
         self.X.mean.gradient, self.X.variance.gradient, self.X.binary_prob.gradient = self.kern.gradients_qX_expectations(variational_posterior=self.X, Z=self.Z, dL_dpsi0=self.grad_dict['dL_dpsi0'], dL_dpsi1=self.grad_dict['dL_dpsi1'], dL_dpsi2=self.grad_dict['dL_dpsi2'])
@@ -101,235 +109,3 @@ class SSGPLVM(SparseGP):
 
         return dim_reduction_plots.plot_latent(self, plot_inducing=plot_inducing, *args, **kwargs)
 
-    def do_test_latents(self, Y):
-        """
-        Compute the latent representation for a set of new points Y
-
-        Notes:
-        This will only work with a univariate Gaussian likelihood (for now)
-        """
-        assert not self.likelihood.is_heteroscedastic
-        N_test = Y.shape[0]
-        input_dim = self.Z.shape[1]
-        means = np.zeros((N_test, input_dim))
-        covars = np.zeros((N_test, input_dim))
-
-        dpsi0 = -0.5 * self.output_dim * self.likelihood.precision
-        dpsi2 = self.dL_dpsi2[0][None, :, :] # TODO: this may change if we ignore het. likelihoods
-        V = self.likelihood.precision * Y
-
-        #compute CPsi1V
-        if self.Cpsi1V is None:
-            psi1V = np.dot(self.psi1.T, self.likelihood.V)
-            tmp, _ = linalg.dtrtrs(self._Lm, np.asfortranarray(psi1V), lower=1, trans=0)
-            tmp, _ = linalg.dpotrs(self.LB, tmp, lower=1)
-            self.Cpsi1V, _ = linalg.dtrtrs(self._Lm, tmp, lower=1, trans=1)
-
-        dpsi1 = np.dot(self.Cpsi1V, V.T)
-
-        start = np.zeros(self.input_dim * 2)
-
-        for n, dpsi1_n in enumerate(dpsi1.T[:, :, None]):
-            args = (self.kern, self.Z, dpsi0, dpsi1_n.T, dpsi2)
-            xopt, fopt, neval, status = SCG(f=latent_cost, gradf=latent_grad, x=start, optargs=args, display=False)
-
-            mu, log_S = xopt.reshape(2, 1, -1)
-            means[n] = mu[0].copy()
-            covars[n] = np.exp(log_S[0]).copy()
-
-        return means, covars
-
-    def dmu_dX(self, Xnew):
-        """
-        Calculate the gradient of the prediction at Xnew w.r.t Xnew.
-        """
-        dmu_dX = np.zeros_like(Xnew)
-        for i in range(self.Z.shape[0]):
-            dmu_dX += self.kern.dK_dX(self.Cpsi1Vf[i:i + 1, :], Xnew, self.Z[i:i + 1, :])
-        return dmu_dX
-
-    def dmu_dXnew(self, Xnew):
-        """
-        Individual gradient of prediction at Xnew w.r.t. each sample in Xnew
-        """
-        dK_dX = np.zeros((Xnew.shape[0], self.num_inducing))
-        ones = np.ones((1, 1))
-        for i in range(self.Z.shape[0]):
-            dK_dX[:, i] = self.kern.dK_dX(ones, Xnew, self.Z[i:i + 1, :]).sum(-1)
-        return np.dot(dK_dX, self.Cpsi1Vf)
-
-    def plot_steepest_gradient_map(self, fignum=None, ax=None, which_indices=None, labels=None, data_labels=None, data_marker='o', data_s=40, resolution=20, aspect='auto', updates=False, ** kwargs):
-        input_1, input_2 = significant_dims = most_significant_input_dimensions(self, which_indices)
-
-        X = np.zeros((resolution ** 2, self.input_dim))
-        indices = np.r_[:X.shape[0]]
-        if labels is None:
-            labels = range(self.output_dim)
-
-        def plot_function(x):
-            X[:, significant_dims] = x
-            dmu_dX = self.dmu_dXnew(X)
-            argmax = np.argmax(dmu_dX, 1)
-            return dmu_dX[indices, argmax], np.array(labels)[argmax]
-
-        if ax is None:
-            fig = pyplot.figure(num=fignum)
-            ax = fig.add_subplot(111)
-
-        if data_labels is None:
-            data_labels = np.ones(self.num_data)
-        ulabels = []
-        for lab in data_labels:
-            if not lab in ulabels:
-                ulabels.append(lab)
-        marker = itertools.cycle(list(data_marker))
-        from GPy.util import Tango
-        for i, ul in enumerate(ulabels):
-            if type(ul) is np.string_:
-                this_label = ul
-            elif type(ul) is np.int64:
-                this_label = 'class %i' % ul
-            else:
-                this_label = 'class %i' % i
-            m = marker.next()
-            index = np.nonzero(data_labels == ul)[0]
-            x = self.X[index, input_1]
-            y = self.X[index, input_2]
-            ax.scatter(x, y, marker=m, s=data_s, color=Tango.nextMedium(), label=this_label)
-
-        ax.set_xlabel('latent dimension %i' % input_1)
-        ax.set_ylabel('latent dimension %i' % input_2)
-
-        from matplotlib.cm import get_cmap
-        from GPy.util.latent_space_visualizations.controllers.imshow_controller import ImAnnotateController
-        if not 'cmap' in kwargs.keys():
-            kwargs.update(cmap=get_cmap('jet'))
-        controller = ImAnnotateController(ax,
-                                      plot_function,
-                                      tuple(self.X.min(0)[:, significant_dims]) + tuple(self.X.max(0)[:, significant_dims]),
-                                      resolution=resolution,
-                                      aspect=aspect,
-                                      **kwargs)
-        ax.legend()
-        ax.figure.tight_layout()
-        if updates:
-            pyplot.show()
-            clear = raw_input('Enter to continue')
-            if clear.lower() in 'yes' or clear == '':
-                controller.deactivate()
-        return controller.view
-
-    def plot_X_1d(self, fignum=None, ax=None, colors=None):
-        """
-        Plot latent space X in 1D:
-
-            - if fig is given, create input_dim subplots in fig and plot in these
-            - if ax is given plot input_dim 1D latent space plots of X into each `axis`
-            - if neither fig nor ax is given create a figure with fignum and plot in there
-
-        colors:
-            colors of different latent space dimensions input_dim
-
-        """
-        import pylab
-        if ax is None:
-            fig = pylab.figure(num=fignum, figsize=(8, min(12, (2 * self.X.shape[1]))))
-        if colors is None:
-            colors = pylab.gca()._get_lines.color_cycle
-            pylab.clf()
-        else:
-            colors = iter(colors)
-        plots = []
-        x = np.arange(self.X.shape[0])
-        for i in range(self.X.shape[1]):
-            if ax is None:
-                a = fig.add_subplot(self.X.shape[1], 1, i + 1)
-            elif isinstance(ax, (tuple, list)):
-                a = ax[i]
-            else:
-                raise ValueError("Need one ax per latent dimnesion input_dim")
-            a.plot(self.X, c='k', alpha=.3)
-            plots.extend(a.plot(x, self.X.T[i], c=colors.next(), label=r"$\mathbf{{X_{{{}}}}}$".format(i)))
-            a.fill_between(x,
-                            self.X.T[i] - 2 * np.sqrt(self.X_variance.T[i]),
-                            self.X.T[i] + 2 * np.sqrt(self.X_variance.T[i]),
-                            facecolor=plots[-1].get_color(),
-                            alpha=.3)
-            a.legend(borderaxespad=0.)
-            a.set_xlim(x.min(), x.max())
-            if i < self.X.shape[1] - 1:
-                a.set_xticklabels('')
-        pylab.draw()
-        if ax is None:
-            fig.tight_layout(h_pad=.01) # , rect=(0, 0, 1, .95))
-        return fig
-
-    def getstate(self):
-        """
-        Get the current state of the class,
-        here just all the indices, rest can get recomputed
-        """
-        return SparseGP._getstate(self) + [self.init]
-
-    def setstate(self, state):
-        self._const_jitter = None
-        self.init = state.pop()
-        SparseGP._setstate(self, state)
-
-
-def latent_cost_and_grad(mu_S, kern, Z, dL_dpsi0, dL_dpsi1, dL_dpsi2):
-    """
-    objective function for fitting the latent variables for test points
-    (negative log-likelihood: should be minimised!)
-    """
-    mu, log_S = mu_S.reshape(2, 1, -1)
-    S = np.exp(log_S)
-
-    psi0 = kern.psi0(Z, mu, S)
-    psi1 = kern.psi1(Z, mu, S)
-    psi2 = kern.psi2(Z, mu, S)
-
-    lik = dL_dpsi0 * psi0 + np.dot(dL_dpsi1.flatten(), psi1.flatten()) + np.dot(dL_dpsi2.flatten(), psi2.flatten()) - 0.5 * np.sum(np.square(mu) + S) + 0.5 * np.sum(log_S)
-
-    mu0, S0 = kern.dpsi0_dmuS(dL_dpsi0, Z, mu, S)
-    mu1, S1 = kern.dpsi1_dmuS(dL_dpsi1, Z, mu, S)
-    mu2, S2 = kern.dpsi2_dmuS(dL_dpsi2, Z, mu, S)
-
-    dmu = mu0 + mu1 + mu2 - mu
-    # dS = S0 + S1 + S2 -0.5 + .5/S
-    dlnS = S * (S0 + S1 + S2 - 0.5) + .5
-    return -lik, -np.hstack((dmu.flatten(), dlnS.flatten()))
-
-def latent_cost(mu_S, kern, Z, dL_dpsi0, dL_dpsi1, dL_dpsi2):
-    """
-    objective function for fitting the latent variables (negative log-likelihood: should be minimised!)
-    This is the same as latent_cost_and_grad but only for the objective
-    """
-    mu, log_S = mu_S.reshape(2, 1, -1)
-    S = np.exp(log_S)
-
-    psi0 = kern.psi0(Z, mu, S)
-    psi1 = kern.psi1(Z, mu, S)
-    psi2 = kern.psi2(Z, mu, S)
-
-    lik = dL_dpsi0 * psi0 + np.dot(dL_dpsi1.flatten(), psi1.flatten()) + np.dot(dL_dpsi2.flatten(), psi2.flatten()) - 0.5 * np.sum(np.square(mu) + S) + 0.5 * np.sum(log_S)
-    return -float(lik)
-
-def latent_grad(mu_S, kern, Z, dL_dpsi0, dL_dpsi1, dL_dpsi2):
-    """
-    This is the same as latent_cost_and_grad but only for the grad
-    """
-    mu, log_S = mu_S.reshape(2, 1, -1)
-    S = np.exp(log_S)
-
-    mu0, S0 = kern.dpsi0_dmuS(dL_dpsi0, Z, mu, S)
-    mu1, S1 = kern.dpsi1_dmuS(dL_dpsi1, Z, mu, S)
-    mu2, S2 = kern.dpsi2_dmuS(dL_dpsi2, Z, mu, S)
-
-    dmu = mu0 + mu1 + mu2 - mu
-    # dS = S0 + S1 + S2 -0.5 + .5/S
-    dlnS = S * (S0 + S1 + S2 - 0.5) + .5
-
-    return -np.hstack((dmu.flatten(), dlnS.flatten()))
-
-

GPy/models/ss_mrd.py

@@ -0,0 +1,34 @@
+"""
+The Maniforld Relevance Determination model with the spike-and-slab prior
+"""
+
+from ..core import Model
+from .ss_gplvm import SSGPLVM
+
+class SSMRD(Model):
+    
+    def __init__(self, Ylist, input_dim, X=None, X_variance=None,
+                 initx = 'PCA', initz = 'permute',
+                 num_inducing=10, Z=None, kernel=None,
+                 inference_method=None, likelihoods=None, name='ss_mrd', Ynames=None):
+        super(SSMRD, self).__init__(name)
+        
+        self.updates = False
+        self.models = [SSGPLVM(y, input_dim, X=X, X_variance=X_variance, num_inducing=num_inducing,Z=Z,init=initx,
+                               kernel=kernel.copy() if kernel else None,inference_method=inference_method,likelihood=likelihoods,
+                               name='model_'+str(i)) for i,y in enumerate(Ylist)]
+        self.add_parameters(*(self.models))
+        
+        [[[self.models[m].X.mean[i,j:j+1].tie('mean_'+str(i)+'_'+str(j)) for m in xrange(len(self.models))] for j in xrange(self.models[0].X.mean.shape[1])] 
+         for i in xrange(self.models[0].X.mean.shape[0])]
+        [[[self.models[m].X.variance[i,j:j+1].tie('var_'+str(i)+'_'+str(j)) for m in xrange(len(self.models))] for j in xrange(self.models[0].X.variance.shape[1])] 
+         for i in xrange(self.models[0].X.variance.shape[0])]
+        
+        self.updates = True
+        
+    def parameters_changed(self):
+        super(SSMRD, self).parameters_changed()        
+        self._log_marginal_likelihood = sum([m._log_marginal_likelihood for m in self.models])
+
+    def log_likelihood(self):
+        return self._log_marginal_likelihood

GPy/plotting/matplot_dep/dim_reduction_plots.py

@@ -31,7 +31,7 @@ def plot_latent(model, labels=None, which_indices=None,
                 resolution=50, ax=None, marker='o', s=40,
                 fignum=None, plot_inducing=False, legend=True,
                 plot_limits=None,
-                aspect='auto', updates=False, **kwargs):
+                aspect='auto', updates=False, predict_kwargs={}, imshow_kwargs={}):
     """
     :param labels: a np.array of size model.num_data containing labels for the points (can be number, strings, etc)
     :param resolution: the resolution of the grid on which to evaluate the predictive variance
@@ -56,11 +56,62 @@ def plot_latent(model, labels=None, which_indices=None,
         X = param_to_array(X)
 
 
+    if X.shape[0] > 1000:
+        print "Warning: subsampling X, as it has more samples then 1000. X.shape={!s}".format(X.shape)
+        subsample = np.random.choice(X.shape[0], size=1000, replace=False)
+        X = X[subsample]
+        labels = labels[subsample]
+        #=======================================================================
+        #     <<<WORK IN PROGRESS>>>
+        #     <<<DO NOT DELETE>>>
+        #     plt.close('all')
+        #     fig, ax = plt.subplots(1,1)
+        #     from GPy.plotting.matplot_dep.dim_reduction_plots import most_significant_input_dimensions
+        #     import matplotlib.patches as mpatches
+        #     i1, i2 = most_significant_input_dimensions(m, None)
+        #     xmin, xmax = 100, -100
+        #     ymin, ymax = 100, -100
+        #     legend_handles = []
+        #
+        #     X = m.X.mean[:, [i1, i2]]
+        #     X = m.X.variance[:, [i1, i2]]
+        #
+        #     xmin = X[:,0].min(); xmax = X[:,0].max()
+        #     ymin = X[:,1].min(); ymax = X[:,1].max()
+        #     range_ = [[xmin, xmax], [ymin, ymax]]
+        #     ul = np.unique(labels)
+        #
+        #     for i, l in enumerate(ul):
+        #         #cdict = dict(red  =[(0., colors[i][0], colors[i][0]), (1., colors[i][0], colors[i][0])],
+        #         #             green=[(0., colors[i][0], colors[i][1]), (1., colors[i][1], colors[i][1])],
+        #         #             blue =[(0., colors[i][0], colors[i][2]), (1., colors[i][2], colors[i][2])],
+        #         #             alpha=[(0., 0., .0), (.5, .5, .5), (1., .5, .5)])
+        #         #cmap = LinearSegmentedColormap('{}'.format(l), cdict)
+        #         cmap = LinearSegmentedColormap.from_list('cmap_{}'.format(str(l)), [colors[i], colors[i]], 255)
+        #         cmap._init()
+        #         #alphas = .5*(1+scipy.special.erf(np.linspace(-2,2, cmap.N+3)))#np.log(np.linspace(np.exp(0), np.exp(1.), cmap.N+3))
+        #         alphas = (scipy.special.erf(np.linspace(0,2.4, cmap.N+3)))#np.log(np.linspace(np.exp(0), np.exp(1.), cmap.N+3))
+        #         cmap._lut[:, -1] = alphas
+        #         print l
+        #         x, y = X[labels==l].T
+        #
+        #         heatmap, xedges, yedges = np.histogram2d(x, y, bins=300, range=range_)
+        #         #heatmap, xedges, yedges = np.histogram2d(x, y, bins=100)
+        #
+        #         im = ax.imshow(heatmap, extent=[xedges[0], xedges[-1], yedges[0], yedges[-1]], cmap=cmap, aspect='auto', interpolation='nearest', label=str(l))
+        #         legend_handles.append(mpatches.Patch(color=colors[i], label=l))
+        #     ax.set_xlim(xmin, xmax)
+        #     ax.set_ylim(ymin, ymax)
+        #     plt.legend(legend_handles, [l.get_label() for l in legend_handles])
+        #     plt.draw()
+        #     plt.show()
+        #=======================================================================
+
     # create a function which computes the shading of latent space according to the output variance
     def plot_function(x):
         Xtest_full = np.zeros((x.shape[0], model.X.shape[1]))
         Xtest_full[:, [input_1, input_2]] = x
-        _, var = model.predict(Xtest_full)
+        _, var = model.predict(Xtest_full, **predict_kwargs)
         var = var[:, :1]
         return np.log(var)
 
@@ -81,9 +132,10 @@ def plot_latent(model, labels=None, which_indices=None,
     view = ImshowController(ax, plot_function,
                             (xmin, ymin, xmax, ymax),
                             resolution, aspect=aspect, interpolation='bilinear',
-                            cmap=pb.cm.binary, **kwargs)
+                            cmap=pb.cm.binary, **imshow_kwargs)
 
     # make sure labels are in order of input:
+    labels = np.asarray(labels)
     ulabels = []
     for lab in labels:
         if not lab in ulabels:
@@ -97,7 +149,7 @@ def plot_latent(model, labels=None, which_indices=None,
         elif type(ul) is np.int64:
             this_label = 'class %i' % ul
         else:
-            this_label = 'class %i' % i
+            this_label = unicode(ul)
         m = marker.next()
 
         index = np.nonzero(labels == ul)[0]

GPy/plotting/matplot_dep/kernel_plots.py

@@ -41,7 +41,7 @@ def plot_bars(fig, ax, x, ard_params, color, name, bottom=0):
                   color=color, edgecolor='k', linewidth=1.2,
                   label=name.replace("_"," "))
 
-def plot_ARD(kernel, fignum=None, ax=None, title='', legend=False):
+def plot_ARD(kernel, fignum=None, ax=None, title='', legend=False, filtering=None):
     """
     If an ARD kernel is present, plot a bar representation using matplotlib
 
@@ -51,6 +51,10 @@ def plot_ARD(kernel, fignum=None, ax=None, title='', legend=False):
         title of the plot,
         pass '' to not print a title
         pass None for a generic title
+    :param filtering: list of names, which to use for plotting ARD parameters.
+                      Only kernels which match names in the list of names in filtering
+                      will be used for plotting.
+    :type filtering: list of names to use for ARD plot
     """
     fig, ax = ax_default(fignum,ax)
 
@@ -62,17 +66,26 @@ def plot_ARD(kernel, fignum=None, ax=None, title='', legend=False):
     Tango.reset()
     bars = []
 
-    ard_params = np.atleast_2d(kernel.input_sensitivity())
+    ard_params = np.atleast_2d(kernel.input_sensitivity(summarize=False))
     bottom = 0
+    last_bottom = bottom
+
     x = np.arange(kernel.input_dim)
 
+    if order is None:
+        order = kernel.parameter_names(recursive=False)
+
     for i in range(ard_params.shape[0]):
+        if kernel.parameters[i].name in order:
             c = Tango.nextMedium()
-        bars.append(plot_bars(fig, ax, x, ard_params[i,:], c, kernel._parameters_[i].name, bottom=bottom))
+            bars.append(plot_bars(fig, ax, x, ard_params[i,:], c, kernel.parameters[i].name, bottom=bottom))
-        bottom += ard_params[i,:]
+            last_bottom = ard_params[i,:]
+            bottom += last_bottom
+        else:
+            print "filtering out {}".format(kernel.parameters[i].name)
 
     ax.set_xlim(-.5, kernel.input_dim - .5)
-    add_bar_labels(fig, ax, [bars[-1]], bottom=bottom-ard_params[i,:])
+    add_bar_labels(fig, ax, [bars[-1]], bottom=bottom-last_bottom)
 
     if legend:
         if title is '':

GPy/plotting/matplot_dep/models_plots.py

@@ -8,13 +8,13 @@ from base_plots import gpplot, x_frame1D, x_frame2D
 from ...util.misc import param_to_array
 from ...models.gp_coregionalized_regression import GPCoregionalizedRegression
 from ...models.sparse_gp_coregionalized_regression import SparseGPCoregionalizedRegression
-
+from scipy import sparse
 
 def plot_fit(model, plot_limits=None, which_data_rows='all',
         which_data_ycols='all', fixed_inputs=[],
         levels=20, samples=0, fignum=None, ax=None, resolution=None,
         plot_raw=False,
-        linecol=Tango.colorsHex['darkBlue'],fillcol=Tango.colorsHex['lightBlue'], Y_metadata=None):
+        linecol=Tango.colorsHex['darkBlue'],fillcol=Tango.colorsHex['lightBlue'], Y_metadata=None, data_symbol='kx'):
     """
     Plot the posterior of the GP.
       - In one dimension, the function is plotted with a shaded region identifying two standard deviations.
@@ -61,11 +61,14 @@ def plot_fit(model, plot_limits=None, which_data_rows='all',
 
     if hasattr(model, 'has_uncertain_inputs') and model.has_uncertain_inputs():
         X = model.X.mean
-        X_variance = param_to_array(model.X.variance)
+        X_variance = model.X.variance
     else:
         X = model.X
-    X, Y = param_to_array(X, model.Y)
+    #X, Y = param_to_array(X, model.Y)
-    if hasattr(model, 'Z'): Z = param_to_array(model.Z)
+    Y = model.Y
+    if sparse.issparse(Y): Y = Y.todense().view(np.ndarray)
+
+    if hasattr(model, 'Z'): Z = model.Z
 
     #work out what the inputs are for plotting (1D or 2D)
     fixed_dims = np.array([i for i,v in fixed_inputs])
@@ -97,7 +100,7 @@ def plot_fit(model, plot_limits=None, which_data_rows='all',
 
         for d in which_data_ycols:
             plots['gpplot'] = gpplot(Xnew, m[:, d], lower[:, d], upper[:, d], ax=ax, edgecol=linecol, fillcol=fillcol)
-            plots['dataplot'] = ax.plot(X[which_data_rows,free_dims], Y[which_data_rows, d], 'kx', mew=1.5)
+            if not plot_raw: plots['dataplot'] = ax.plot(X[which_data_rows,free_dims], Y[which_data_rows, d], data_symbol, mew=1.5)
 
         #optionally plot some samples
         if samples: #NOTE not tested with fixed_inputs
@@ -147,11 +150,15 @@ def plot_fit(model, plot_limits=None, which_data_rows='all',
         if plot_raw:
             m, _ = model._raw_predict(Xgrid)
         else:
-            m, _ = model.predict(Xgrid)
+            if isinstance(model,GPCoregionalizedRegression) or isinstance(model,SparseGPCoregionalizedRegression):
+                meta = {'output_index': Xgrid[:,-1:].astype(np.int)}
+            else:
+                meta = None
+            m, v = model.predict(Xgrid, full_cov=False, Y_metadata=meta)
         for d in which_data_ycols:
             m_d = m[:,d].reshape(resolution, resolution).T
             plots['contour'] = ax.contour(x, y, m_d, levels, vmin=m.min(), vmax=m.max(), cmap=pb.cm.jet)
-            plots['dataplot'] = ax.scatter(X[which_data_rows, free_dims[0]], X[which_data_rows, free_dims[1]], 40, Y[which_data_rows, d], cmap=pb.cm.jet, vmin=m.min(), vmax=m.max(), linewidth=0.)
+            if not plot_raw: plots['dataplot'] = ax.scatter(X[which_data_rows, free_dims[0]], X[which_data_rows, free_dims[1]], 40, Y[which_data_rows, d], cmap=pb.cm.jet, vmin=m.min(), vmax=m.max(), linewidth=0.)
 
         #set the limits of the plot to some sensible values
         ax.set_xlim(xmin[0], xmax[0])

GPy/plotting/matplot_dep/variational_plots.py

@@ -93,7 +93,7 @@ def plot_SpikeSlab(parameterized, fignum=None, ax=None, colors=None, side_by_sid
             a.set_xticklabels('')
         # binary prob plot
         a = fig.add_subplot(*sub2)
-        a.bar(x,gamma[:,i],bottom=0.,linewidth=0,align='center')
+        a.bar(x,gamma[:,i],bottom=0.,linewidth=0,width=1.0,align='center')
         a.set_xlim(x.min(), x.max())
         a.set_ylim([0.,1.])
     pb.draw()

GPy/plotting/matplot_dep/visualize.py

@@ -74,19 +74,21 @@ class vector_show(matplotlib_show):
     """
     def __init__(self, vals, axes=None):
         matplotlib_show.__init__(self, vals, axes)
-        self.handle = self.axes.plot(np.arange(0, len(vals))[:, None], self.vals)
+        #assert vals.ndim == 2, "Please give a vector in [n x 1] to plot"
+        #assert vals.shape[1] == 1, "only showing a vector in one dimension"
+        self.size = vals.size
+        self.handle = self.axes.plot(np.arange(0, vals.size)[:, None], vals)[0]
 
     def modify(self, vals):
         self.vals = vals.copy()
-        for handle, vals in zip(self.handle, self.vals.T):
+        xdata, ydata = self.handle.get_data()
-            xdata, ydata = handle.get_data()
+        assert vals.size == self.size, "values passed into modify changed size! vals.size:{} != in.size:{}".format(vals.size, self.size)
-            handle.set_data(xdata, vals)
+        self.handle.set_data(xdata, self.vals)
         self.axes.figure.canvas.draw()
 
 
 class lvm(matplotlib_show):
-    
+    def __init__(self, vals, model, data_visualize, latent_axes=None, sense_axes=None, latent_index=[0,1], disable_drag=False):
-    def __init__(self, vals, model, data_visualize, latent_axes=None, sense_axes=None, latent_index=[0,1]):
         """Visualize a latent variable model
 
         :param model: the latent variable model to visualize.
@@ -94,22 +96,24 @@ class lvm(matplotlib_show):
         :type data_visualize: visualize.data_show  type.
         :param latent_axes: the axes where the latent visualization should be plotted.
         """
-        if vals == None:
+        if vals is None:
             if isinstance(model.X, VariationalPosterior):
-                vals = param_to_array(model.X.mean)
+                vals = model.X.mean.values
             else:
-                vals = param_to_array(model.X)
+                vals = model.X.values
-         
+        if len(vals.shape)==1:
-        vals = param_to_array(vals)
+            vals = vals[None,:]
         matplotlib_show.__init__(self, vals, axes=latent_axes)
 
         if isinstance(latent_axes,mpl.axes.Axes):
             self.cid = latent_axes.figure.canvas.mpl_connect('button_press_event', self.on_click)
+            if not disable_drag:
                 self.cid = latent_axes.figure.canvas.mpl_connect('motion_notify_event', self.on_move)
             self.cid = latent_axes.figure.canvas.mpl_connect('axes_leave_event', self.on_leave)
             self.cid = latent_axes.figure.canvas.mpl_connect('axes_enter_event', self.on_enter)
         else:
             self.cid = latent_axes[0].figure.canvas.mpl_connect('button_press_event', self.on_click)
+            if not disable_drag:
                 self.cid = latent_axes[0].figure.canvas.mpl_connect('motion_notify_event', self.on_move)
             self.cid = latent_axes[0].figure.canvas.mpl_connect('axes_leave_event', self.on_leave)
             self.cid = latent_axes[0].figure.canvas.mpl_connect('axes_enter_event', self.on_enter)
@@ -122,6 +126,7 @@ class lvm(matplotlib_show):
         self.move_on = False
         self.latent_index = latent_index
         self.latent_dim = model.input_dim
+        self.disable_drag = disable_drag
  
         # The red cross which shows current latent point.
         self.latent_values = vals
@@ -131,7 +136,7 @@ class lvm(matplotlib_show):
 
     def modify(self, vals):
         """When latent values are modified update the latent representation and ulso update the output visualization."""
-        self.vals = vals.copy()
+        self.vals = vals.view(np.ndarray).copy()
         y = self.model.predict(self.vals)[0]
         self.data_visualize.modify(y)
         self.latent_handle.set_data(self.vals[0,self.latent_index[0]], self.vals[0,self.latent_index[1]])
@@ -144,8 +149,12 @@ class lvm(matplotlib_show):
         pass
 
     def on_click(self, event):
-        print 'click!'
         if event.inaxes!=self.latent_axes: return
+        if self.disable_drag:
+            self.move_on = True
+            self.called = True
+            self.on_move(event)
+        else:
             self.move_on = not self.move_on
             self.called = True
 
@@ -217,11 +226,11 @@ class lvm_dimselect(lvm):
         self.labels = labels
         lvm.__init__(self,vals,model,data_visualize,latent_axes,sense_axes,latent_index)
         self.show_sensitivities()
-        print "use left and right mouse butons to select dimensions"
+        print self.latent_values
+        print "use left and right mouse buttons to select dimensions"
 
 
     def on_click(self, event):
-
         if event.inaxes==self.sense_axes:
             new_index = max(0,min(int(np.round(event.xdata-0.5)),self.model.input_dim-1))
             if event.button == 1:
@@ -247,6 +256,7 @@ class lvm_dimselect(lvm):
 
 
     def on_leave(self,event):
+        print type(self.latent_values)
         latent_values = self.latent_values.copy()
         y = self.model.predict(latent_values[None,:])[0]
         self.data_visualize.modify(y)
@@ -272,8 +282,11 @@ class image_show(matplotlib_show):
     :param preset_mean: the preset mean of a scaled image.
     :type preset_mean: double
     :param preset_std: the preset standard deviation of a scaled image.
-    :type preset_std: double"""
+    :type preset_std: double
-    def __init__(self, vals, axes=None, dimensions=(16,16), transpose=False, order='C', invert=False, scale=False, palette=[], preset_mean=0., preset_std=1., select_image=0):
+    :param cmap: the colormap for image visualization
+    :type cmap: matplotlib.cm"""
+
+    def __init__(self, vals, axes=None, dimensions=(16,16), transpose=False, order='C', invert=False, scale=False, palette=[], preset_mean=0., preset_std=1., select_image=0, cmap=None):
         matplotlib_show.__init__(self, vals, axes)
         self.dimensions = dimensions
         self.transpose = transpose
@@ -288,8 +301,10 @@ class image_show(matplotlib_show):
         self.set_image(self.vals)
         if not self.palette == []: # Can just show the image (self.set_image() took care of setting the palette)
             self.handle = self.axes.imshow(self.vals, interpolation='nearest')
-        else: # Use a boring gray map.
+        elif cmap==None: # Use a jet map.
-            self.handle = self.axes.imshow(self.vals, cmap=plt.cm.gray, interpolation='nearest') # @UndefinedVariable
+            self.handle = self.axes.imshow(self.vals, cmap=plt.cm.jet, interpolation='nearest') # @UndefinedVariable
+        else: # Use the selected map.
+            self.handle = self.axes.imshow(self.vals, cmap=cmap, interpolation='nearest') # @UndefinedVariable
         plt.show()
 
     def modify(self, vals):
@@ -391,14 +406,13 @@ class mocap_data_show_vpython(vpython_show):
     def process_values(self):
         raise NotImplementedError, "this needs to be implemented to use the data_show class"
 
-
 class mocap_data_show(matplotlib_show):
     """Base class for visualizing motion capture data."""
 
     def __init__(self, vals, axes=None, connect=None):
         if axes==None:
             fig = plt.figure()
-            axes = fig.add_subplot(111, projection='3d')
+            axes = fig.add_subplot(111, projection='3d', aspect='equal')
         matplotlib_show.__init__(self, vals, axes)
 
         self.connect = connect
@@ -436,6 +450,7 @@ class mocap_data_show(matplotlib_show):
         self.process_values()
         self.initialize_axes_modify()
         self.draw_vertices()
+        self.initialize_axes()
         self.finalize_axes_modify()
         self.draw_edges()
         self.axes.figure.canvas.draw()
@@ -443,11 +458,12 @@ class mocap_data_show(matplotlib_show):
     def process_values(self):
         raise NotImplementedError, "this needs to be implemented to use the data_show class"
 
-    def initialize_axes(self):
+    def initialize_axes(self, boundary=0.05):
         """Set up the axes with the right limits and scaling."""
-        self.x_lim = np.array([self.vals[:, 0].min(), self.vals[:, 0].max()])
+        bs = [(self.vals[:, i].max()-self.vals[:, i].min())*boundary for i in xrange(3)]
-        self.y_lim = np.array([self.vals[:, 1].min(), self.vals[:, 1].max()])
+        self.x_lim = np.array([self.vals[:, 0].min()-bs[0], self.vals[:, 0].max()+bs[0]])
-        self.z_lim = np.array([self.vals[:, 2].min(), self.vals[:, 2].max()])
+        self.y_lim = np.array([self.vals[:, 1].min()-bs[1], self.vals[:, 1].max()+bs[1]])
+        self.z_lim = np.array([self.vals[:, 2].min()-bs[2], self.vals[:, 2].max()+bs[2]])
 
     def initialize_axes_modify(self):
         self.points_handle.remove()
@@ -457,10 +473,10 @@ class mocap_data_show(matplotlib_show):
         self.axes.set_xlim(self.x_lim)
         self.axes.set_ylim(self.y_lim)
         self.axes.set_zlim(self.z_lim)
-        self.axes.auto_scale_xyz([-1., 1.], [-1., 1.], [-1.5, 1.5])
+        self.axes.auto_scale_xyz([-1., 1.], [-1., 1.], [-1., 1.])
         
-        #self.axes.set_aspect('equal')
+#        self.axes.set_aspect('equal')
-        self.axes.autoscale(enable=False)
+#         self.axes.autoscale(enable=False)
 
     def finalize_axes_modify(self):
         self.axes.set_xlim(self.x_lim)
@@ -470,6 +486,8 @@ class mocap_data_show(matplotlib_show):
 class stick_show(mocap_data_show):
     """Show a three dimensional point cloud as a figure. Connect elements of the figure together using the matrix connect."""
     def __init__(self, vals, connect=None, axes=None):
+        if len(vals.shape)==1:
+            vals = vals[None,:]
         mocap_data_show.__init__(self, vals, axes=axes, connect=connect)
 
     def process_values(self):

GPy/testing/kernel_tests.py

@@ -304,23 +304,13 @@ class KernelTestsMiscellaneous(unittest.TestCase):
     def setUp(self):
         N, D = 100, 10
         self.X = np.linspace(-np.pi, +np.pi, N)[:,None] * np.random.uniform(-10,10,D)
-        self.rbf = GPy.kern.RBF(2, active_dims=slice(0,4,2))
+        self.rbf = GPy.kern.RBF(2, active_dims=np.arange(0,4,2))
         self.linear = GPy.kern.Linear(2, active_dims=(3,9))
         self.matern = GPy.kern.Matern32(3, active_dims=np.array([1,7,9]))
         self.sumkern = self.rbf + self.linear
         self.sumkern += self.matern
         self.sumkern.randomize()
 
-    def test_active_dims(self):
-        # test the automatic dim detection expression for slices:
-        start, stop = 0, 277
-        for i in range(start,stop,7):
-            for j in range(1,4):
-                GPy.kern.Kern(int(np.round((i+1)/j)), slice(0, i+1, j), "testkern")
-        # test the ability to have only one dim
-        sk = GPy.kern.RBF(2) + GPy.kern.Matern32(2)
-        self.assertEqual(sk.input_dim, 2)
-
     def test_which_parts(self):
         self.assertTrue(np.allclose(self.sumkern.K(self.X, which_parts=[self.linear, self.matern]), self.linear.K(self.X)+self.matern.K(self.X)))
         self.assertTrue(np.allclose(self.sumkern.K(self.X, which_parts=[self.linear, self.rbf]), self.linear.K(self.X)+self.rbf.K(self.X)))
@@ -344,10 +334,15 @@ class KernelTestsNonContinuous(unittest.TestCase):
         self.X2[(N0*2):, -1] = 1
 
     def test_IndependentOutputs(self):
-        k = GPy.kern.RBF(self.D)
+        k = GPy.kern.RBF(self.D, active_dims=range(self.D))
         kern = GPy.kern.IndependentOutputs(k, -1, 'ind_single')
         self.assertTrue(check_kernel_gradient_functions(kern, X=self.X, X2=self.X2, verbose=verbose, fixed_X_dims=-1))
-        k = [GPy.kern.RBF(1, active_dims=[1], name='rbf1'), GPy.kern.RBF(self.D, name='rbf012'), GPy.kern.RBF(2, active_dims=[0,2], name='rbf02')]
+        k = [GPy.kern.RBF(1, active_dims=[1], name='rbf1'), GPy.kern.RBF(self.D, active_dims=range(self.D), name='rbf012'), GPy.kern.RBF(2, active_dims=[0,2], name='rbf02')]
+        kern = GPy.kern.IndependentOutputs(k, -1, name='ind_split')
+        self.assertTrue(check_kernel_gradient_functions(kern, X=self.X, X2=self.X2, verbose=verbose, fixed_X_dims=-1))
+
+    def test_Hierarchical(self):
+        k = [GPy.kern.RBF(2, active_dims=[0,2], name='rbf1'), GPy.kern.RBF(2, active_dims=[0,2], name='rbf2')]
         kern = GPy.kern.IndependentOutputs(k, -1, name='ind_split')
         self.assertTrue(check_kernel_gradient_functions(kern, X=self.X, X2=self.X2, verbose=verbose, fixed_X_dims=-1))
 

GPy/testing/model_tests.py

@@ -94,22 +94,18 @@ class MiscTests(unittest.TestCase):
         np.testing.assert_equal(m.log_likelihood(), m2.log_likelihood())
 
         m.kern.lengthscale.randomize()
-        m._trigger_params_changed()
         m2.kern.lengthscale = m.kern.lengthscale
         np.testing.assert_equal(m.log_likelihood(), m2.log_likelihood())
 
         m.kern.lengthscale.randomize()
-        m._trigger_params_changed()
         m2['.*lengthscale'] = m.kern.lengthscale
         np.testing.assert_equal(m.log_likelihood(), m2.log_likelihood())
 
         m.kern.lengthscale.randomize()
-        m._trigger_params_changed()
         m2['.*lengthscale'] = m.kern['.*lengthscale']
         np.testing.assert_equal(m.log_likelihood(), m2.log_likelihood())
 
         m.kern.lengthscale.randomize()
-        m._trigger_params_changed()
         m2.kern.lengthscale = m.kern['.*lengthscale']
         np.testing.assert_equal(m.log_likelihood(), m2.log_likelihood())
 
@@ -130,6 +126,23 @@ class MiscTests(unittest.TestCase):
         m2.kern[:] = m.kern[''].values()
         np.testing.assert_equal(m.log_likelihood(), m2.log_likelihood())
 
+    def test_big_model(self):
+        m = GPy.examples.dimensionality_reduction.mrd_simulation(optimize=0, plot=0, plot_sim=0)
+        m.X.fix()
+        print m
+        m.unfix()
+        m.checkgrad()
+        print m
+        m.fix()
+        print m
+        m.inducing_inputs.unfix()
+        print m
+        m.checkgrad()
+        m.unfix()
+        m.checkgrad()
+        m.checkgrad()
+        print m
+
     def test_model_set_params(self):
         m = GPy.models.GPRegression(self.X, self.Y)
         lengthscale = np.random.uniform()
@@ -410,6 +423,45 @@ class GradientTests(np.testing.TestCase):
         m = GPy.models.GPHeteroscedasticRegression(X,Y,kern)
         self.assertTrue(m.checkgrad())
 
+    def test_gp_kronecker_gaussian(self):
+        N1, N2 = 30, 20
+        X1 = np.random.randn(N1, 1)
+        X2 = np.random.randn(N2, 1)
+        X1.sort(0); X2.sort(0)
+        k1 = GPy.kern.RBF(1)  # + GPy.kern.White(1)
+        k2 = GPy.kern.RBF(1)  # + GPy.kern.White(1)
+        Y = np.random.randn(N1, N2)
+        m = GPy.models.GPKroneckerGaussianRegression(X1, X2, Y, k1, k2)
+
+        # build the model the dumb way
+        assert (N1*N2<1000), "too much data for standard GPs!"
+        yy, xx = np.meshgrid(X2, X1)
+        Xgrid = np.vstack((xx.flatten(order='F'), yy.flatten(order='F'))).T
+        kg = GPy.kern.RBF(1, active_dims=[0]) * GPy.kern.RBF(1, active_dims=[1])
+        mm = GPy.models.GPRegression(Xgrid, Y.reshape(-1, 1, order='F'), kernel=kg)
+
+        m.randomize()
+        mm[:] = m[:]
+        assert np.allclose(m.log_likelihood(), mm.log_likelihood())
+        assert np.allclose(m.gradient, mm.gradient)
+        X1test = np.random.randn(100, 1)
+        X2test = np.random.randn(100, 1)
+        mean1, var1 = m.predict(X1test, X2test)
+        yy, xx = np.meshgrid(X2test, X1test)
+        Xgrid = np.vstack((xx.flatten(order='F'), yy.flatten(order='F'))).T
+        mean2, var2 = mm.predict(Xgrid)
+        assert np.allclose(mean1, mean2)
+        assert np.allclose(var1, var2)
+
+    def test_gp_VGPC(self):
+        num_obs = 25
+        X = np.random.randint(0,140,num_obs)
+        X = X[:,None]
+        Y = 25. + np.sin(X/20.) * 2. + np.random.rand(num_obs)[:,None]
+        kern = GPy.kern.Bias(1) + GPy.kern.RBF(1)
+        m = GPy.models.GPVariationalGaussianApproximation(X,Y,kern)
+        self.assertTrue(m.checkgrad())
+
 
 if __name__ == "__main__":
     print "Running unit tests, please be (very) patient..."

GPy/testing/parameterized_tests.py

@@ -8,6 +8,7 @@ import GPy
 import numpy as np
 from GPy.core.parameterization.parameter_core import HierarchyError
 from GPy.core.parameterization.observable_array import ObsAr
+from GPy.core.parameterization.transformations import NegativeLogexp
 
 class ArrayCoreTest(unittest.TestCase):
     def setUp(self):
@@ -27,21 +28,36 @@ class ArrayCoreTest(unittest.TestCase):
 class ParameterizedTest(unittest.TestCase):
 
     def setUp(self):
-        self.rbf = GPy.kern.RBF(1)
+        self.rbf = GPy.kern.RBF(20)
         self.white = GPy.kern.White(1)
         from GPy.core.parameterization import Param
         from GPy.core.parameterization.transformations import Logistic
-        self.param = Param('param', np.random.rand(25,2), Logistic(0, 1))
+        self.param = Param('param', np.random.uniform(0,1,(25,2)), Logistic(0, 1))
 
         self.test1 = GPy.core.Parameterized("test model")
         self.test1.param = self.param
         self.test1.kern = self.rbf+self.white
         self.test1.add_parameter(self.test1.kern)
         self.test1.add_parameter(self.param, 0)
+        # print self.test1:
+        #=============================================================================
+        # test_model.          |    Value    |  Constraint   |  Prior  |  Tied to
+        # param                |  (25L, 2L)  |   {0.0,1.0}   |         |
+        # add.rbf.variance     |        1.0  |  0.0,1.0 +ve  |         |
+        # add.rbf.lengthscale  |        1.0  |  0.0,1.0 +ve  |         |
+        # add.white.variance   |        1.0  |  0.0,1.0 +ve  |         |
+        #=============================================================================
 
         x = np.linspace(-2,6,4)[:,None]
         y = np.sin(x)
         self.testmodel = GPy.models.GPRegression(x,y)
+        # print self.testmodel:
+        #=============================================================================
+        # GP_regression.           |  Value  |  Constraint  |  Prior  |  Tied to
+        # rbf.variance             |    1.0  |     +ve      |         |
+        # rbf.lengthscale          |    1.0  |     +ve      |         |
+        # Gaussian_noise.variance  |    1.0  |     +ve      |         |
+        #=============================================================================
 
     def test_add_parameter(self):
         self.assertEquals(self.rbf._parent_index_, 0)
@@ -95,7 +111,7 @@ class ParameterizedTest(unittest.TestCase):
         self.assertListEqual(self.test1.kern.param_array.tolist(), val[:2].tolist())
 
     def test_add_parameter_already_in_hirarchy(self):
-        self.assertRaises(HierarchyError, self.test1.add_parameter, self.white._parameters_[0])
+        self.assertRaises(HierarchyError, self.test1.add_parameter, self.white.parameters[0])
 
     def test_default_constraints(self):
         self.assertIs(self.rbf.variance.constraints._param_index_ops, self.rbf.constraints._param_index_ops)
@@ -142,6 +158,13 @@ class ParameterizedTest(unittest.TestCase):
         self.testmodel.randomize()
         self.assertEqual(val, self.testmodel.kern.lengthscale)
 
+    def test_add_parameter_in_hierarchy(self):
+        from GPy.core import Param
+        self.test1.kern.rbf.add_parameter(Param("NEW", np.random.rand(2), NegativeLogexp()), 1)
+        self.assertListEqual(self.test1.constraints[NegativeLogexp()].tolist(), range(self.param.size+1, self.param.size+1 + 2))
+        self.assertListEqual(self.test1.constraints[GPy.transformations.Logistic(0,1)].tolist(), range(self.param.size))
+        self.assertListEqual(self.test1.constraints[GPy.transformations.Logexp(0,1)].tolist(), np.r_[50, 53:55].tolist())
+
     def test_regular_expression_misc(self):
         self.testmodel.kern.lengthscale.fix()
         val = float(self.testmodel.kern.lengthscale)

GPy/testing/pickle_tests.py

@@ -4,7 +4,8 @@ Created on 13 Mar 2014
 @author: maxz
 '''
 import unittest, itertools
-import cPickle as pickle
+#import cPickle as pickle
+import pickle
 import numpy as np
 from GPy.core.parameterization.index_operations import ParameterIndexOperations,\
     ParameterIndexOperationsView
@@ -15,8 +16,7 @@ from GPy.core.parameterization.priors import Gaussian
 from GPy.kern._src.rbf import RBF
 from GPy.kern._src.linear import Linear
 from GPy.kern._src.static import Bias, White
-from GPy.examples.dimensionality_reduction import mrd_simulation,\
+from GPy.examples.dimensionality_reduction import mrd_simulation
-    bgplvm_simulation
 from GPy.examples.regression import toy_rbf_1d_50
 from GPy.core.parameterization.variational import NormalPosterior
 from GPy.models.gp_regression import GPRegression
@@ -89,28 +89,29 @@ class Test(ListDictTestCase):
         self.assertIs(pcopy.constraints, pcopy.rbf.lengthscale.constraints._param_index_ops)
         self.assertIs(pcopy.constraints, pcopy.linear.constraints._param_index_ops)
         self.assertListEqual(par.param_array.tolist(), pcopy.param_array.tolist())
-        self.assertListEqual(par.full_gradient.tolist(), pcopy.full_gradient.tolist())
+        pcopy.gradient = 10 # gradient does not get copied anymore
+        self.assertListEqual(par.gradient_full.tolist(), pcopy.gradient_full.tolist())
         self.assertSequenceEqual(str(par), str(pcopy))
         self.assertIsNot(par.param_array, pcopy.param_array)
-        self.assertIsNot(par.full_gradient, pcopy.full_gradient)
+        self.assertIsNot(par.gradient_full, pcopy.gradient_full)
         with tempfile.TemporaryFile('w+b') as f:
             par.pickle(f)
             f.seek(0)
             pcopy = pickle.load(f)
         self.assertListEqual(par.param_array.tolist(), pcopy.param_array.tolist())
         pcopy.gradient = 10
-        np.testing.assert_allclose(par.linear.full_gradient, pcopy.linear.full_gradient)
+        np.testing.assert_allclose(par.linear.gradient_full, pcopy.linear.gradient_full)
-        np.testing.assert_allclose(pcopy.linear.full_gradient, 10)
+        np.testing.assert_allclose(pcopy.linear.gradient_full, 10)
         self.assertSequenceEqual(str(par), str(pcopy))
 
     def test_model(self):
         par = toy_rbf_1d_50(optimize=0, plot=0)
         pcopy = par.copy()
         self.assertListEqual(par.param_array.tolist(), pcopy.param_array.tolist())
-        self.assertListEqual(par.full_gradient.tolist(), pcopy.full_gradient.tolist())
+        self.assertListEqual(par.gradient_full.tolist(), pcopy.gradient_full.tolist())
         self.assertSequenceEqual(str(par), str(pcopy))
         self.assertIsNot(par.param_array, pcopy.param_array)
-        self.assertIsNot(par.full_gradient, pcopy.full_gradient)
+        self.assertIsNot(par.gradient_full, pcopy.gradient_full)
         self.assertTrue(pcopy.checkgrad())
         self.assert_(np.any(pcopy.gradient!=0.0))
         with tempfile.TemporaryFile('w+b') as f:
@@ -118,26 +119,29 @@ class Test(ListDictTestCase):
             f.seek(0)
             pcopy = pickle.load(f)
         self.assertListEqual(par.param_array.tolist(), pcopy.param_array.tolist())
-        np.testing.assert_allclose(par.full_gradient, pcopy.full_gradient)
+        np.testing.assert_allclose(par.gradient_full, pcopy.gradient_full)
         self.assertSequenceEqual(str(par), str(pcopy))
         self.assert_(pcopy.checkgrad())
 
     def test_modelrecreation(self):
         par = toy_rbf_1d_50(optimize=0, plot=0)
         pcopy = GPRegression(par.X.copy(), par.Y.copy(), kernel=par.kern.copy())
-        self.assertListEqual(par.param_array.tolist(), pcopy.param_array.tolist())
+        np.testing.assert_allclose(par.param_array, pcopy.param_array)
-        self.assertListEqual(par.full_gradient.tolist(), pcopy.full_gradient.tolist())
+        np.testing.assert_allclose(par.gradient_full, pcopy.gradient_full)
         self.assertSequenceEqual(str(par), str(pcopy))
         self.assertIsNot(par.param_array, pcopy.param_array)
-        self.assertIsNot(par.full_gradient, pcopy.full_gradient)
+        self.assertIsNot(par.gradient_full, pcopy.gradient_full)
         self.assertTrue(pcopy.checkgrad())
         self.assert_(np.any(pcopy.gradient!=0.0))
+        pcopy.optimize('bfgs')
+        par.optimize('bfgs')
+        np.testing.assert_allclose(pcopy.param_array, par.param_array, atol=.001)
         with tempfile.TemporaryFile('w+b') as f:
             par.pickle(f)
             f.seek(0)
             pcopy = pickle.load(f)
-        self.assertListEqual(par.param_array.tolist(), pcopy.param_array.tolist())
+        np.testing.assert_allclose(par.param_array, pcopy.param_array)
-        np.testing.assert_allclose(par.full_gradient, pcopy.full_gradient)
+        np.testing.assert_allclose(par.gradient_full, pcopy.gradient_full)
         self.assertSequenceEqual(str(par), str(pcopy))
         self.assert_(pcopy.checkgrad())
 
@@ -147,19 +151,20 @@ class Test(ListDictTestCase):
         par = NormalPosterior(X,Xv)
         par.gradient = 10
         pcopy = par.copy()
+        pcopy.gradient = 10
         self.assertListEqual(par.param_array.tolist(), pcopy.param_array.tolist())
-        self.assertListEqual(par.full_gradient.tolist(), pcopy.full_gradient.tolist())
+        self.assertListEqual(par.gradient_full.tolist(), pcopy.gradient_full.tolist())
         self.assertSequenceEqual(str(par), str(pcopy))
         self.assertIsNot(par.param_array, pcopy.param_array)
-        self.assertIsNot(par.full_gradient, pcopy.full_gradient)
+        self.assertIsNot(par.gradient_full, pcopy.gradient_full)
         with tempfile.TemporaryFile('w+b') as f:
             par.pickle(f)
             f.seek(0)
             pcopy = pickle.load(f)
         self.assertListEqual(par.param_array.tolist(), pcopy.param_array.tolist())
         pcopy.gradient = 10
-        np.testing.assert_allclose(par.full_gradient, pcopy.full_gradient)
+        np.testing.assert_allclose(par.gradient_full, pcopy.gradient_full)
-        np.testing.assert_allclose(pcopy.mean.full_gradient, 10)
+        np.testing.assert_allclose(pcopy.mean.gradient_full, 10)
         self.assertSequenceEqual(str(par), str(pcopy))
 
     def test_model_concat(self):
@@ -167,10 +172,11 @@ class Test(ListDictTestCase):
         par.randomize()
         pcopy = par.copy()
         self.assertListEqual(par.param_array.tolist(), pcopy.param_array.tolist())
-        self.assertListEqual(par.full_gradient.tolist(), pcopy.full_gradient.tolist())
+        self.assertListEqual(par.gradient_full.tolist(), pcopy.gradient_full.tolist())
         self.assertSequenceEqual(str(par), str(pcopy))
         self.assertIsNot(par.param_array, pcopy.param_array)
-        self.assertIsNot(par.full_gradient, pcopy.full_gradient)
+        self.assertIsNot(par.gradient_full, pcopy.gradient_full)
+        self.assertTrue(par.checkgrad())
         self.assertTrue(pcopy.checkgrad())
         self.assert_(np.any(pcopy.gradient!=0.0))
         with tempfile.TemporaryFile('w+b') as f:
@@ -178,7 +184,7 @@ class Test(ListDictTestCase):
             f.seek(0)
             pcopy = pickle.load(f)
         self.assertListEqual(par.param_array.tolist(), pcopy.param_array.tolist())
-        np.testing.assert_allclose(par.full_gradient, pcopy.full_gradient)
+        np.testing.assert_allclose(par.gradient_full, pcopy.gradient_full)
         self.assertSequenceEqual(str(par), str(pcopy))
         self.assert_(pcopy.checkgrad())
 

GPy/util/__init__.py

@@ -16,6 +16,7 @@ import diag
 import initialization
 import multioutput
 import linalg_gpu
+import mpi
 
 try:
     import sympy

GPy/util/caching.py

@@ -1,84 +1,124 @@
 from ..core.parameterization.parameter_core import Observable
-import itertools
+import collections, weakref, logging
 
 class Cacher(object):
-    """
-
-
-
-
-    """
-
     def __init__(self, operation, limit=5, ignore_args=(), force_kwargs=()):
+        """
+        Parameters:
+        ***********
+        :param callable operation: function to cache
+        :param int limit: depth of cacher
+        :param [int] ignore_args: list of indices, pointing at arguments to ignore in *args of operation(*args). This includes self!
+        :param [str] force_kwargs: list of kwarg names (strings). If a kwarg with that name is given, the cacher will force recompute and wont cache anything.
+        :param int verbose: verbosity level. 0: no print outs, 1: casual print outs, 2: debug level print outs
+        """
         self.limit = int(limit)
         self.ignore_args = ignore_args
         self.force_kwargs = force_kwargs
-        self.operation=operation
+        self.operation = operation
-        self.cached_inputs = []
+        self.order = collections.deque()
-        self.cached_outputs = []
+        self.cached_inputs = {}  # point from cache_ids to a list of [ind_ids], which where used in cache cache_id
-        self.inputs_changed = []
+
+        #=======================================================================
+        # point from each ind_id to [ref(obj), cache_ids]
+        # 0: a weak reference to the object itself
+        # 1: the cache_ids in which this ind_id is used (len will be how many times we have seen this ind_id)
+        self.cached_input_ids = {}
+        #=======================================================================
+
+        self.cached_outputs = {}  # point from cache_ids to outputs
+        self.inputs_changed = {}  # point from cache_ids to bools
+
+    def id(self, obj):
+        """returns the self.id of an object, to be used in caching individual self.ids"""
+        return hex(id(obj))
+
+    def combine_inputs(self, args, kw, ignore_args):
+        "Combines the args and kw in a unique way, such that ordering of kwargs does not lead to recompute"
+        inputs= args + tuple(c[1] for c in sorted(kw.items(), key=lambda x: x[0]))
+        # REMOVE the ignored arguments from input and PREVENT it from being checked!!!
+        return [a for i,a in enumerate(inputs) if i not in ignore_args]
+
+    def prepare_cache_id(self, combined_args_kw):
+        "get the cacheid (conc. string of argument self.ids in order)"
+        cache_id = "".join(self.id(a) for a in combined_args_kw)
+        return cache_id
+
+    def ensure_cache_length(self, cache_id):
+        "Ensures the cache is within its limits and has one place free"
+        if len(self.order) == self.limit:
+            # we have reached the limit, so lets release one element
+            cache_id = self.order.popleft()
+            combined_args_kw = self.cached_inputs[cache_id]
+            for ind in combined_args_kw:
+                if ind is not None:
+                    ind_id = self.id(ind)
+                    tmp = self.cached_input_ids.get(ind_id, None)
+                    if tmp is not None:
+                        ref, cache_ids = tmp
+                        if len(cache_ids) == 1 and ref() is not None:
+                            ref().remove_observer(self, self.on_cache_changed)
+                            del self.cached_input_ids[ind_id]
+                        else:
+                            cache_ids.remove(cache_id)
+                            self.cached_input_ids[ind_id] = [ref, cache_ids]
+            del self.cached_outputs[cache_id]
+            del self.inputs_changed[cache_id]
+            del self.cached_inputs[cache_id]
+
+    def add_to_cache(self, cache_id, inputs, output):
+        """This adds cache_id to the cache, with inputs and output"""
+        self.inputs_changed[cache_id] = False
+        self.cached_outputs[cache_id] = output
+        self.order.append(cache_id)
+        self.cached_inputs[cache_id] = inputs
+        for a in inputs:
+            if a is not None:
+                ind_id = self.id(a)
+                v = self.cached_input_ids.get(ind_id, [weakref.ref(a), []])
+                v[1].append(cache_id)
+                if len(v[1]) == 1:
+                    a.add_observer(self, self.on_cache_changed)
+                self.cached_input_ids[ind_id] = v
 
     def __call__(self, *args, **kw):
         """
         A wrapper function for self.operation,
         """
+        #=======================================================================
+        # !WARNING CACHE OFFSWITCH!
+        # return self.operation(*args, **kw)
+        #=======================================================================
 
-        #ensure that specified arguments are ignored
+        # 1: Check whether we have forced recompute arguments:
-        items = sorted(kw.items(), key=lambda x: x[0])
-        oa_all = args + tuple(a for _,a in items)
-        if len(self.ignore_args) != 0:
-            oa = [a for i,a in itertools.chain(enumerate(args), items) if i not in self.ignore_args and i not in self.force_kwargs]
-        else:
-            oa = oa_all
-
-        # this makes sure we only add an observer once, and that None can be in args
-        observable_args = []
-        for a in oa:
-            if (not any(a is ai for ai in observable_args)) and a is not None:
-                observable_args.append(a)
-
-        #make sure that all the found argument really are observable:
-        #otherswise don't cache anything, pass args straight though
-        if not all([isinstance(arg, Observable) for arg in observable_args]):
-            return self.operation(*args, **kw)
-
         if len(self.force_kwargs) != 0:
-            # check if there are force args, which force reloading
             for k in self.force_kwargs:
                 if k in kw and kw[k] is not None:
                     return self.operation(*args, **kw)
-        # TODO: WARNING !!! Cache OFFSWITCH !!! WARNING
-        # return self.operation(*args, **kw)
 
-        #if the result is cached, return the cached computation
+        # 2: prepare_cache_id and get the unique self.id string for this call
-        state = [all(a is b for a, b in itertools.izip_longest(args, cached_i)) for cached_i in self.cached_inputs]
+        inputs = self.combine_inputs(args, kw, self.ignore_args)
+        cache_id = self.prepare_cache_id(inputs)
+        # 2: if anything is not cachable, we will just return the operation, without caching
+        if reduce(lambda a, b: a or (not (isinstance(b, Observable) or b is None)), inputs, False):
+            #print 'WARNING: '+self.operation.__name__ + ' not cacheable!'
+            #print [not (isinstance(b, Observable)) for b in inputs]
+            return self.operation(*args, **kw)
+        # 3&4: check whether this cache_id has been cached, then has it changed?
         try:
-            if any(state):
+            if(self.inputs_changed[cache_id]):
-                i = state.index(True)
+                # 4: This happens, when elements have changed for this cache self.id
-                if self.inputs_changed[i]:
+                self.inputs_changed[cache_id] = False
-                    #(elements of) the args have changed since we last computed: update
+                self.cached_outputs[cache_id] = self.operation(*args, **kw)
-                    self.cached_outputs[i] = self.operation(*args, **kw)
+        except KeyError:
-                    self.inputs_changed[i] = False
+            # 3: This is when we never saw this chache_id:
-                return self.cached_outputs[i]
+            self.ensure_cache_length(cache_id)
-            else:
+            self.add_to_cache(cache_id, inputs, self.operation(*args, **kw))
-                #first time we've seen these arguments: compute
-
-                #first make sure the depth limit isn't exceeded
-                if len(self.cached_inputs) == self.limit:
-                    args_ = self.cached_inputs.pop(0)
-                    args_ = [a for i,a in enumerate(args_) if i not in self.ignore_args and i not in self.force_kwargs]
-                    [a.remove_observer(self, self.on_cache_changed) for a in args_ if a is not None]
-                    self.inputs_changed.pop(0)
-                    self.cached_outputs.pop(0)
-                #compute
-                self.cached_inputs.append(oa_all)
-                self.cached_outputs.append(self.operation(*args, **kw))
-                self.inputs_changed.append(False)
-                [a.add_observer(self, self.on_cache_changed) for a in observable_args]
-                return self.cached_outputs[-1]#return
         except:
             self.reset()
             raise
+        # 5: We have seen this cache_id and it is cached:
+        return self.cached_outputs[cache_id]
 
     def on_cache_changed(self, direct, which=None):
         """
@@ -86,17 +126,21 @@ class Cacher(object):
 
         this function gets 'hooked up' to the inputs when we cache them, and upon their elements being changed we update here.
         """
-        self.inputs_changed = [any([a is direct or a is which for a in args]) or old_ic for args, old_ic in zip(self.cached_inputs, self.inputs_changed)]
+        for what in [direct, which]:
+            if what is not None:
+                ind_id = self.id(what)
+                _, cache_ids = self.cached_input_ids.get(ind_id, [None, []])
+                for cache_id in cache_ids:
+                    self.inputs_changed[cache_id] = True
 
     def reset(self):
         """
         Totally reset the cache
         """
-        [[a.remove_observer(self, self.on_cache_changed) for a in args if isinstance(a, Observable)] for args in self.cached_inputs]
+        [a().remove_observer(self, self.on_cache_changed) if (a() is not None) else None for [a, _] in self.cached_input_ids.values()]
-        [[a.remove_observer(self, self.reset) for a in args if isinstance(a, Observable)] for args in self.cached_inputs]
+        self.cached_input_ids = {}
-        self.cached_inputs = []
+        self.cached_outputs = {}
-        self.cached_outputs = []
+        self.inputs_changed = {}
-        self.inputs_changed = []
 
     def __deepcopy__(self, memo=None):
         return Cacher(self.operation, self.limit, self.ignore_args, self.force_kwargs)
@@ -124,7 +168,7 @@ class Cacher_wrap(object):
         return partial(self, obj)
     def __call__(self, *args, **kwargs):
         obj = args[0]
-        #import ipdb;ipdb.set_trace()
+        # import ipdb;ipdb.set_trace()
         try:
             caches = obj.__cachers
         except AttributeError:

GPy/util/config.py

@@ -5,18 +5,19 @@ import ConfigParser
 import os
 config = ConfigParser.ConfigParser()
 
-home = os.getenv('HOME') or os.getenv('USERPROFILE')
+# This is the default configuration file that always needs to be present.
-user_file = os.path.join(home,'.gpy_config.cfg')
+default_file = os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..', 'defaults.cfg'))
-default_file = os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..', 'gpy_config.cfg'))
-# print user_file, os.path.isfile(user_file)
-# print default_file, os.path.isfile(default_file)
 
-# 1. check if the user has a ~/.gpy_config.cfg
+# These files are optional
-if os.path.isfile(user_file):
+# This specifies configurations that are typically specific to the machine (it is found alongside the GPy installation).
-    config.read(user_file)
+local_file = os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..', 'installation.cfg'))
-elif os.path.isfile(default_file):
+
-    # 2. if not, use the default one
+# This specifies configurations specific to the user (it is found in the user home directory)
-    config.read(default_file)
+home = os.getenv('HOME') or os.getenv('USERPROFILE')
-else:
+user_file = os.path.join(home,'.gpy_user.cfg')
-    #3. panic
+
-    raise ValueError, "no configuration file found"
+# Read in the given files.
+config.readfp(open(default_file))
+config.read([local_file, user_file])
+if not config:
+    raise ValueError, "No configuration file found at either " + user_file + " or " + local_file + " or " + default_file + "."

GPy/util/data_resources.json

@@ -1,66 +1,397 @@
 {
-   "rogers_girolami_data":{
+    "ankur_pose_data": {
-      "files":[
+        "citation": "3D Human Pose from Silhouettes by Relevance Vector Regression (In CVPR'04). A. Agarwal and B. Triggs.",
-         [
+        "details": "Artificially generated data of silhouettes given poses. Note that the data does not display a left/right ambiguity because across the entire data set one of the arms sticks out more the the other, disambiguating the pose as to which way the individual is facing.",
-            "firstcoursemldata.tar.gz"
+        "files": [
-         ]
-      ],
-      "license":null,
-      "citation":"A First Course in Machine Learning. Simon Rogers and Mark Girolami: Chapman & Hall/CRC, ISBN-13: 978-1439824146",
-      "details":"Data from the textbook 'A First Course in Machine Learning'. Available from http://www.dcs.gla.ac.uk/~srogers/firstcourseml/.",
-      "urls":[
-         "https://www.dropbox.com/sh/7p6tu1t29idgliq/_XqlH_3nt9/"
-      ],
-      "suffices":[
-         [
-            "?dl=1"
-         ]
-      ],
-      "size":21949154
-   },
-   "ankur_pose_data":{
-      "files":[
             [
                 "ankurDataPoseSilhouette.mat"
             ]
         ],
-      "citation":"3D Human Pose from Silhouettes by Relevance Vector Regression (In CVPR'04). A. Agarwal and B. Triggs.",
+        "license": null,
-      "license":null,
+        "size": 1,
-      "urls":[
+        "urls": [
             "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/ankur_pose_data/"
-      ],
+        ]
-      "details":"Artificially generated data of silhouettes given poses. Note that the data does not display a left/right ambiguity because across the entire data set one of the arms sticks out more the the other, disambiguating the pose as to which way the individual is facing.",
-      "size":1
     },
-   "football_data":{
+    "boston_housing": {
-      "files":[
+        "citation": "Harrison, D. and Rubinfeld, D.L. 'Hedonic prices and the demand for clean air', J. Environ. Economics & Management, vol.5, 81-102, 1978.",
+        "details": "The Boston Housing data relates house values in Boston to a range of input variables.",
+        "files": [
             [
-	     "E0.csv", "E1.csv", "E2.csv", "E3.csv"
+                "Index",
+                "housing.data",
+                "housing.names"
             ]
         ],
-      "citation":"",
+        "license": null,
-      "license":null,
+        "size": 51276,
-      "urls":[
+        "urls": [
+            "http://archive.ics.uci.edu/ml/machine-learning-databases/housing/"
+        ]
+    },
+    "boxjenkins_airline": {
+        "citation": "Box & Jenkins (1976), in file: data/airpass, Description: International airline passengers: monthly totals in thousands. Jan 49 \\u2013 Dec 60",
+        "details": "International airline passengers, monthly totals from January 1949 to December 1960.",
+        "files": [
+            [
+                "boxjenkins_airline.csv"
+            ]
+        ],
+        "license": "You may copy and redistribute the data. You may make derivative works from the data. You may use the data for commercial purposes. You may not sublicence the data when redistributing it. You may not redistribute the data under a different license. Source attribution on any use of this data: Must refer source.",
+        "size": 46779,
+        "urls": [
+            "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/boxjenkins_airline/"
+        ]
+    },
+    "brendan_faces": {
+        "citation": "Frey, B. J., Colmenarez, A and Huang, T. S. Mixtures of Local Linear Subspaces for Face Recognition. Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition 1998, 32-37, June 1998. Computer Society Press, Los Alamitos, CA.",
+        "details": "A video of Brendan Frey's face popularized as a benchmark for visualization by the Locally Linear Embedding.",
+        "files": [
+            [
+                "frey_rawface.mat"
+            ]
+        ],
+        "license": null,
+        "size": 1100584,
+        "urls": [
+            "http://www.cs.nyu.edu/~roweis/data/"
+        ]
+    },
+    "cifar-10": {
+        "citation": "Learning Multiple Layers of Features from Tiny Images, Alex Krizhevsky, 2009, Tech report available here: http://www.cs.toronto.edu/~kriz/learning-features-2009-TR.pdf",
+        "details": "The CIFAR-10 and CIFAR-100 are labeled subsets of the 80 million tiny images dataset. They were collected by Alex Krizhevsky, Vinod Nair, and Geoffrey Hinton. Details are available on this webpage: http://www.cs.toronto.edu/~kriz/cifar.html. The CIFAR-10 dataset consists of 60000 32x32 colour images in 10 classes, with 6000 images per class. There are 50000 training images and 10000 test images.",
+        "files": [
+            [
+                "cifar-10-python.tar.gz"
+            ]
+        ],
+        "license": null,
+        "size": 0,
+        "urls": [
+            "http://www.cs.toronto.edu/~kriz/"
+        ]
+    },
+    "cmu_mocap_full": {
+        "citation": "Please include this in your acknowledgements: The data used in this project was obtained from mocap.cs.cmu.edu.\\nThe database was created with funding from NSF EIA-0196217.",
+        "details": "CMU Motion Capture data base. Captured by a Vicon motion capture system consisting of 12 infrared MX-40 cameras, each of which is capable of recording at 120 Hz with images of 4 megapixel resolution. Motions are captured in a working volume of approximately 3m x 8m. The capture subject wears 41 markers and a stylish black garment.",
+        "files": [
+            [
+                "allasfamc.zip"
+            ]
+        ],
+        "license": "From http://mocap.cs.cmu.edu. This data is free for use in research projects. You may include this data in commercially-sold products, but you may not resell this data directly, even in converted form. If you publish results obtained using this data, we would appreciate it if you would send the citation to your published paper to jkh+mocap@cs.cmu.edu, and also would add this text to your acknowledgments section: The data used in this project was obtained from mocap.cs.cmu.edu. The database was created with funding from NSF EIA-0196217.",
+        "size": null,
+        "urls": [
+            "http://mocap.cs.cmu.edu/subjects"
+        ]
+    },
+    "creep_rupture": {
+        "citation": "Materials Algorithms Project Data Library: MAP_DATA_CREEP_RUPTURE. F. Brun and T. Yoshida.",
+        "details": "Provides 2066 creep rupture test results of steels (mainly of two kinds of steels: 2.25Cr and 9-12 wt% Cr ferritic steels). See http://www.msm.cam.ac.uk/map/data/materials/creeprupt-b.html.",
+        "files": [
+            [
+                "creeprupt.tar"
+            ]
+        ],
+        "license": null,
+        "size": 602797,
+        "urls": [
+            "http://www.msm.cam.ac.uk/map/data/tar/"
+        ]
+    },
+    "decampos_characters": {
+        "citation": "T. de Campos, B. R. Babu, and M. Varma. Character recognition in natural images. VISAPP 2009.",
+        "details": "Examples of hand written digits taken from the de Campos et al paper on Character Recognition in Natural Images.",
+        "files": [
+            [
+                "characters.npy",
+                "digits.npy"
+            ]
+        ],
+        "license": null,
+        "size": 2031872,
+        "urls": [
+            "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/decampos_digits/"
+        ]
+    },
+    "della_gatta": {
+        "citation": "Direct targets of the TRP63 transcription factor revealed by a combination of gene expression profiling and reverse engineering. Giusy Della Gatta, Mukesh Bansal, Alberto Ambesi-Impiombato, Dario Antonini, Caterina Missero, and Diego di Bernardo, Genome Research 2008",
+        "details": "The full gene expression data set from della Gatta et al (http://www.ncbi.nlm.nih.gov/pmc/articles/PMC2413161/) processed by RMA.",
+        "files": [
+            [
+                "DellaGattadata.mat"
+            ]
+        ],
+        "license": null,
+        "size": 3729650,
+        "urls": [
+            "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/della_gatta/"
+        ]
+    },
+    "drosophila_protein": {
+        "citation": "Becker K, Balsa-Canto E, Cicin-Sain D, Hoermann A, Janssens H, et al. (2013) Reverse-Engineering Post-Transcriptional Regulation of Gap Genes in Drosophila melanogaster. PLoS Comput Biol 9(10): e1003281. doi:10.1371/journal.pcbi.1003281",
+        "details": "Expression of the gap genes Krüppel, knirps, and giant in Drosophila melanogaster. Data includes quantitative datasets of gap gene mRNA and protein expression to solve and fit a model of post-transcriptional regulation, and establish its structural and practical identifiability",
+        "files": [
+            [
+                "becker_et_al.csv"
+            ]
+        ],
+        "license": null,
+        "size": 20258,
+        "urls": [
+            "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/drosophila_protein/"
+        ]
+    },
+    "spellman_yeast": {
+        "citation": "Paul T. Spellman, Gavin Sherlock, Michael Q. Zhang, Vishwanath R. Iyer, Kirk Anders, Michael B. Eisen, Patrick O. Brown, David Botstein, and Bruce Futcher 'Comprehensive Identification of Cell Cycle-regulated Genes of the Yeast Saccharomyces cerevisiae by Microarray Hybridization.'  Molecular Biology of the Cell 9, 3273-3297",
+        "details": "Two colour spotted cDNA array data set of a series of experiments to identify which genes in Yeast are cell cycle regulated.",
+        "files": [
+            [
+                "combined.txt"
+            ]
+        ],
+        "license": null,
+        "size": 2510955,
+        "urls": [
+            "http://genome-www.stanford.edu/cellcycle/data/rawdata/"
+        ]
+    },
+    "lee_yeast_ChIP": {
+        "citation": "Tong Ihn Lee, Nicola J. Rinaldi, Francois Robert, Duncan T. Odom, Ziv Bar-Joseph, Georg K. Gerber, Nancy M. Hannett, Christopher T. Harbison, Craig M. Thompson, Itamar Simon, Julia Zeitlinger, Ezra G. Jennings, Heather L. Murray, D. Benjamin Gordon, Bing Ren, John J. Wyrick, Jean-Bosco Tagne, Thomas L. Volkert, Ernest Fraenkel, David K. Gifford, Richard A. Young 'Transcriptional Regulatory Networks in Saccharomyces cerevisiae' Science 298 (5594) pg 799--804. DOI: 10.1126/science.1075090",
+        "details": "Binding location analysis for 106 regulators in yeast. The data consists of p-values for binding of regulators to genes derived from ChIP-chip experiments.",
+        "files": [
+            [
+                "binding_by_gene.tsv"
+            ]
+        ],
+        "license": null,
+        "size": 1674161,
+        "urls": [
+            "http://jura.wi.mit.edu/young_public/regulatory_network/"
+        ]
+    },
+    "epomeo_gpx": {
+        "citation": "",
+        "details": "Five different GPS traces of the same run up Mount Epomeo in Ischia. The traces are from different sources. endomondo_1 and endomondo_2 are traces from the mobile phone app Endomondo, with a split in the middle. garmin_watch_via_endomondo is the trace from a Garmin watch, with a segment missing about 4 kilometers in. viewranger_phone and viewranger_tablet are traces from a phone and a tablet through the viewranger app. The viewranger_phone data comes from the same mobile phone as the Endomondo data (i.e. there are 3 GPS devices, but one device recorded two traces).",
+        "files": [
+            [
+                "endomondo_1.gpx",
+                "endomondo_2.gpx",
+                "garmin_watch_via_endomondo.gpx",
+                "viewranger_phone.gpx",
+                "viewranger_tablet.gpx"
+            ]
+        ],
+        "license": null,
+        "size": 2031872,
+        "urls": [
+            "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/epomeo_gpx/"
+        ]
+    },
+    "football_data": {
+        "citation": "",
+        "details": "Results of English football matches since 1993/94 season.",
+        "files": [
+            [
+                "E0.csv",
+                "E1.csv",
+                "E2.csv",
+                "E3.csv"
+            ]
+        ],
+        "license": null,
+        "size": 1,
+        "urls": [
             "http://www.football-data.co.uk/mmz4281/"
-      ],
+        ]
-      "details":"Results of English football matches since 1993/94 season.",
-      "size":1
     },
-   "google_trends":{
+    "fruitfly_tomancak": {
-      "files":[
+        "citation": "",
+        "details": "",
+        "files": [
             [
+                "tomancak_exprs.csv",
+                "tomancak_se.csv",
+                "tomancak_prctile5.csv",
+                "tomancak_prctile25.csv",
+                "tomancak_prctile50.csv",
+                "tomancak_prctile75.csv",
+                "tomancak_prctile95.csv"
             ]
         ],
-      "citation":"",
+        "license": null,
-      "license":null,
+        "size": 59000000,
-      "urls":[
+        "urls": [
-          "http://www.google.com/trends/"
+            "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/fruitfly_tomancak/"
-      ],
+        ]
-      "details":"Google trends results.",
-      "size":0
     },
-   "osu_accad":{
+    "fruitfly_tomancak_cel_files": {
-      "files":[
+        "citation": "'Systematic determination of patterns of gene expression during Drosophila embryogenesis' Pavel Tomancak, Amy Beaton, Richard Weiszmann, Elaine Kwan, ShengQiang Shu, Suzanna E Lewis, Stephen Richards, Michael Ashburner, Volker Hartenstein, Susan E Celniker, and Gerald M Rubin",
+        "details": "Gene expression results from blastoderm development in Drosophila Melanogaster.",
+        "files": [ 
+            [
+                "embryo_tc_4_1.CEL",
+                "embryo_tc_4_2.CEL",
+                "embryo_tc_4_3.CEL",
+                "embryo_tc_4_4.CEL",
+                "embryo_tc_4_5.CEL",
+                "embryo_tc_4_6.CEL",
+                "embryo_tc_4_7.CEL",
+                "embryo_tc_4_8.CEL",
+                "embryo_tc_4_9.CEL",
+                "embryo_tc_4_10.CEL",
+                "embryo_tc_4_11.CEL",
+                "embryo_tc_4_12.CEL",
+                "embryo_tc_6_1.CEL",
+                "embryo_tc_6_2.CEL",
+                "embryo_tc_6_3.CEL",
+                "embryo_tc_6_4.CEL",
+                "embryo_tc_6_5.CEL",
+                "embryo_tc_6_6.CEL",
+                "embryo_tc_6_7.CEL",
+                "embryo_tc_6_8.CEL",
+                "embryo_tc_6_9.CEL",
+                "embryo_tc_6_10.CEL",
+                "embryo_tc_6_11.CEL",
+                "embryo_tc_6_12.CEL",
+                "embryo_tc_8_1.CEL",
+                "embryo_tc_8_2.CEL",
+                "embryo_tc_8_3.CEL",
+                "embryo_tc_8_4.CEL",
+                "embryo_tc_8_5.CEL",
+                "embryo_tc_8_6.CEL",
+                "embryo_tc_8_7.CEL",
+                "embryo_tc_8_8.CEL",
+                "embryo_tc_8_9.CEL",
+                "embryo_tc_8_10.CEL",
+                "embryo_tc_8_11.CEL",
+                "embryo_tc_8_12.CEL",
+                "CG_AffyOligo_Gadfly3_01_13_03",
+                "embryo_tc_rma_release2.txt",
+                "embryo_tc_rma_release3.txt",
+                "na_affy_oligo.dros",
+                "README.TXT"
+            ]
+        ],
+        "license": null,
+        "size": 389000000,
+        "urls": [
+            "ftp://ftp.fruitfly.org/pub/embryo_tc_array_data/"
+        ]
+    },
+    "google_trends": {
+        "citation": "",
+        "details": "Google trends results.",
+        "files": [
+            [
+                
+            ]
+        ],
+        "license": null,
+        "size": 0,
+        "urls": [
+            "http://www.google.com/trends/"
+        ]
+    },
+    
+    "hapmap3": {
+        "citation": "Gibbs, Richard A., et al. 'The international HapMap project.' Nature 426.6968 (2003): 789-796.",
+        "details": "HapMap Project: Single Nucleotide Polymorphism sequenced in all human populations. \n        The HapMap phase three SNP dataset - 1184 samples out of 11 populations.\n        See http://www.nature.com/nature/journal/v426/n6968/abs/nature02168.html for details.\n\n        SNP_matrix (A) encoding [see Paschou et all. 2007 (PCA-Correlated SNPs...)]:\n        Let (B1,B2) be the alphabetically sorted bases, which occur in the j-th SNP, then\n\n              /  1, iff SNPij==(B1,B1)\n        Aij = |  0, iff SNPij==(B1,B2)\n              \\\\ -1, iff SNPij==(B2,B2)\n\n        The SNP data and the meta information (such as iid, sex and phenotype) are\n        stored in the dataframe datadf, index is the Individual ID, \n        with following columns for metainfo:\n\n            * family_id   -> Family ID\n            * paternal_id -> Paternal ID\n            * maternal_id -> Maternal ID\n            * sex         -> Sex (1=male; 2=female; other=unknown)\n            * phenotype   -> Phenotype (-9, or 0 for unknown)\n            * population  -> Population string (e.g. 'ASW' - 'YRI')\n            * rest are SNP rs (ids)\n\n        More information is given in infodf:\n\n            * Chromosome:\n                - autosomal chromosemes                -> 1-22\n                - X    X chromosome                    -> 23\n                - Y    Y chromosome                    -> 24\n                - XY   Pseudo-autosomal region of X    -> 25\n                - MT   Mitochondrial                   -> 26\n            * Relative Positon (to Chromosome) [base pairs]\n\n        ",
+        "files": [
+            [
+                "hapmap3_r2_b36_fwd.consensus.qc.poly.map.bz2",
+                "hapmap3_r2_b36_fwd.consensus.qc.poly.ped.bz2",
+                "relationships_w_pops_121708.txt"
+            ]
+        ],
+        "license": "International HapMap Project Public Access License (http://hapmap.ncbi.nlm.nih.gov/cgi-perl/registration#licence)",
+        "size": 3458246739,
+        "urls": [
+            "http://hapmap.ncbi.nlm.nih.gov/downloads/genotypes/latest_phaseIII_ncbi_b36/plink_format/"
+        ]
+    },
+    "isomap_face_data": {
+        "citation": "A Global Geometric Framework for Nonlinear Dimensionality Reduction, J. B. Tenenbaum, V. de Silva and J. C. Langford, Science 290 (5500): 2319-2323, 22 December 2000",
+        "details": "Face data made available by Tenenbaum, de Silva and Langford to demonstrate isomap, available from http://isomap.stanford.edu/datasets.html.",
+        "files": [
+            [
+                "face_data.mat"
+            ]
+        ],
+        "license": null,
+        "size": 24229368,
+        "urls": [
+            "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/isomap_face_data/"
+        ]
+    },
+    "mauna_loa": {
+        "citation": "Mauna Loa Data. Dr. Pieter Tans, NOAA/ESRL (www.esrl.noaa.gov/gmd/ccgg/trends/) and Dr. Ralph Keeling, Scripps Institution of Oceanography (scrippsco2.ucsd.edu/).",
+        "details": "The 'average' column contains the monthly mean CO2 mole fraction determined from daily averages.  The mole fraction of CO2, expressed as parts per million (ppm) is the number of molecules of CO2 in every one million molecules of dried air (water vapor removed).  If there are missing days concentrated either early or late in the month, the monthly mean is corrected to the middle of the month using the average seasonal cycle.  Missing months are denoted by -99.99. The 'interpolated' column includes average values from the preceding column and interpolated values where data are missing.  Interpolated values are computed in two steps.  First, we compute for each month the average seasonal cycle in a 7-year window around each monthly value.  In this way the seasonal cycle is allowed to change slowly over time.  We then determine the 'trend' value for each month by removing the seasonal cycle; this result is shown in the 'trend' column.  Trend values are linearly interpolated for missing months. The interpolated monthly mean is then the sum of the average seasonal cycle value and the trend value for the missing month.\n\nNOTE: In general, the data presented for the last year are subject to change, depending on recalibration of the reference gas mixtures used, and other quality control procedures. Occasionally, earlier years may also be changed for the same reasons.  Usually these changes are minor.\n\nCO2 expressed as a mole fraction in dry air, micromol/mol, abbreviated as ppm \n\n (-99.99 missing data;  -1 no data for daily means in month)",
+        "files": [
+            [
+                "co2_mm_mlo.txt"
+            ]
+        ],
+        "license": "-------------------------------------------------------------------- USE OF NOAA ESRL DATA\n\n  These data are made freely available to the public and the scientific community in the belief that their wide dissemination will lead to greater understanding and new scientific insights. The availability of these data does not constitute publication of the data.  NOAA relies on the ethics and integrity of the user to insure that ESRL receives fair credit for their work.  If the data  are obtained for potential use in a publication or presentation,  ESRL should be informed at the outset of the nature of this work.   If the ESRL data are essential to the work, or if an important  result or conclusion depends on the ESRL data, co-authorship may be appropriate.  This should be discussed at an early stage in the work.  Manuscripts using the ESRL data should be sent to ESRL for review before they are submitted for publication so we can insure that the quality and limitations of the data are accurately represented.\n\n  Contact:   Pieter Tans (303 497 6678; pieter.tans@noaa.gov)\n\n  RECIPROCITY  Use of these data implies an agreement to reciprocate. Laboratories making similar measurements agree to make their own data available to the general public and to the scientific community in an equally complete and easily accessible form. Modelers are encouraged to make available to the community, upon request, their own tools used in the interpretation of the ESRL data, namely well documented model code, transport fields, and additional information necessary for other scientists to repeat the work and to run modified versions. Model availability includes collaborative support for new users of the models.\n --------------------------------------------------------------------\n\n     See www.esrl.noaa.gov/gmd/ccgg/trends/ for additional details.",
+        "size": 46779,
+        "urls": [
+            "ftp://aftp.cmdl.noaa.gov/products/trends/co2/"
+        ]
+    },
+    "olivetti_faces": {
+        "citation": "Ferdinando Samaria and Andy Harter, Parameterisation of a Stochastic Model for Human Face Identification. Proceedings of 2nd IEEE Workshop on Applications of Computer Vision, Sarasota FL, December 1994",
+        "details": "Olivetti Research Labs Face data base, acquired between December 1992 and December 1994 in the Olivetti Research Lab, Cambridge (which later became AT&T Laboratories, Cambridge). When using these images please give credit to AT&T Laboratories, Cambridge. ",
+        "files": [
+            [
+                "att_faces.zip"
+            ],
+            [
+                "olivettifaces.mat"
+            ]
+        ],
+        "license": null,
+        "size": 8561331,
+        "urls": [
+            "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/olivetti_faces/",
+            "http://www.cs.nyu.edu/~roweis/data/"
+        ]
+    },
+    "olivetti_glasses": {
+        "citation": "Information recorded in olivetti_faces entry. Should be used from there.",
+        "details": "Information recorded in olivetti_faces entry. Should be used from there.",
+        "files": [
+            [
+                "has_glasses.np"
+            ],
+            [
+                "olivettifaces.mat"
+            ]
+        ],
+        "license": null,
+        "size": 4261047,
+        "urls": [
+            "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/olivetti_faces/",
+            "http://www.cs.nyu.edu/~roweis/data/"
+        ]
+    },
+    "olympic_marathon_men": {
+        "citation": null,
+        "details": "Olympic mens' marathon gold medal winning times from 1896 to 2012. Time given in pace (minutes per kilometer). Data is originally downloaded and collated from Wikipedia, we are not responsible for errors in the data",
+        "files": [
+            [
+                "olympicMarathonTimes.csv"
+            ]
+        ],
+        "license": null,
+        "size": 584,
+        "urls": [
+            "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/olympic_marathon_men/"
+        ]
+    },
+    "osu_accad": {
+        "citation": "The Open Motion Data Project by The Ohio State University Advanced Computing Center for the Arts and Design, http://accad.osu.edu/research/mocap/mocap_data.htm.",
+        "details": "Motion capture data of different motions from the Open Motion Data Project at Ohio State University.",
+        "files": [
             [
                 "swagger1TXT.ZIP",
                 "handspring1TXT.ZIP",
@@ -81,103 +412,49 @@
                 "connections.txt"
             ]
         ],
-      "license":"Data is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License (http://creativecommons.org/licenses/by-nc-sa/3.0/).",
+        "license": "Data is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License (http://creativecommons.org/licenses/by-nc-sa/3.0/).",
-      "citation":"The Open Motion Data Project by The Ohio State University Advanced Computing Center for the Arts and Design, http://accad.osu.edu/research/mocap/mocap_data.htm.",
+        "size": 15922790,
-      "details":"Motion capture data of different motions from the Open Motion Data Project at Ohio State University.",
+        "urls": [
-      "urls":[
             "http://accad.osu.edu/research/mocap/data/",
             "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/stick/"
-      ],
+        ]
-      "size":15922790
     },
-   "isomap_face_data":{
+    "osu_run1": {
-      "files":[
+        "citation": "The Open Motion Data Project by The Ohio State University Advanced Computing Center for the Arts and Design, http://accad.osu.edu/research/mocap/mocap_data.htm.",
+        "details": "Motion capture data of a stick man running from the Open Motion Data Project at Ohio State University.",
+        "files": [
             [
-            "face_data.mat"
+                "run1TXT.ZIP"
+            ],
+            [
+                "connections.txt"
             ]
         ],
-      "license":null,
+        "license": "Data is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License (http://creativecommons.org/licenses/by-nc-sa/3.0/).",
-      "citation":"A Global Geometric Framework for Nonlinear Dimensionality Reduction, J. B. Tenenbaum, V. de Silva and J. C. Langford, Science 290 (5500): 2319-2323, 22 December 2000",
+        "size": 338103,
-      "details":"Face data made available by Tenenbaum, de Silva and Langford to demonstrate isomap, available from http://isomap.stanford.edu/datasets.html.",
+        "urls": [
-      "urls":[
+            "http://accad.osu.edu/research/mocap/data/",
-         "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/isomap_face_data/"
+            "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/stick/"
-      ],
-      "size":24229368
-   },
-   "boston_housing":{
-      "files":[
-         [
-            "Index",
-            "housing.data",
-            "housing.names"
         ]
-      ],
-      "license":null,
-      "citation":"Harrison, D. and Rubinfeld, D.L. 'Hedonic prices and the demand for clean air', J. Environ. Economics & Management, vol.5, 81-102, 1978.",
-      "details":"The Boston Housing data relates house values in Boston to a range of input variables.",
-      "urls":[
-         "http://archive.ics.uci.edu/ml/machine-learning-databases/housing/"
-      ],
-      "size":51276
     },
-   "cmu_mocap_full":{
+    "pumadyn-32nm": {
-      "files":[
+        "citation": "Created by Zoubin Ghahramani using the Matlab Robotics Toolbox of Peter Corke. Corke, P. I. (1996). A Robotics Toolbox for MATLAB. IEEE Robotics and Automation Magazine, 3 (1): 24-32.",
-         [
+        "details": "Pumadyn non linear 32 input data set with moderate noise. See http://www.cs.utoronto.ca/~delve/data/pumadyn/desc.html for details.",
-            "allasfamc.zip"
+        "files": [
-         ]
-      ],
-      "license":"From http://mocap.cs.cmu.edu. This data is free for use in research projects. You may include this data in commercially-sold products, but you may not resell this data directly, even in converted form. If you publish results obtained using this data, we would appreciate it if you would send the citation to your published paper to jkh+mocap@cs.cmu.edu, and also would add this text to your acknowledgments section: The data used in this project was obtained from mocap.cs.cmu.edu. The database was created with funding from NSF EIA-0196217.",
-      "citation":"Please include this in your acknowledgements: The data used in this project was obtained from mocap.cs.cmu.edu.\nThe database was created with funding from NSF EIA-0196217.",
-      "details":"CMU Motion Capture data base. Captured by a Vicon motion capture system consisting of 12 infrared MX-40 cameras, each of which is capable of recording at 120 Hz with images of 4 megapixel resolution. Motions are captured in a working volume of approximately 3m x 8m. The capture subject wears 41 markers and a stylish black garment.",
-      "urls":[
-         "http://mocap.cs.cmu.edu/subjects"
-      ],
-      "size":null
-   },
-   "brendan_faces":{
-      "files":[
-         [
-            "frey_rawface.mat"
-         ]
-      ],
-      "license":null,
-      "citation":"Frey, B. J., Colmenarez, A and Huang, T. S. Mixtures of Local Linear Subspaces for Face Recognition. Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition 1998, 32-37, June 1998. Computer Society Press, Los Alamitos, CA.",
-      "details":"A video of Brendan Frey's face popularized as a benchmark for visualization by the Locally Linear Embedding.",
-      "urls":[
-         "http://www.cs.nyu.edu/~roweis/data/"
-      ],
-      "size":1100584
-   },
-   "olympic_marathon_men":{
-      "files":[
-         [
-            "olympicMarathonTimes.csv"
-         ]
-      ],
-      "license":null,
-      "citation":null,
-      "details":"Olympic mens' marathon gold medal winning times from 1896 to 2012. Time given in pace (minutes per kilometer). Data is originally downloaded and collated from Wikipedia, we are not responsible for errors in the data",
-      "urls":[
-         "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/olympic_marathon_men/"
-      ],
-      "size":584
-   },
-   "pumadyn-32nm":{
-      "files":[
             [
                 "pumadyn-32nm.tar.gz"
             ]
         ],
-      "license":"Data is made available by the Delve system at the University of Toronto",
+        "license": "Data is made available by the Delve system at the University of Toronto",
-      "citation":"Created by Zoubin Ghahramani using the Matlab Robotics Toolbox of Peter Corke. Corke, P. I. (1996). A Robotics Toolbox for MATLAB. IEEE Robotics and Automation Magazine, 3 (1): 24-32.",
+        "size": 5861646,
-      "details":"Pumadyn non linear 32 input data set with moderate noise. See http://www.cs.utoronto.ca/~delve/data/pumadyn/desc.html for details.",
+        "urls": [
-      "urls":[
             "ftp://ftp.cs.toronto.edu/pub/neuron/delve/data/tarfiles/pumadyn-family/"
-      ],
+        ]
-      "size":5861646
     },
-   "ripley_prnn_data":{
+    "ripley_prnn_data": {
-      "files":[
+        "citation": "Pattern Recognition and Neural Networks by B.D. Ripley (1996) Cambridge University Press ISBN 0 521 46986 7",
+        "details": "Data sets from Brian Ripley's Pattern Recognition and Neural Networks",
+        "files": [
             [
                 "Cushings.dat",
                 "README",
@@ -193,16 +470,92 @@
                 "virus3.dat"
             ]
         ],
-      "license":null,
+        "license": null,
-      "citation":"Pattern Recognition and Neural Networks by B.D. Ripley (1996) Cambridge University Press ISBN 0 521 46986 7",
+        "size": 93565,
-      "details":"Data sets from Brian Ripley's Pattern Recognition and Neural Networks",
+        "urls": [
-      "urls":[
             "http://www.stats.ox.ac.uk/pub/PRNN/"
-      ],
+        ]
-      "size":93565
     },
-   "three_phase_oil_flow":{
+    "robot_wireless": {
-      "files":[
+        "citation": "WiFi-SLAM using Gaussian Process Latent Variable Models by Brian Ferris, Dieter Fox and Neil Lawrence in IJCAI'07 Proceedings pages 2480-2485. Data used in A Unifying Probabilistic Perspective for Spectral Dimensionality Reduction: Insights and New Models by Neil D. Lawrence, JMLR 13 pg 1609--1638, 2012.",
+        "details": "Data created by Brian Ferris and Dieter Fox. Consists of WiFi access point strengths taken during a circuit of the Paul Allen building at the University of Washington.",
+        "files": [
+            [
+                "uw-floor.txt"
+            ]
+        ],
+        "license": null,
+        "size": 284390,
+        "urls": [
+            "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/robot_wireless/"
+        ]
+    },
+    "rogers_girolami_data": {
+        "citation": "A First Course in Machine Learning. Simon Rogers and Mark Girolami: Chapman & Hall/CRC, ISBN-13: 978-1439824146",
+        "details": "Data from the textbook 'A First Course in Machine Learning'. Available from http://www.dcs.gla.ac.uk/~srogers/firstcourseml/.",
+        "files": [
+            [
+                "firstcoursemldata.tar.gz"
+            ]
+        ],
+        "license": null,
+        "size": 21949154,
+        "suffices": [
+            [
+                "?dl=1"
+            ]
+        ],
+        "urls": [
+            "https://www.dropbox.com/sh/7p6tu1t29idgliq/_XqlH_3nt9/"
+        ]
+    },
+    "singlecell": {
+        "citation": "Guoji Guo, Mikael Huss, Guo Qing Tong, Chaoyang Wang, Li Li Sun, Neil D. Clarke, Paul Robson, Resolution of Cell Fate Decisions Revealed by Single-Cell Gene Expression Analysis from Zygote to Blastocyst, Developmental Cell, Volume 18, Issue 4, 20 April 2010, Pages 675-685, ISSN 1534-5807, http://dx.doi.org/10.1016/j.devcel.2010.02.012. (http://www.sciencedirect.com/science/article/pii/S1534580710001103) Keywords: DEVBIO",
+        "details": "qPCR TaqMan array single cell experiment in mouse. The data is taken from the early stages of development when the Blastocyst is forming. At the 32 cell stage the data is already separated into the trophectoderm (TE) which goes onto form the placenta and the inner cellular mass (ICM). The ICM further differentiates into the epiblast (EPI)---which gives rise to the endoderm, mesoderm and ectoderm---and the primitive endoderm (PE) which develops into the amniotic sack. Guo et al selected 48 genes for expression measurement. They labelled the resulting cells and their labels are included as an aide to visualization.",
+        "files": [
+            [
+                "singlecell.csv"
+            ]
+        ],
+        "license": "ScienceDirect: http://www.elsevier.com/locate/termsandconditions?utm_source=sciencedirect&utm_medium=link&utm_campaign=terms",
+        "size": 233.1,
+        "urls": [
+            "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/singlecell/"
+        ]
+    },
+    "sod1_mouse": {
+        "citation": "Transcriptomic indices of fast and slow disease progression in two mouse models of amyotrophic lateral sclerosis' Nardo G1, Iennaco R, Fusi N, Heath PR, Marino M, Trolese MC, Ferraiuolo L, Lawrence N, Shaw PJ, Bendotti C Brain. 2013 Nov;136(Pt 11):3305-32. doi: 10.1093/brain/awt250. Epub 2013 Sep 24.",
+        "details": "Gene expression data from two separate strains of mice: C57 and 129Sv in wild type and SOD1 mutant strains.",
+        "files": [
+            [
+                "sod1_C57_129_exprs.csv",
+                "sod1_C57_129_se.csv"
+            ]
+        ],
+        "license": null,
+        "size": 0,
+        "urls": [
+            "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/sod1_mouse/"
+        ]
+    },
+    "swiss_roll": {
+        "citation": "A Global Geometric Framework for Nonlinear Dimensionality Reduction, J. B. Tenenbaum, V. de Silva and J. C. Langford, Science 290 (5500): 2319-2323, 22 December 2000",
+        "details": "Swiss roll data made available by Tenenbaum, de Silva and Langford to demonstrate isomap, available from http://isomap.stanford.edu/datasets.html.",
+        "files": [
+            [
+                "swiss_roll_data.mat"
+            ]
+        ],
+        "license": null,
+        "size": 800256,
+        "urls": [
+            "http://isomap.stanford.edu/"
+        ]
+    },
+    "three_phase_oil_flow": {
+        "citation": "Bishop, C. M. and G. D. James (1993). Analysis of multiphase flows using dual-energy gamma densitometry and neural networks. Nuclear Instruments and Methods in Physics Research A327, 580-593",
+        "details": "The three phase oil data used initially for demonstrating the Generative Topographic mapping.",
+        "files": [
             [
                 "DataTrnLbls.txt",
                 "DataTrn.txt",
@@ -212,198 +565,24 @@
                 "DataVdnLbls.txt"
             ]
         ],
-      "license":null,
+        "license": null,
-      "citation":"Bishop, C. M. and G. D. James (1993). Analysis of multiphase flows using dual-energy gamma densitometry and neural networks. Nuclear Instruments and Methods in Physics Research A327, 580-593",
+        "size": 712796,
-      "details":"The three phase oil data used initially for demonstrating the Generative Topographic mapping.",
+        "urls": [
-      "urls":[
             "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/three_phase_oil_flow/"
-      ],
-      "size":712796
-   },
-   "robot_wireless":{
-      "files":[
-         [
-            "uw-floor.txt"
         ]
-      ],
-      "license":null,
-      "citation":"WiFi-SLAM using Gaussian Process Latent Variable Models by Brian Ferris, Dieter Fox and Neil Lawrence in IJCAI'07 Proceedings pages 2480-2485. Data used in A Unifying Probabilistic Perspective for Spectral Dimensionality Reduction: Insights and New Models by Neil D. Lawrence, JMLR 13 pg 1609--1638, 2012.",
-      "details":"Data created by Brian Ferris and Dieter Fox. Consists of WiFi access point strengths taken during a circuit of the Paul Allen building at the University of Washington.",
-      "urls":[
-         "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/robot_wireless/"
-      ],
-      "size":284390
     },
-   "xw_pen":{
+    "xw_pen": {
-      "files":[
+        "citation": "Michael E. Tipping and Neil D. Lawrence. Variational inference for Student-t models: Robust Bayesian interpolation and generalised component analysis. Neurocomputing, 69:123--141, 2005",
+        "details": "Accelerometer pen data used for robust regression by Tipping and Lawrence.",
+        "files": [
             [
                 "xw_pen_15.csv"
             ]
         ],
-      "license":null,
+        "license": null,
-      "citation":"Michael E. Tipping and Neil D. Lawrence. Variational inference for Student-t models: Robust Bayesian interpolation and generalised component analysis. Neurocomputing, 69:123--141, 2005",
+        "size": 3410,
-      "details":"Accelerometer pen data used for robust regression by Tipping and Lawrence.",
+        "urls": [
-      "urls":[
             "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/xw_pen/"
-      ],
-      "size":3410
-   },
-   "swiss_roll":{
-      "files":[
-         [
-            "swiss_roll_data.mat"
         ]
-      ],
-      "license":null,
-      "citation":"A Global Geometric Framework for Nonlinear Dimensionality Reduction, J. B. Tenenbaum, V. de Silva and J. C. Langford, Science 290 (5500): 2319-2323, 22 December 2000",
-      "details":"Swiss roll data made available by Tenenbaum, de Silva and Langford to demonstrate isomap, available from http://isomap.stanford.edu/datasets.html.",
-      "urls":[
-         "http://isomap.stanford.edu/"
-      ],
-      "size":800256
-   },
-   "osu_run1":{
-      "files":[
-         [
-            "run1TXT.ZIP"
-         ],
-         [
-            "connections.txt"
-         ]
-      ],
-      "license":"Data is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License (http://creativecommons.org/licenses/by-nc-sa/3.0/).",
-      "citation":"The Open Motion Data Project by The Ohio State University Advanced Computing Center for the Arts and Design, http://accad.osu.edu/research/mocap/mocap_data.htm.",
-      "details":"Motion capture data of a stick man running from the Open Motion Data Project at Ohio State University.",
-      "urls":[
-         "http://accad.osu.edu/research/mocap/data/",
-         "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/stick/"
-      ],
-      "size":338103
-   },
-   "creep_rupture":{
-      "files":[
-         [
-            "creeprupt.tar"
-         ]
-      ],
-      "license":null,
-      "citation":"Materials Algorithms Project Data Library: MAP_DATA_CREEP_RUPTURE. F. Brun and T. Yoshida.",
-      "details":"Provides 2066 creep rupture test results of steels (mainly of two kinds of steels: 2.25Cr and 9-12 wt% Cr ferritic steels). See http://www.msm.cam.ac.uk/map/data/materials/creeprupt-b.html.",
-      "urls":[
-         "http://www.msm.cam.ac.uk/map/data/tar/"
-      ],
-      "size":602797
-   },
-   "olivetti_faces":{
-      "files":[
-         [
-            "att_faces.zip"
-         ],
-         [
-            "olivettifaces.mat"
-         ]
-      ],
-      "license":null,
-      "citation":"Ferdinando Samaria and Andy Harter, Parameterisation of a Stochastic Model for Human Face Identification. Proceedings of 2nd IEEE Workshop on Applications of Computer Vision, Sarasota FL, December 1994",
-      "details":"Olivetti Research Labs Face data base, acquired between December 1992 and December 1994 in the Olivetti Research Lab, Cambridge (which later became AT&T Laboratories, Cambridge). When using these images please give credit to AT&T Laboratories, Cambridge. ",
-      "urls":[
-         "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/olivetti_faces/",
-         "http://www.cs.nyu.edu/~roweis/data/"
-      ],
-      "size":8561331
-   },
-   "olivetti_glasses":{
-      "files":[
-         [
-            "has_glasses.np"
-         ],
-         [
-            "olivettifaces.mat"
-         ]
-      ],
-      "license":null,
-      "citation":"Information recorded in olivetti_faces entry. Should be used from there.",
-      "details":"Information recorded in olivetti_faces entry. Should be used from there.",
-      "urls":[
-         "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/olivetti_faces/",
-         "http://www.cs.nyu.edu/~roweis/data/"
-      ],
-      "size":4261047
-   },
-   "della_gatta":{
-      "files":[
-         [
-            "DellaGattadata.mat"
-         ]
-      ],
-      "license":null,
-      "citation":"Direct targets of the TRP63 transcription factor revealed by a combination of gene expression profiling and reverse engineering. Giusy Della Gatta, Mukesh Bansal, Alberto Ambesi-Impiombato, Dario Antonini, Caterina Missero, and Diego di Bernardo, Genome Research 2008",
-      "details":"The full gene expression data set from della Gatta et al (http://www.ncbi.nlm.nih.gov/pmc/articles/PMC2413161/) processed by RMA.",
-      "urls":[
-         "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/della_gatta/"
-      ],
-      "size":3729650
-   },
-   "epomeo_gpx":{
-      "files":[
-         [
-            "endomondo_1.gpx",
-            "endomondo_2.gpx",
-            "garmin_watch_via_endomondo.gpx",
-            "viewranger_phone.gpx",
-            "viewranger_tablet.gpx"
-         ]
-      ],
-      "license":null,
-      "citation":"",
-      "details":"Five different GPS traces of the same run up Mount Epomeo in Ischia. The traces are from different sources. endomondo_1 and endomondo_2 are traces from the mobile phone app Endomondo, with a split in the middle. garmin_watch_via_endomondo is the trace from a Garmin watch, with a segment missing about 4 kilometers in. viewranger_phone and viewranger_tablet are traces from a phone and a tablet through the viewranger app. The viewranger_phone data comes from the same mobile phone as the Endomondo data (i.e. there are 3 GPS devices, but one device recorded two traces).",
-      "urls":[
-         "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/epomeo_gpx/"
-      ],
-      "size":2031872
-   },
-   "mauna_loa":{
-      "files":[
-         [
-            "co2_mm_mlo.txt"
-         ]
-      ],
-      "license":"-------------------------------------------------------------------- USE OF NOAA ESRL DATA\n\n  These data are made freely available to the public and the scientific community in the belief that their wide dissemination will lead to greater understanding and new scientific insights. The availability of these data does not constitute publication of the data.  NOAA relies on the ethics and integrity of the user to insure that ESRL receives fair credit for their work.  If the data  are obtained for potential use in a publication or presentation,  ESRL should be informed at the outset of the nature of this work.   If the ESRL data are essential to the work, or if an important  result or conclusion depends on the ESRL data, co-authorship may be appropriate.  This should be discussed at an early stage in the work.  Manuscripts using the ESRL data should be sent to ESRL for review before they are submitted for publication so we can insure that the quality and limitations of the data are accurately represented.\n\n  Contact:   Pieter Tans (303 497 6678; pieter.tans@noaa.gov)\n\n  RECIPROCITY  Use of these data implies an agreement to reciprocate. Laboratories making similar measurements agree to make their own data available to the general public and to the scientific community in an equally complete and easily accessible form. Modelers are encouraged to make available to the community, upon request, their own tools used in the interpretation of the ESRL data, namely well documented model code, transport fields, and additional information necessary for other scientists to repeat the work and to run modified versions. Model availability includes collaborative support for new users of the models.\n --------------------------------------------------------------------\n\n     See www.esrl.noaa.gov/gmd/ccgg/trends/ for additional details.",
-      "citation":"Mauna Loa Data. Dr. Pieter Tans, NOAA/ESRL (www.esrl.noaa.gov/gmd/ccgg/trends/) and Dr. Ralph Keeling, Scripps Institution of Oceanography (scrippsco2.ucsd.edu/).",
-      "details":"The 'average' column contains the monthly mean CO2 mole fraction determined from daily averages.  The mole fraction of CO2, expressed as parts per million (ppm) is the number of molecules of CO2 in every one million molecules of dried air (water vapor removed).  If there are missing days concentrated either early or late in the month, the monthly mean is corrected to the middle of the month using the average seasonal cycle.  Missing months are denoted by -99.99. The 'interpolated' column includes average values from the preceding column and interpolated values where data are missing.  Interpolated values are computed in two steps.  First, we compute for each month the average seasonal cycle in a 7-year window around each monthly value.  In this way the seasonal cycle is allowed to change slowly over time.  We then determine the 'trend' value for each month by removing the seasonal cycle; this result is shown in the 'trend' column.  Trend values are linearly interpolated for missing months. The interpolated monthly mean is then the sum of the average seasonal cycle value and the trend value for the missing month.\n\nNOTE: In general, the data presented for the last year are subject to change, depending on recalibration of the reference gas mixtures used, and other quality control procedures. Occasionally, earlier years may also be changed for the same reasons.  Usually these changes are minor.\n\nCO2 expressed as a mole fraction in dry air, micromol/mol, abbreviated as ppm \n\n (-99.99 missing data;  -1 no data for daily means in month)",
-      "urls":[
-         "ftp://aftp.cmdl.noaa.gov/products/trends/co2/"
-      ],
-      "size":46779
-   },
-   "boxjenkins_airline":{
-      "files":[
-         [
-            "boxjenkins_airline.csv"
-         ]
-      ],
-      "license":"You may copy and redistribute the data. You may make derivative works from the data. You may use the data for commercial purposes. You may not sublicence the data when redistributing it. You may not redistribute the data under a different license. Source attribution on any use of this data: Must refer source.",
-      "citation":"Box & Jenkins (1976), in file: data/airpass, Description: International airline passengers: monthly totals in thousands. Jan 49 – Dec 60",
-      "details":"International airline passengers, monthly totals from January 1949 to December 1960.",
-      "urls":[
-                  "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/boxjenkins_airline/"
-      ],
-      "size":46779
-   },
-
-   "decampos_characters":{
-      "files":[
-         [
-            "characters.npy",
-            "digits.npy"
-         ]
-      ],
-      "license":null,
-      "citation":"T. de Campos, B. R. Babu, and M. Varma. Character recognition in natural images. VISAPP 2009.",
-      "details":"Examples of hand written digits taken from the de Campos et al paper on Character Recognition in Natural Images.",
-      "urls":[
-         "http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/decampos_digits/"
-      ],
-      "size":2031872
     }
 }

GPy/util/datasets.py

@@ -12,6 +12,8 @@ import datetime
 import json
 import re
 
+from config import *
+
 ipython_available=True
 try:
     import IPython
@@ -29,7 +31,8 @@ def reporthook(a,b,c):
     sys.stdout.flush()
 
 # Global variables
-data_path = os.path.join(os.path.dirname(__file__), 'datasets')
+data_path = os.path.expandvars(config.get('datasets', 'dir'))
+#data_path = os.path.join(os.path.dirname(__file__), 'datasets')
 default_seed = 10000
 overide_manual_authorize=False
 neil_url = 'http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/'
@@ -108,7 +111,11 @@ def download_url(url, store_directory, save_name = None, messages = True, suffix
             raise ValueError('Tried url ' + url + suffix + ' and received server error ' + str(response.code))
     with open(save_name, 'wb') as f:
         meta = response.info()
-        file_size = int(meta.getheaders("Content-Length")[0])
+        content_length_str = meta.getheaders("Content-Length")
+        if content_length_str:
+            file_size = int(content_length_str[0])
+        else:
+            file_size = None
         status = ""
         file_size_dl = 0
         block_sz = 8192
@@ -120,9 +127,15 @@ def download_url(url, store_directory, save_name = None, messages = True, suffix
             file_size_dl += len(buff)
             f.write(buff)
             sys.stdout.write(" "*(len(status)) + "\r")
-            status = r"[{perc: <{ll}}] {dl:7.3f}/{full:.3f}MB".format(dl=file_size_dl/(1.*1e6), 
+            if file_size:
-                                                                       full=file_size/(1.*1e6), ll=line_length, 
+                status = r"[{perc: <{ll}}] {dl:7.3f}/{full:.3f}MB".format(dl=file_size_dl/(1048576.),
+                                                                       full=file_size/(1048576.), ll=line_length,
                                                                        perc="="*int(line_length*float(file_size_dl)/file_size))
+            else:
+                status = r"[{perc: <{ll}}] {dl:7.3f}MB".format(dl=file_size_dl/(1048576.),
+                                                                       ll=line_length,
+                                                                       perc="."*int(line_length*float(file_size_dl/(10*1048576.))))
+
             sys.stdout.write(status)
             sys.stdout.flush()
         sys.stdout.write(" "*(len(status)) + "\r")
@@ -227,7 +240,7 @@ def cmu_urls_files(subj_motions, messages = True):
         if not os.path.exists(cur_skel_file):
             # Current skel file doesn't exist.
             if not os.path.isdir(skel_dir):
-                os.mkdir(skel_dir)
+                os.makedirs(skel_dir)
             # Add skel file to list.
             url_required = True
             file_download.append(subjects[i] + '.asf')
@@ -350,32 +363,159 @@ def football_data(season='1314', data_set='football_data'):
         Y = table[:, 4:]
     return data_details_return({'X': X, 'Y': Y}, data_set)
 
-# This will be for downloading google trends data.
+def sod1_mouse(data_set='sod1_mouse'):
-def google_trends(query_terms=['big data', 'machine learning', 'data science'], data_set='google_trends'):
+    if not data_available(data_set):
-    """Data downloaded from Google trends for given query terms. Warning, if you use this function multiple times in a row you get blocked due to terms of service violations."""
+        download_data(data_set)
-    # Inspired by this notebook:
+    from pandas import read_csv
-    # http://nbviewer.ipython.org/github/sahuguet/notebooks/blob/master/GoogleTrends%20meet%20Notebook.ipynb
+    dir_path = os.path.join(data_path, data_set)
+    filename = os.path.join(dir_path, 'sod1_C57_129_exprs.csv')
+    Y = read_csv(filename, header=0, index_col=0)
+    num_repeats=4
+    num_time=4
+    num_cond=4
+    X = 1
+    return data_details_return({'X': X, 'Y': Y}, data_set)
 
+def spellman_yeast(data_set='spellman_yeast'):
+    if not data_available(data_set):
+        download_data(data_set)
+    from pandas import read_csv
+    dir_path = os.path.join(data_path, data_set)
+    filename = os.path.join(dir_path, 'combined.txt')
+    Y = read_csv(filename, header=0, index_col=0, sep='\t')
+    return data_details_return({'Y': Y}, data_set)
+
+def spellman_yeast_cdc15(data_set='spellman_yeast'):
+    if not data_available(data_set):
+        download_data(data_set)
+    from pandas import read_csv
+    dir_path = os.path.join(data_path, data_set)
+    filename = os.path.join(dir_path, 'combined.txt')
+    Y = read_csv(filename, header=0, index_col=0, sep='\t')
+    t = np.asarray([10, 30, 50, 70, 80, 90, 100, 110, 120, 130, 140, 150, 170, 180, 190, 200, 210, 220, 230, 240, 250, 270, 290])
+    times = ['cdc15_'+str(time) for time in t]
+    Y = Y[times].T
+    t = t[:, None]
+    return data_details_return({'Y' : Y, 't': t, 'info': 'Time series of synchronized yeast cells from the CDC-15 experiment of Spellman et al (1998).'}, data_set)
+
+def lee_yeast_ChIP(data_set='lee_yeast_ChIP'):
+    if not data_available(data_set):
+        download_data(data_set)
+    from pandas import read_csv
+    import zipfile
+    dir_path = os.path.join(data_path, data_set)
+    filename = os.path.join(dir_path, 'binding_by_gene.tsv')
+    S = read_csv(filename, header=1, index_col=0, sep='\t')
+    transcription_factors = [col for col in S.columns if col[:7] != 'Unnamed']
+    annotations = S[['Unnamed: 1', 'Unnamed: 2', 'Unnamed: 3']]
+    S = S[transcription_factors]
+    return data_details_return({'annotations' : annotations, 'Y' : S, 'transcription_factors': transcription_factors}, data_set)
+
+
+
+def fruitfly_tomancak(data_set='fruitfly_tomancak', gene_number=None):
+    if not data_available(data_set):
+        download_data(data_set)
+    from pandas import read_csv
+    dir_path = os.path.join(data_path, data_set)
+    filename = os.path.join(dir_path, 'tomancak_exprs.csv')
+    Y = read_csv(filename, header=0, index_col=0).T
+    num_repeats = 3
+    num_time = 12
+    xt = np.linspace(0, num_time-1, num_time)
+    xr = np.linspace(0, num_repeats-1, num_repeats)
+    xtime, xrepeat = np.meshgrid(xt, xr)
+    X = np.vstack((xtime.flatten(), xrepeat.flatten())).T
+    return data_details_return({'X': X, 'Y': Y, 'gene_number' : gene_number}, data_set)
+
+def drosophila_protein(data_set='drosophila_protein'):
+    if not data_available(data_set):
+        download_data(data_set)
+    from pandas import read_csv
+    dir_path = os.path.join(data_path, data_set)
+    filename = os.path.join(dir_path, 'becker_et_al.csv')
+    Y = read_csv(filename, header=0)
+    return data_details_return({'Y': Y}, data_set)
+
+def drosophila_knirps(data_set='drosophila_protein'):
+    if not data_available(data_set):
+        download_data(data_set)
+    from pandas import read_csv
+    dir_path = os.path.join(data_path, data_set)
+    filename = os.path.join(dir_path, 'becker_et_al.csv')
+    # in the csv file we have facts_kni and ext_kni. We treat facts_kni as protein and ext_kni as mRNA
+    df = read_csv(filename, header=0)
+    t = df['t'][:,None]
+    x = df['x'][:,None]
+
+    g = df['expression1'][:,None]
+    p = df['expression2'][:,None]
+
+    leng = x.shape[0]
+
+    T = np.vstack([t,t])
+    S = np.vstack([x,x])
+    inx = np.zeros(leng*2)[:,None]
+
+    inx[leng*2/2:leng*2]=1
+    X = np.hstack([T,S,inx])
+    Y = np.vstack([g,p])
+    return data_details_return({'Y': Y, 'X': X}, data_set)
+
+# This will be for downloading google trends data.
+def google_trends(query_terms=['big data', 'machine learning', 'data science'], data_set='google_trends', refresh_data=False):
+    """Data downloaded from Google trends for given query terms. Warning, if you use this function multiple times in a row you get blocked due to terms of service violations. The function will cache the result of your query, if you wish to refresh an old query set refresh_data to True. The function is inspired by this notebook: http://nbviewer.ipython.org/github/sahuguet/notebooks/blob/master/GoogleTrends%20meet%20Notebook.ipynb"""
+    query_terms.sort()
+    import pandas
+
+    # Create directory name for data
+    dir_path = os.path.join(data_path,'google_trends')
+    if not os.path.isdir(dir_path):
+        os.makedirs(dir_path)
+    dir_name = '-'.join(query_terms)
+    dir_name = dir_name.replace(' ', '_')
+    dir_path = os.path.join(dir_path,dir_name)
+    file = 'data.csv'
+    file_name = os.path.join(dir_path,file)
+    if not os.path.exists(file_name) or refresh_data:
+        print "Accessing Google trends to acquire the data. Note that repeated accesses will result in a block due to a google terms of service violation. Failure at this point may be due to such blocks."
         # quote the query terms.
-    for i, element in enumerate(query_terms):
+        quoted_terms = []
-        query_terms[i] = urllib2.quote(element)
+        for term in query_terms:
-    query = 'http://www.google.com/trends/fetchComponent?q=%s&cid=TIMESERIES_GRAPH_0&export=3' % ",".join(query_terms)
+            quoted_terms.append(urllib2.quote(term))
+        print "Query terms: ", ', '.join(query_terms)
+
+        print "Fetching query:"
+        query = 'http://www.google.com/trends/fetchComponent?q=%s&cid=TIMESERIES_GRAPH_0&export=3' % ",".join(quoted_terms)
 
         data = urllib2.urlopen(query).read()
-
+        print "Done."
         # In the notebook they did some data cleaning: remove Javascript header+footer, and translate new Date(....,..,..) into YYYY-MM-DD.
         header = """// Data table response\ngoogle.visualization.Query.setResponse("""
         data = data[len(header):-2]
         data = re.sub('new Date\((\d+),(\d+),(\d+)\)', (lambda m: '"%s-%02d-%02d"' % (m.group(1).strip(), 1+int(m.group(2)), int(m.group(3)))), data)
         timeseries = json.loads(data)
-    #import pandas as pd
         columns = [k['label'] for k in timeseries['table']['cols']]
         rows = map(lambda x: [k['v'] for k in x['c']], timeseries['table']['rows'])
-    terms = len(columns)-1
+        df = pandas.DataFrame(rows, columns=columns)
-    X = np.asarray([(pb.datestr2num(row[0]), i) for i in range(terms) for row in rows ])
+        if not os.path.isdir(dir_path):
-    Y = np.asarray([[row[i+1]] for i in range(terms) for row in rows ])
+            os.makedirs(dir_path)
+
+        df.to_csv(file_name)
+    else:
+        print "Reading cached data for google trends. To refresh the cache set 'refresh_data=True' when calling this function."
+        print "Query terms: ", ', '.join(query_terms)
+
+        df = pandas.read_csv(file_name, parse_dates=[0])
+
+    columns = df.columns
+    terms = len(query_terms)
+    import datetime
+    X = np.asarray([(row, i) for i in range(terms) for row in df.index])
+    Y = np.asarray([[df.ix[row][query_terms[i]]] for i in range(terms) for row in df.index ])
     output_info = columns[1:]
-    return data_details_return({'X': X, 'Y': Y, 'query_terms': output_info, 'info': "Data downloaded from google trends with query terms: " + ', '.join(output_info) + '.'}, data_set)
+
+    return data_details_return({'data frame' : df, 'X': X, 'Y': Y, 'query_terms': output_info, 'info': "Data downloaded from google trends with query terms: " + ', '.join(output_info) + '.'}, data_set)
 
 # The data sets
 def oil(data_set='three_phase_oil_flow'):
@@ -519,7 +659,23 @@ def ripley_synth(data_set='ripley_prnn_data'):
     ytest = test[:, 2:3]
     return data_details_return({'X': X, 'Y': y, 'Xtest': Xtest, 'Ytest': ytest, 'info': 'Synthetic data generated by Ripley for a two class classification problem.'}, data_set)
 
-def mauna_loa(data_set='mauna_loa', num_train=543, refresh_data=False):
+def global_average_temperature(data_set='global_temperature', num_train=1000, refresh_data=False):
+    path = os.path.join(data_path, data_set)
+    if data_available(data_set) and not refresh_data:
+        print 'Using cached version of the data set, to use latest version set refresh_data to True'
+    else:
+        download_data(data_set)
+    data = np.loadtxt(os.path.join(data_path, data_set, 'GLBTS.long.data'))
+    print 'Most recent data observation from month ', data[-1, 1], ' in year ', data[-1, 0]
+    allX = data[data[:, 3]!=-99.99, 2:3]
+    allY = data[data[:, 3]!=-99.99, 3:4]
+    X = allX[:num_train, 0:1]
+    Xtest = allX[num_train:, 0:1]
+    Y = allY[:num_train, 0:1]
+    Ytest = allY[num_train:, 0:1]
+    return data_details_return({'X': X, 'Y': Y, 'Xtest': Xtest, 'Ytest': Ytest, 'info': "Mauna Loa data with " + str(num_train) + " values used as training points."}, data_set)
+
+def mauna_loa(data_set='mauna_loa', num_train=545, refresh_data=False):
     path = os.path.join(data_path, data_set)
     if data_available(data_set) and not refresh_data:
         print 'Using cached version of the data set, to use latest version set refresh_data to True'
@@ -560,7 +716,7 @@ def osu_run1(data_set='osu_run1', sample_every=4):
     return data_details_return({'Y': Y, 'connect' : connect}, data_set)
 
 def swiss_roll_generated(num_samples=1000, sigma=0.0):
-    with open(os.path.join(data_path, 'swiss_roll.pickle')) as f:
+    with open(os.path.join(os.path.dirname(__file__), 'datasets', 'swiss_roll.pickle')) as f:
         data = pickle.load(f)
     Na = data['Y'].shape[0]
     perm = np.random.permutation(np.r_[:Na])[:num_samples]
@@ -612,14 +768,20 @@ def hapmap3(data_set='hapmap3'):
         import bz2
     except ImportError as i:
         raise i, "Need pandas for hapmap dataset, make sure to install pandas (http://pandas.pydata.org/) before loading the hapmap dataset"
-    if not data_available(data_set):
+
-        download_data(data_set)
+    dir_path = os.path.join(data_path,'hapmap3')
-    dirpath = os.path.join(data_path,'hapmap3')
     hapmap_file_name = 'hapmap3_r2_b36_fwd.consensus.qc.poly'
-    preprocessed_data_paths = [os.path.join(dirpath,hapmap_file_name + file_name) for file_name in \
+    unpacked_files = [os.path.join(dir_path, hapmap_file_name+ending) for ending in ['.ped', '.map']]
+    unpacked_files_exist = reduce(lambda a, b:a and b, map(os.path.exists, unpacked_files))
+
+    if not unpacked_files_exist and not data_available(data_set):
+        download_data(data_set)
+
+    preprocessed_data_paths = [os.path.join(dir_path,hapmap_file_name + file_name) for file_name in \
                                ['.snps.pickle',
                                 '.info.pickle',
                                 '.nan.pickle']]
+
     if not reduce(lambda a,b: a and b, map(os.path.exists, preprocessed_data_paths)):
         if not overide_manual_authorize and not prompt_user("Preprocessing requires ~25GB "
                             "of memory and can take a (very) long time, continue? [Y/n]"):
@@ -633,8 +795,7 @@ def hapmap3(data_set='hapmap3'):
                                                                perc="="*int(20.*progress/100.))
             stdout.write(status); stdout.flush()
             return status
-        unpacked_files = [os.path.join(dirpath, hapmap_file_name+ending) for ending in ['.ped', '.map']]
+        if not unpacked_files_exist:
-        if not reduce(lambda a,b: a and b, map(os.path.exists, unpacked_files)):
             status=write_status('unpacking...', 0, '')
             curr = 0
             for newfilepath in unpacked_files:
@@ -651,6 +812,7 @@ def hapmap3(data_set='hapmap3'):
                             status=write_status('unpacking...', curr+12.*file_processed/(file_size), status)
                 curr += 12
                 status=write_status('unpacking...', curr, status)
+                os.remove(filepath)
         status=write_status('reading .ped...', 25, status)
         # Preprocess data:
         snpstrnp = np.loadtxt(unpacked_files[0], dtype=str)
@@ -658,7 +820,7 @@ def hapmap3(data_set='hapmap3'):
         mapnp = np.loadtxt(unpacked_files[1], dtype=str)
         status=write_status('reading relationships.txt...', 42, status)
         # and metainfo:
-        infodf = DataFrame.from_csv(os.path.join(dirpath,'./relationships_w_pops_121708.txt'), header=0, sep='\t')
+        infodf = DataFrame.from_csv(os.path.join(dir_path,'./relationships_w_pops_121708.txt'), header=0, sep='\t')
         infodf.set_index('IID', inplace=1)
         status=write_status('filtering nan...', 45, status)
         snpstr = snpstrnp[:,6:].astype('S1').reshape(snpstrnp.shape[0], -1, 2)
@@ -718,6 +880,21 @@ def hapmap3(data_set='hapmap3'):
                   populations=populations)
     return hapmap
 
+def singlecell(data_set='singlecell'):
+    if not data_available(data_set):
+        download_data(data_set)
+
+    from pandas import read_csv
+    dir_path = os.path.join(data_path, data_set)
+    filename = os.path.join(dir_path, 'singlecell.csv')
+    Y = read_csv(filename, header=0, index_col=0)
+    genes = Y.columns
+    labels = Y.index
+    # data = np.loadtxt(os.path.join(dir_path, 'singlecell.csv'), delimiter=",", dtype=str)
+    return data_details_return({'Y': Y, 'info' : "qPCR singlecell experiment in Mouse, measuring 48 gene expressions in 1-64 cell states. The labels have been created as in Guo et al. [2010]",
+                                'genes': genes, 'labels':labels,
+                                }, data_set)
+
 def swiss_roll_1000():
     return swiss_roll(num_samples=1000)
 
@@ -811,7 +988,8 @@ def olivetti_faces(data_set='olivetti_faces'):
     for subject in range(40):
         for image in range(10):
             image_path = os.path.join(path, 'orl_faces', 's'+str(subject+1), str(image+1) + '.pgm')
-            Y.append(GPy.util.netpbmfile.imread(image_path).flatten())
+            from GPy.util import netpbmfile
+            Y.append(netpbmfile.imread(image_path).flatten())
             lbls.append(subject)
     Y = np.asarray(Y)
     lbls = np.asarray(lbls)[:, None]
@@ -1013,6 +1191,30 @@ def creep_data(data_set='creep_rupture'):
     X = all_data[:, features].copy()
     return data_details_return({'X': X, 'y': y}, data_set)
 
+def cifar10_patches(data_set='cifar-10'):
+    """The Candian Institute for Advanced Research 10 image data set. Code for loading in this data is taken from this Boris Babenko's blog post, original code available here: http://bbabenko.tumblr.com/post/86756017649/learning-low-level-vision-feautres-in-10-lines-of-code"""
+    dir_path = os.path.join(data_path, data_set)
+    filename = os.path.join(dir_path, 'cifar-10-python.tar.gz')
+    if not data_available(data_set):
+        download_data(data_set)
+        import tarfile
+        # This code is from Boris Babenko's blog post.
+        # http://bbabenko.tumblr.com/post/86756017649/learning-low-level-vision-feautres-in-10-lines-of-code
+        tfile = tarfile.open(filename, 'r:gz')
+        tfile.extractall(dir_path)
+
+    with open(os.path.join(dir_path, 'cifar-10-batches-py','data_batch_1'),'rb') as f:
+        data = pickle.load(f)
+
+    images = data['data'].reshape((-1,3,32,32)).astype('float32')/255
+    images = np.rollaxis(images, 1, 4)
+    patches = np.zeros((0,5,5,3))
+    for x in range(0,32-5,5):
+        for y in range(0,32-5,5):
+            patches = np.concatenate((patches, images[:,x:x+5,y:y+5,:]), axis=0)
+    patches = patches.reshape((patches.shape[0],-1))
+    return data_details_return({'Y': patches, "info" : "32x32 pixel patches extracted from the CIFAR-10 data by Boris Babenko to demonstrate k-means features."}, data_set)
+
 def cmu_mocap_49_balance(data_set='cmu_mocap'):
     """Load CMU subject 49's one legged balancing motion that was used by Alvarez, Luengo and Lawrence at AISTATS 2009."""
     train_motions = ['18', '19']

GPy/util/datasets/COPYRIGHT

@@ -1 +0,0 @@
-These datasets are reproduced for educational purposes only. No copyright infringement intended!

GPy/util/datasets/column_2C

@@ -1,310 +0,0 @@
-63.03 22.55 39.61 40.48 98.67 -0.25 1
-39.06 10.06 25.02 29 114.41 4.56 1
-68.83 22.22 50.09 46.61 105.99 -3.53 1
-69.3 24.65 44.31 44.64 101.87 11.21 1
-49.71 9.65 28.32 40.06 108.17 7.92 1
-40.25 13.92 25.12 26.33 130.33 2.23 1
-53.43 15.86 37.17 37.57 120.57 5.99 1
-45.37 10.76 29.04 34.61 117.27 -10.68 1
-43.79 13.53 42.69 30.26 125 13.29 1
-36.69 5.01 41.95 31.68 84.24 0.66 1
-49.71 13.04 31.33 36.67 108.65 -7.83 1
-31.23 17.72 15.5 13.52 120.06 0.5 1
-48.92 19.96 40.26 28.95 119.32 8.03 1
-53.57 20.46 33.1 33.11 110.97 7.04 1
-57.3 24.19 47 33.11 116.81 5.77 1
-44.32 12.54 36.1 31.78 124.12 5.42 1
-63.83 20.36 54.55 43.47 112.31 -0.62 1
-31.28 3.14 32.56 28.13 129.01 3.62 1
-38.7 13.44 31 25.25 123.16 1.43 1
-41.73 12.25 30.12 29.48 116.59 -1.24 1
-43.92 14.18 37.83 29.74 134.46 6.45 1
-54.92 21.06 42.2 33.86 125.21 2.43 1
-63.07 24.41 54 38.66 106.42 15.78 1
-45.54 13.07 30.3 32.47 117.98 -4.99 1
-36.13 22.76 29 13.37 115.58 -3.24 1
-54.12 26.65 35.33 27.47 121.45 1.57 1
-26.15 10.76 14 15.39 125.2 -10.09 1
-43.58 16.51 47 27.07 109.27 8.99 1
-44.55 21.93 26.79 22.62 111.07 2.65 1
-66.88 24.89 49.28 41.99 113.48 -2.01 1
-50.82 15.4 42.53 35.42 112.19 10.87 1
-46.39 11.08 32.14 35.31 98.77 6.39 1
-44.94 17.44 27.78 27.49 117.98 5.57 1
-38.66 12.99 40 25.68 124.91 2.7 1
-59.6 32 46.56 27.6 119.33 1.47 1
-31.48 7.83 24.28 23.66 113.83 4.39 1
-32.09 6.99 36 25.1 132.26 6.41 1
-35.7 19.44 20.7 16.26 137.54 -0.26 1
-55.84 28.85 47.69 27 123.31 2.81 1
-52.42 19.01 35.87 33.41 116.56 1.69 1
-35.49 11.7 15.59 23.79 106.94 -3.46 1
-46.44 8.4 29.04 38.05 115.48 2.05 1
-53.85 19.23 32.78 34.62 121.67 5.33 1
-66.29 26.33 47.5 39.96 121.22 -0.8 1
-56.03 16.3 62.28 39.73 114.02 -2.33 1
-50.91 23.02 47 27.9 117.42 -2.53 1
-48.33 22.23 36.18 26.1 117.38 6.48 1
-41.35 16.58 30.71 24.78 113.27 -4.5 1
-40.56 17.98 34 22.58 121.05 -1.54 1
-41.77 17.9 20.03 23.87 118.36 2.06 1
-55.29 20.44 34 34.85 115.88 3.56 1
-74.43 41.56 27.7 32.88 107.95 5 1
-50.21 29.76 36.1 20.45 128.29 5.74 1
-30.15 11.92 34 18.23 112.68 11.46 1
-41.17 17.32 33.47 23.85 116.38 -9.57 1
-47.66 13.28 36.68 34.38 98.25 6.27 1
-43.35 7.47 28.07 35.88 112.78 5.75 1
-46.86 15.35 38 31.5 116.25 1.66 1
-43.2 19.66 35 23.54 124.85 -2.92 1
-48.11 14.93 35.56 33.18 124.06 7.95 1
-74.38 32.05 78.77 42.32 143.56 56.13 1
-89.68 32.7 83.13 56.98 129.96 92.03 1
-44.53 9.43 52 35.1 134.71 29.11 1
-77.69 21.38 64.43 56.31 114.82 26.93 1
-76.15 21.94 82.96 54.21 123.93 10.43 1
-83.93 41.29 62 42.65 115.01 26.59 1
-78.49 22.18 60 56.31 118.53 27.38 1
-75.65 19.34 64.15 56.31 95.9 69.55 1
-72.08 18.95 51 53.13 114.21 1.01 1
-58.6 -0.26 51.5 58.86 102.04 28.06 1
-72.56 17.39 52 55.18 119.19 32.11 1
-86.9 32.93 47.79 53.97 135.08 101.72 1
-84.97 33.02 60.86 51.95 125.66 74.33 1
-55.51 20.1 44 35.42 122.65 34.55 1
-72.22 23.08 91 49.14 137.74 56.8 1
-70.22 39.82 68.12 30.4 148.53 145.38 1
-86.75 36.04 69.22 50.71 139.41 110.86 1
-58.78 7.67 53.34 51.12 98.5 51.58 1
-67.41 17.44 60.14 49.97 111.12 33.16 1
-47.74 12.09 39 35.66 117.51 21.68 1
-77.11 30.47 69.48 46.64 112.15 70.76 1
-74.01 21.12 57.38 52.88 120.21 74.56 1
-88.62 29.09 47.56 59.53 121.76 51.81 1
-81.1 24.79 77.89 56.31 151.84 65.21 1
-76.33 42.4 57.2 33.93 124.27 50.13 1
-45.44 9.91 45 35.54 163.07 20.32 1
-59.79 17.88 59.21 41.91 119.32 22.12 1
-44.91 10.22 44.63 34.7 130.08 37.36 1
-56.61 16.8 42 39.81 127.29 24.02 1
-71.19 23.9 43.7 47.29 119.86 27.28 1
-81.66 28.75 58.23 52.91 114.77 30.61 1
-70.95 20.16 62.86 50.79 116.18 32.52 1
-85.35 15.84 71.67 69.51 124.42 76.02 1
-58.1 14.84 79.65 43.26 113.59 50.24 1
-94.17 15.38 67.71 78.79 114.89 53.26 1
-57.52 33.65 50.91 23.88 140.98 148.75 1
-96.66 19.46 90.21 77.2 120.67 64.08 1
-74.72 19.76 82.74 54.96 109.36 33.31 1
-77.66 22.43 93.89 55.22 123.06 61.21 1
-58.52 13.92 41.47 44.6 115.51 30.39 1
-84.59 30.36 65.48 54.22 108.01 25.12 1
-79.94 18.77 63.31 61.16 114.79 38.54 1
-70.4 13.47 61.2 56.93 102.34 25.54 1
-49.78 6.47 53 43.32 110.86 25.34 1
-77.41 29.4 63.23 48.01 118.45 93.56 1
-65.01 27.6 50.95 37.41 116.58 7.02 1
-65.01 9.84 57.74 55.18 94.74 49.7 1
-78.43 33.43 76.28 45 138.55 77.16 1
-63.17 6.33 63 56.84 110.64 42.61 1
-68.61 15.08 63.01 53.53 123.43 39.5 1
-63.9 13.71 62.12 50.19 114.13 41.42 1
-85 29.61 83.35 55.39 126.91 71.32 1
-42.02 -6.55 67.9 48.58 111.59 27.34 1
-69.76 19.28 48.5 50.48 96.49 51.17 1
-80.99 36.84 86.96 44.14 141.09 85.87 1
-129.83 8.4 48.38 121.43 107.69 418.54 1
-70.48 12.49 62.42 57.99 114.19 56.9 1
-86.04 38.75 47.87 47.29 122.09 61.99 1
-65.54 24.16 45.78 41.38 136.44 16.38 1
-60.75 15.75 43.2 45 113.05 31.69 1
-54.74 12.1 41 42.65 117.64 40.38 1
-83.88 23.08 87.14 60.8 124.65 80.56 1
-80.07 48.07 52.4 32.01 110.71 67.73 1
-65.67 10.54 56.49 55.12 109.16 53.93 1
-74.72 14.32 32.5 60.4 107.18 37.02 1
-48.06 5.69 57.06 42.37 95.44 32.84 1
-70.68 21.7 59.18 48.97 103.01 27.81 1
-80.43 17 66.54 63.43 116.44 57.78 1
-90.51 28.27 69.81 62.24 100.89 58.82 1
-77.24 16.74 49.78 60.5 110.69 39.79 1
-50.07 9.12 32.17 40.95 99.71 26.77 1
-69.78 13.78 58 56 118.93 17.91 1
-69.63 21.12 52.77 48.5 116.8 54.82 1
-81.75 20.12 70.56 61.63 119.43 55.51 1
-52.2 17.21 78.09 34.99 136.97 54.94 1
-77.12 30.35 77.48 46.77 110.61 82.09 1
-88.02 39.84 81.77 48.18 116.6 56.77 1
-83.4 34.31 78.42 49.09 110.47 49.67 1
-72.05 24.7 79.87 47.35 107.17 56.43 1
-85.1 21.07 91.73 64.03 109.06 38.03 1
-69.56 15.4 74.44 54.16 105.07 29.7 1
-89.5 48.9 72 40.6 134.63 118.35 1
-85.29 18.28 100.74 67.01 110.66 58.88 1
-60.63 20.6 64.54 40.03 117.23 104.86 1
-60.04 14.31 58.04 45.73 105.13 30.41 1
-85.64 42.69 78.75 42.95 105.14 42.89 1
-85.58 30.46 78.23 55.12 114.87 68.38 1
-55.08 -3.76 56 58.84 109.92 31.77 1
-65.76 9.83 50.82 55.92 104.39 39.31 1
-79.25 23.94 40.8 55.3 98.62 36.71 1
-81.11 20.69 60.69 60.42 94.02 40.51 1
-48.03 3.97 58.34 44.06 125.35 35 1
-63.4 14.12 48.14 49.29 111.92 31.78 1
-57.29 15.15 64 42.14 116.74 30.34 1
-41.19 5.79 42.87 35.39 103.35 27.66 1
-66.8 14.55 72.08 52.25 82.46 41.69 1
-79.48 26.73 70.65 52.74 118.59 61.7 1
-44.22 1.51 46.11 42.71 108.63 42.81 1
-57.04 0.35 49.2 56.69 103.05 52.17 1
-64.27 12.51 68.7 51.77 95.25 39.41 1
-92.03 35.39 77.42 56.63 115.72 58.06 1
-67.26 7.19 51.7 60.07 97.8 42.14 1
-118.14 38.45 50.84 79.7 81.02 74.04 1
-115.92 37.52 76.8 78.41 104.7 81.2 1
-53.94 9.31 43.1 44.64 124.4 25.08 1
-83.7 20.27 77.11 63.43 125.48 69.28 1
-56.99 6.87 57.01 50.12 109.98 36.81 1
-72.34 16.42 59.87 55.92 70.08 12.07 1
-95.38 24.82 95.16 70.56 89.31 57.66 1
-44.25 1.1 38 43.15 98.27 23.91 1
-64.81 15.17 58.84 49.64 111.68 21.41 1
-78.4 14.04 79.69 64.36 104.73 12.39 1
-56.67 13.46 43.77 43.21 93.69 21.11 1
-50.83 9.06 56.3 41.76 79 23.04 1
-61.41 25.38 39.1 36.03 103.4 21.84 1
-56.56 8.96 52.58 47.6 98.78 50.7 1
-67.03 13.28 66.15 53.75 100.72 33.99 1
-80.82 19.24 61.64 61.58 89.47 44.17 1
-80.65 26.34 60.9 54.31 120.1 52.47 1
-68.72 49.43 68.06 19.29 125.02 54.69 1
-37.9 4.48 24.71 33.42 157.85 33.61 1
-64.62 15.23 67.63 49.4 90.3 31.33 1
-75.44 31.54 89.6 43.9 106.83 54.97 1
-71 37.52 84.54 33.49 125.16 67.77 1
-81.06 20.8 91.78 60.26 125.43 38.18 1
-91.47 24.51 84.62 66.96 117.31 52.62 1
-81.08 21.26 78.77 59.83 90.07 49.16 1
-60.42 5.27 59.81 55.15 109.03 30.27 1
-85.68 38.65 82.68 47.03 120.84 61.96 1
-82.41 29.28 77.05 53.13 117.04 62.77 1
-43.72 9.81 52 33.91 88.43 40.88 1
-86.47 40.3 61.14 46.17 97.4 55.75 1
-74.47 33.28 66.94 41.19 146.47 124.98 1
-70.25 10.34 76.37 59.91 119.24 32.67 1
-72.64 18.93 68 53.71 116.96 25.38 1
-71.24 5.27 86 65.97 110.7 38.26 1
-63.77 12.76 65.36 51.01 89.82 56 1
-58.83 37.58 125.74 21.25 135.63 117.31 1
-74.85 13.91 62.69 60.95 115.21 33.17 1
-75.3 16.67 61.3 58.63 118.88 31.58 1
-63.36 20.02 67.5 43.34 131 37.56 1
-67.51 33.28 96.28 34.24 145.6 88.3 1
-76.31 41.93 93.28 34.38 132.27 101.22 1
-73.64 9.71 63 63.92 98.73 26.98 1
-56.54 14.38 44.99 42.16 101.72 25.77 1
-80.11 33.94 85.1 46.17 125.59 100.29 1
-95.48 46.55 59 48.93 96.68 77.28 1
-74.09 18.82 76.03 55.27 128.41 73.39 1
-87.68 20.37 93.82 67.31 120.94 76.73 1
-48.26 16.42 36.33 31.84 94.88 28.34 1
-38.51 16.96 35.11 21.54 127.63 7.99 -1
-54.92 18.97 51.6 35.95 125.85 2 -1
-44.36 8.95 46.9 35.42 129.22 4.99 -1
-48.32 17.45 48 30.87 128.98 -0.91 -1
-45.7 10.66 42.58 35.04 130.18 -3.39 -1
-30.74 13.35 35.9 17.39 142.41 -2.01 -1
-50.91 6.68 30.9 44.24 118.15 -1.06 -1
-38.13 6.56 50.45 31.57 132.11 6.34 -1
-51.62 15.97 35 35.66 129.39 1.01 -1
-64.31 26.33 50.96 37.98 106.18 3.12 -1
-44.49 21.79 31.47 22.7 113.78 -0.28 -1
-54.95 5.87 53 49.09 126.97 -0.63 -1
-56.1 13.11 62.64 43 116.23 31.17 -1
-69.4 18.9 75.97 50.5 103.58 -0.44 -1
-89.83 22.64 90.56 67.2 100.5 3.04 -1
-59.73 7.72 55.34 52 125.17 3.24 -1
-63.96 16.06 63.12 47.9 142.36 6.3 -1
-61.54 19.68 52.89 41.86 118.69 4.82 -1
-38.05 8.3 26.24 29.74 123.8 3.89 -1
-43.44 10.1 36.03 33.34 137.44 -3.11 -1
-65.61 23.14 62.58 42.47 124.13 -4.08 -1
-53.91 12.94 39 40.97 118.19 5.07 -1
-43.12 13.82 40.35 29.3 128.52 0.97 -1
-40.68 9.15 31.02 31.53 139.12 -2.51 -1
-37.73 9.39 42 28.35 135.74 13.68 -1
-63.93 19.97 40.18 43.96 113.07 -11.06 -1
-61.82 13.6 64 48.22 121.78 1.3 -1
-62.14 13.96 58 48.18 133.28 4.96 -1
-69 13.29 55.57 55.71 126.61 10.83 -1
-56.45 19.44 43.58 37 139.19 -1.86 -1
-41.65 8.84 36.03 32.81 116.56 -6.05 -1
-51.53 13.52 35 38.01 126.72 13.93 -1
-39.09 5.54 26.93 33.55 131.58 -0.76 -1
-34.65 7.51 43 27.14 123.99 -4.08 -1
-63.03 27.34 51.61 35.69 114.51 7.44 -1
-47.81 10.69 54 37.12 125.39 -0.4 -1
-46.64 15.85 40 30.78 119.38 9.06 -1
-49.83 16.74 28 33.09 121.44 1.91 -1
-47.32 8.57 35.56 38.75 120.58 1.63 -1
-50.75 20.24 37 30.52 122.34 2.29 -1
-36.16 -0.81 33.63 36.97 135.94 -2.09 -1
-40.75 1.84 50 38.91 139.25 0.67 -1
-42.92 -5.85 58 48.76 121.61 -3.36 -1
-63.79 21.35 66 42.45 119.55 12.38 -1
-72.96 19.58 61.01 53.38 111.23 0.81 -1
-67.54 14.66 58 52.88 123.63 25.97 -1
-54.75 9.75 48 45 123.04 8.24 -1
-50.16 -2.97 42 53.13 131.8 -8.29 -1
-40.35 10.19 37.97 30.15 128.01 0.46 -1
-63.62 16.93 49.35 46.68 117.09 -0.36 -1
-54.14 11.94 43 42.21 122.21 0.15 -1
-74.98 14.92 53.73 60.05 105.65 1.59 -1
-42.52 14.38 25.32 28.14 128.91 0.76 -1
-33.79 3.68 25.5 30.11 128.33 -1.78 -1
-54.5 6.82 47 47.68 111.79 -4.41 -1
-48.17 9.59 39.71 38.58 135.62 5.36 -1
-46.37 10.22 42.7 36.16 121.25 -0.54 -1
-52.86 9.41 46.99 43.45 123.09 1.86 -1
-57.15 16.49 42.84 40.66 113.81 5.02 -1
-37.14 16.48 24 20.66 125.01 7.37 -1
-51.31 8.88 57 42.44 126.47 -2.14 -1
-42.52 16.54 42 25.97 120.63 7.88 -1
-39.36 7.01 37 32.35 117.82 1.9 -1
-35.88 1.11 43.46 34.77 126.92 -1.63 -1
-43.19 9.98 28.94 33.22 123.47 1.74 -1
-67.29 16.72 51 50.57 137.59 4.96 -1
-51.33 13.63 33.26 37.69 131.31 1.79 -1
-65.76 13.21 44 52.55 129.39 -1.98 -1
-40.41 -1.33 30.98 41.74 119.34 -6.17 -1
-48.8 18.02 52 30.78 139.15 10.44 -1
-50.09 13.43 34.46 36.66 119.13 3.09 -1
-64.26 14.5 43.9 49.76 115.39 5.95 -1
-53.68 13.45 41.58 40.24 113.91 2.74 -1
-49 13.11 51.87 35.88 126.4 0.54 -1
-59.17 14.56 43.2 44.6 121.04 2.83 -1
-67.8 16.55 43.26 51.25 119.69 4.87 -1
-61.73 17.11 46.9 44.62 120.92 3.09 -1
-33.04 -0.32 19.07 33.37 120.39 9.35 -1
-74.57 15.72 58.62 58.84 105.42 0.6 -1
-44.43 14.17 32.24 30.26 131.72 -3.6 -1
-36.42 13.88 20.24 22.54 126.08 0.18 -1
-51.08 14.21 35.95 36.87 115.8 6.91 -1
-34.76 2.63 29.5 32.12 127.14 -0.46 -1
-48.9 5.59 55.5 43.32 137.11 19.85 -1
-46.24 10.06 37 36.17 128.06 -5.1 -1
-46.43 6.62 48.1 39.81 130.35 2.45 -1
-39.66 16.21 36.67 23.45 131.92 -4.97 -1
-45.58 18.76 33.77 26.82 116.8 3.13 -1
-66.51 20.9 31.73 45.61 128.9 1.52 -1
-82.91 29.89 58.25 53.01 110.71 6.08 -1
-50.68 6.46 35 44.22 116.59 -0.21 -1
-89.01 26.08 69.02 62.94 111.48 6.06 -1
-54.6 21.49 29.36 33.11 118.34 -1.47 -1
-34.38 2.06 32.39 32.32 128.3 -3.37 -1
-45.08 12.31 44.58 32.77 147.89 -8.94 -1
-47.9 13.62 36 34.29 117.45 -4.25 -1
-53.94 20.72 29.22 33.22 114.37 -0.42 -1
-61.45 22.69 46.17 38.75 125.67 -2.71 -1
-45.25 8.69 41.58 36.56 118.55 0.21 -1
-33.84 5.07 36.64 28.77 123.95 -0.2 -1

GPy/util/datasets/connections.txt

@@ -1,22 +0,0 @@
-LFHD, RFHD
-RFHD, RBHD
-RBHD, LBHD
-LBHD, LFHD
-LELB, LWRB
-LWRB, LFIN
-LELB, LSHO
-LSHO, RSHO
-RSHO, STRN
-LSHO, STRN
-RSHO, RELB
-RELB, RWRB
-RWRB, RFIN
-LSHO, LFWT
-RSHO, RFWT
-LFWT, RFWT
-LFWT, LKNE
-RFWT, RKNE
-LKNE, LHEE
-RKNE, RHEE
-RMT5, RHEE
-LMT5, LHEE

GPy/util/datasets/data_resources_create.py

@@ -1,168 +0,0 @@
-import json
-
-neil_url = 'http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/'
-sam_url = 'http://www.cs.nyu.edu/~roweis/data/'
-cmu_url = 'http://mocap.cs.cmu.edu/subjects/'
-
-data_resources = {'ankur_pose_data' : {'urls' : [neil_url + 'ankur_pose_data/'],
-                                       'files' : [['ankurDataPoseSilhouette.mat']],
-                                       'license' : None,
-                                       'citation' : """3D Human Pose from Silhouettes by Relevance Vector Regression (In CVPR'04). A. Agarwal and B. Triggs.""",
-                                       'details' : """Artificially generated data of silhouettes given poses. Note that the data does not display a left/right ambiguity because across the entire data set one of the arms sticks out more the the other, disambiguating the pose as to which way the individual is facing."""},
-
-                  'boston_housing' : {'urls' : ['http://archive.ics.uci.edu/ml/machine-learning-databases/housing/'],
-                                      'files' : [['Index', 'housing.data', 'housing.names']],
-                                      'citation' : """Harrison, D. and Rubinfeld, D.L. 'Hedonic prices and the demand for clean air', J. Environ. Economics & Management, vol.5, 81-102, 1978.""",
-                                      'details' : """The Boston Housing data relates house values in Boston to a range of input variables.""",
-                                      'license' : None,
-                                      'size' : 51276
-                                      },
-                  'brendan_faces' : {'urls' : [sam_url],
-                                     'files': [['frey_rawface.mat']],
-                                     'citation' : 'Frey, B. J., Colmenarez, A and Huang, T. S. Mixtures of Local Linear Subspaces for Face Recognition. Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition 1998, 32-37, June 1998. Computer Society Press, Los Alamitos, CA.',
-                                     'details' : """A video of Brendan Frey's face popularized as a benchmark for visualization by the Locally Linear Embedding.""",
-                                     'license': None,
-                                     'size' : 1100584},
-                  'cmu_mocap_full' : {'urls' : ['http://mocap.cs.cmu.edu'],
-                                      'files' : [['allasfamc.zip']],
-                                      'citation' : """Please include this in your acknowledgements: The data used in this project was obtained from mocap.cs.cmu.edu.'
-                                      'The database was created with funding from NSF EIA-0196217.""",
-                                      'details' : """CMU Motion Capture data base. Captured by a Vicon motion capture system consisting of 12 infrared MX-40 cameras, each of which is capable of recording at 120 Hz with images of 4 megapixel resolution. Motions are captured in a working volume of approximately 3m x 8m. The capture subject wears 41 markers and a stylish black garment.""",
-                                      'license' : """From http://mocap.cs.cmu.edu. This data is free for use in research projects. You may include this data in commercially-sold products, but you may not resell this data directly, even in converted form. If you publish results obtained using this data, we would appreciate it if you would send the citation to your published paper to jkh+mocap@cs.cmu.edu, and also would add this text to your acknowledgments section: The data used in this project was obtained from mocap.cs.cmu.edu. The database was created with funding from NSF EIA-0196217.""",
-                                      'size' : None},
-                  'creep_rupture' : {'urls' : ['http://www.msm.cam.ac.uk/map/data/tar/'],
-                                     'files' : [['creeprupt.tar']],
-                                     'citation' : 'Materials Algorithms Project Data Library: MAP_DATA_CREEP_RUPTURE. F. Brun and T. Yoshida.',
-                                     'details' : """Provides 2066 creep rupture test results of steels (mainly of two kinds of steels: 2.25Cr and 9-12 wt% Cr ferritic steels). See http://www.msm.cam.ac.uk/map/data/materials/creeprupt-b.html.""",
-                                     'license' : None,
-                                     'size' : 602797},
-                  'della_gatta' : {'urls' : [neil_url + 'della_gatta/'],
-                                   'files': [['DellaGattadata.mat']],
-                                   'citation' : 'Direct targets of the TRP63 transcription factor revealed by a combination of gene expression profiling and reverse engineering. Giusy Della Gatta, Mukesh Bansal, Alberto Ambesi-Impiombato, Dario Antonini, Caterina Missero, and Diego di Bernardo, Genome Research 2008',
-                                   'details': "The full gene expression data set from della Gatta et al (http://www.ncbi.nlm.nih.gov/pmc/articles/PMC2413161/) processed by RMA.",
-                                   'license':None,
-                                   'size':3729650},
-                  'epomeo_gpx' : {'urls' : [neil_url + 'epomeo_gpx/'],
-                                   'files': [['endomondo_1.gpx', 'endomondo_2.gpx', 'garmin_watch_via_endomondo.gpx','viewranger_phone.gpx','viewranger_tablet.gpx']],
-                                   'citation' : '',
-                                   'details': "Five different GPS traces of the same run up Mount Epomeo in Ischia. The traces are from different sources. endomondo_1 and endomondo_2 are traces from the mobile phone app Endomondo, with a split in the middle. garmin_watch_via_endomondo is the trace from a Garmin watch, with a segment missing about 4 kilometers in. viewranger_phone and viewranger_tablet are traces from a phone and a tablet through the viewranger app. The viewranger_phone data comes from the same mobile phone as the Endomondo data (i.e. there are 3 GPS devices, but one device recorded two traces).",
-                                   'license':None,
-                                   'size': 2031872},
-                  'three_phase_oil_flow': {'urls' : [neil_url + 'three_phase_oil_flow/'],
-                                           'files' : [['DataTrnLbls.txt', 'DataTrn.txt', 'DataTst.txt', 'DataTstLbls.txt', 'DataVdn.txt', 'DataVdnLbls.txt']],
-                                           'citation' : 'Bishop, C. M. and G. D. James (1993). Analysis of multiphase flows using dual-energy gamma densitometry and neural networks. Nuclear Instruments and Methods in Physics Research A327, 580-593',
-                                           'details' : """The three phase oil data used initially for demonstrating the Generative Topographic mapping.""",
-                                           'license' : None,
-                                           'size' : 712796},
-                  'rogers_girolami_data' : {'urls' : ['https://www.dropbox.com/sh/7p6tu1t29idgliq/_XqlH_3nt9/'],
-                                            'files' : [['firstcoursemldata.tar.gz']],
-                                            'suffices' : [['?dl=1']],
-                                            'citation' : 'A First Course in Machine Learning. Simon Rogers and Mark Girolami: Chapman & Hall/CRC, ISBN-13: 978-1439824146',
-                                            'details' : """Data from the textbook 'A First Course in Machine Learning'. Available from http://www.dcs.gla.ac.uk/~srogers/firstcourseml/.""",
-                                            'license' : None,
-                                            'size' : 21949154},
-                  'olivetti_faces' : {'urls' : [neil_url + 'olivetti_faces/', sam_url],
-                                      'files' : [['att_faces.zip'], ['olivettifaces.mat']],
-                                            'citation' : 'Ferdinando Samaria and Andy Harter, Parameterisation of a Stochastic Model for Human Face Identification. Proceedings of 2nd IEEE Workshop on Applications of Computer Vision, Sarasota FL, December 1994',
-                                            'details' : """Olivetti Research Labs Face data base, acquired between December 1992 and December 1994 in the Olivetti Research Lab, Cambridge (which later became AT&T Laboratories, Cambridge). When using these images please give credit to AT&T Laboratories, Cambridge. """,
-                                            'license': None,
-                                            'size' : 8561331},
-                  'olympic_marathon_men' : {'urls' : [neil_url + 'olympic_marathon_men/'],
-                                            'files' : [['olympicMarathonTimes.csv']],
-                                            'citation' : None,
-                                            'details' : """Olympic mens' marathon gold medal winning times from 1896 to 2012. Time given in pace (minutes per kilometer). Data is originally downloaded and collated from Wikipedia, we are not responsible for errors in the data""",
-                                            'license': None,
-                                            'size' : 584},
-                  'osu_run1' : {'urls': ['http://accad.osu.edu/research/mocap/data/', neil_url + 'stick/'],
-                                'files': [['run1TXT.ZIP'],['connections.txt']],
-                                'details' : "Motion capture data of a stick man running from the Open Motion Data Project at Ohio State University.",
-                                'citation' : 'The Open Motion Data Project by The Ohio State University Advanced Computing Center for the Arts and Design, http://accad.osu.edu/research/mocap/mocap_data.htm.',
-                                'license' : 'Data is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License (http://creativecommons.org/licenses/by-nc-sa/3.0/).',
-                                'size': 338103},
-                  'osu_accad' : {'urls': ['http://accad.osu.edu/research/mocap/data/', neil_url + 'stick/'],
-                                'files': [['swagger1TXT.ZIP','handspring1TXT.ZIP','quickwalkTXT.ZIP','run1TXT.ZIP','sprintTXT.ZIP','dogwalkTXT.ZIP','camper_04TXT.ZIP','dance_KB3_TXT.ZIP','per20_TXT.ZIP','perTWO07_TXT.ZIP','perTWO13_TXT.ZIP','perTWO14_TXT.ZIP','perTWO15_TXT.ZIP','perTWO16_TXT.ZIP'],['connections.txt']],
-                                'details' : "Motion capture data of different motions from the Open Motion Data Project at Ohio State University.",
-                                'citation' : 'The Open Motion Data Project by The Ohio State University Advanced Computing Center for the Arts and Design, http://accad.osu.edu/research/mocap/mocap_data.htm.',
-                                'license' : 'Data is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License (http://creativecommons.org/licenses/by-nc-sa/3.0/).',
-                                'size': 15922790},
-                  'pumadyn-32nm' : {'urls' : ['ftp://ftp.cs.toronto.edu/pub/neuron/delve/data/tarfiles/pumadyn-family/'],
-                                    'files' : [['pumadyn-32nm.tar.gz']],
-                                    'details' : """Pumadyn non linear 32 input data set with moderate noise. See http://www.cs.utoronto.ca/~delve/data/pumadyn/desc.html for details.""",
-                                    'citation' : """Created by Zoubin Ghahramani using the Matlab Robotics Toolbox of Peter Corke. Corke, P. I. (1996). A Robotics Toolbox for MATLAB. IEEE Robotics and Automation Magazine, 3 (1): 24-32.""",
-                                    'license' : """Data is made available by the Delve system at the University of Toronto""",
-                                    'size' : 5861646},
-                  'robot_wireless' : {'urls' : [neil_url + 'robot_wireless/'],
-                                      'files' : [['uw-floor.txt']],
-                                      'citation' : """WiFi-SLAM using Gaussian Process Latent Variable Models by Brian Ferris, Dieter Fox and Neil Lawrence in IJCAI'07 Proceedings pages 2480-2485. Data used in A Unifying Probabilistic Perspective for Spectral Dimensionality Reduction: Insights and New Models by Neil D. Lawrence, JMLR 13 pg 1609--1638, 2012.""",
-                                      'details' : """Data created by Brian Ferris and Dieter Fox. Consists of WiFi access point strengths taken during a circuit of the Paul Allen building at the University of Washington.""",
-                                      'license' : None,
-                                      'size' : 284390},
-                  'swiss_roll' : {'urls' : ['http://isomap.stanford.edu/'],
-                                  'files' : [['swiss_roll_data.mat']],
-                                  'details' : """Swiss roll data made available by Tenenbaum, de Silva and Langford to demonstrate isomap, available from http://isomap.stanford.edu/datasets.html.""",
-                                  'citation' : 'A Global Geometric Framework for Nonlinear Dimensionality Reduction, J. B. Tenenbaum, V. de Silva and J. C. Langford, Science 290 (5500): 2319-2323, 22 December 2000',
-                                  'license' : None,
-                                  'size' : 800256},
-                  'ripley_prnn_data' : {'urls' : ['http://www.stats.ox.ac.uk/pub/PRNN/'],
-                                        'files' : [['Cushings.dat', 'README', 'crabs.dat', 'fglass.dat', 'fglass.grp', 'pima.te', 'pima.tr', 'pima.tr2', 'synth.te', 'synth.tr', 'viruses.dat', 'virus3.dat']],
-                                        'details' : """Data sets from Brian Ripley's Pattern Recognition and Neural Networks""",
-                                        'citation': """Pattern Recognition and Neural Networks by B.D. Ripley (1996) Cambridge University Press ISBN 0 521 46986 7""",
-                                        'license' : None,
-                                        'size' : 93565},
-                  'isomap_face_data' : {'urls' : [neil_url + 'isomap_face_data/'],
-                                        'files' : [['face_data.mat']],
-                                        'details' : """Face data made available by Tenenbaum, de Silva and Langford to demonstrate isomap, available from http://isomap.stanford.edu/datasets.html.""",
-                                        'citation' : 'A Global Geometric Framework for Nonlinear Dimensionality Reduction, J. B. Tenenbaum, V. de Silva and J. C. Langford, Science 290 (5500): 2319-2323, 22 December 2000',
-                                        'license' : None,
-                                        'size' : 24229368},
-                  'xw_pen' : {'urls' : [neil_url + 'xw_pen/'],
-                                        'files' : [['xw_pen_15.csv']],
-                                        'details' : """Accelerometer pen data used for robust regression by Tipping and Lawrence.""",
-                                        'citation' : 'Michael E. Tipping and Neil D. Lawrence. Variational inference for Student-t models: Robust Bayesian interpolation and generalised component analysis. Neurocomputing, 69:123--141, 2005',
-                                        'license' : None,
-                                        'size' : 3410},
-                  'hapmap3' : {'urls' : ['http://hapmap.ncbi.nlm.nih.gov/downloads/genotypes/latest_phaseIII_ncbi_b36/plink_format/'],
-                                 'files' : [['hapmap3_r2_b36_fwd.consensus.qc.poly.map.bz2', 'hapmap3_r2_b36_fwd.consensus.qc.poly.ped.bz2', 'relationships_w_pops_121708.txt']],
-                                 'details' : """
-        HapMap Project: Single Nucleotide Polymorphism sequenced in all human populations. 
-        The HapMap phase three SNP dataset - 1184 samples out of 11 populations.
-        See http://www.nature.com/nature/journal/v426/n6968/abs/nature02168.html for details.
-
-        SNP_matrix (A) encoding [see Paschou et all. 2007 (PCA-Correlated SNPs...)]:
-        Let (B1,B2) be the alphabetically sorted bases, which occur in the j-th SNP, then
-
-              /  1, iff SNPij==(B1,B1)
-        Aij = |  0, iff SNPij==(B1,B2)
-              \ -1, iff SNPij==(B2,B2)
-
-        The SNP data and the meta information (such as iid, sex and phenotype) are
-        stored in the dataframe datadf, index is the Individual ID, 
-        with following columns for metainfo:
-
-            * family_id   -> Family ID
-            * paternal_id -> Paternal ID
-            * maternal_id -> Maternal ID
-            * sex         -> Sex (1=male; 2=female; other=unknown)
-            * phenotype   -> Phenotype (-9, or 0 for unknown)
-            * population  -> Population string (e.g. 'ASW' - 'YRI')
-            * rest are SNP rs (ids)
-
-        More information is given in infodf:
-
-            * Chromosome:
-                - autosomal chromosemes                -> 1-22
-                - X    X chromosome                    -> 23
-                - Y    Y chromosome                    -> 24
-                - XY   Pseudo-autosomal region of X    -> 25
-                - MT   Mitochondrial                   -> 26
-            * Relative Positon (to Chromosome) [base pairs]
-
-        """,
-                                 'citation': """Gibbs, Richard A., et al. "The international HapMap project." Nature 426.6968 (2003): 789-796.""",
-                                 'license' : """International HapMap Project Public Access License (http://hapmap.ncbi.nlm.nih.gov/cgi-perl/registration#licence)""",
-                                 'size' : 2*1729092237 + 62265},
-                  }
-
-with open('data_resources.json', 'w') as f:
-    print "writing data_resources"
-    json.dump(data_resources, f)

GPy/util/datasets/diabetes.txt

@@ -1,769 +0,0 @@
-6,148,72,35,0,33.6,0.627,50,1
-1,85,66,29,0,26.6,0.351,31,0
-8,183,64,0,0,23.3,0.672,32,1
-1,89,66,23,94,28.1,0.167,21,0
-0,137,40,35,168,43.1,2.288,33,1
-5,116,74,0,0,25.6,0.201,30,0
-3,78,50,32,88,31.0,0.248,26,1
-10,115,0,0,0,35.3,0.134,29,0
-2,197,70,45,543,30.5,0.158,53,1
-8,125,96,0,0,0.0,0.232,54,1
-4,110,92,0,0,37.6,0.191,30,0
-10,168,74,0,0,38.0,0.537,34,1
-10,139,80,0,0,27.1,1.441,57,0
-1,189,60,23,846,30.1,0.398,59,1
-5,166,72,19,175,25.8,0.587,51,1
-7,100,0,0,0,30.0,0.484,32,1
-0,118,84,47,230,45.8,0.551,31,1
-7,107,74,0,0,29.6,0.254,31,1
-1,103,30,38,83,43.3,0.183,33,0
-1,115,70,30,96,34.6,0.529,32,1
-3,126,88,41,235,39.3,0.704,27,0
-8,99,84,0,0,35.4,0.388,50,0
-7,196,90,0,0,39.8,0.451,41,1
-9,119,80,35,0,29.0,0.263,29,1
-11,143,94,33,146,36.6,0.254,51,1
-10,125,70,26,115,31.1,0.205,41,1
-7,147,76,0,0,39.4,0.257,43,1
-1,97,66,15,140,23.2,0.487,22,0
-13,145,82,19,110,22.2,0.245,57,0
-5,117,92,0,0,34.1,0.337,38,0
-5,109,75,26,0,36.0,0.546,60,0
-3,158,76,36,245,31.6,0.851,28,1
-3,88,58,11,54,24.8,0.267,22,0
-6,92,92,0,0,19.9,0.188,28,0
-10,122,78,31,0,27.6,0.512,45,0
-4,103,60,33,192,24.0,0.966,33,0
-11,138,76,0,0,33.2,0.420,35,0
-9,102,76,37,0,32.9,0.665,46,1
-2,90,68,42,0,38.2,0.503,27,1
-4,111,72,47,207,37.1,1.390,56,1
-3,180,64,25,70,34.0,0.271,26,0
-7,133,84,0,0,40.2,0.696,37,0
-7,106,92,18,0,22.7,0.235,48,0
-9,171,110,24,240,45.4,0.721,54,1
-7,159,64,0,0,27.4,0.294,40,0
-0,180,66,39,0,42.0,1.893,25,1
-1,146,56,0,0,29.7,0.564,29,0
-2,71,70,27,0,28.0,0.586,22,0
-7,103,66,32,0,39.1,0.344,31,1
-7,105,0,0,0,0.0,0.305,24,0
-1,103,80,11,82,19.4,0.491,22,0
-1,101,50,15,36,24.2,0.526,26,0
-5,88,66,21,23,24.4,0.342,30,0
-8,176,90,34,300,33.7,0.467,58,1
-7,150,66,42,342,34.7,0.718,42,0
-1,73,50,10,0,23.0,0.248,21,0
-7,187,68,39,304,37.7,0.254,41,1
-0,100,88,60,110,46.8,0.962,31,0
-0,146,82,0,0,40.5,1.781,44,0
-0,105,64,41,142,41.5,0.173,22,0
-2,84,0,0,0,0.0,0.304,21,0
-8,133,72,0,0,32.9,0.270,39,1
-5,44,62,0,0,25.0,0.587,36,0
-2,141,58,34,128,25.4,0.699,24,0
-7,114,66,0,0,32.8,0.258,42,1
-5,99,74,27,0,29.0,0.203,32,0
-0,109,88,30,0,32.5,0.855,38,1
-2,109,92,0,0,42.7,0.845,54,0
-1,95,66,13,38,19.6,0.334,25,0
-4,146,85,27,100,28.9,0.189,27,0
-2,100,66,20,90,32.9,0.867,28,1
-5,139,64,35,140,28.6,0.411,26,0
-13,126,90,0,0,43.4,0.583,42,1
-4,129,86,20,270,35.1,0.231,23,0
-1,79,75,30,0,32.0,0.396,22,0
-1,0,48,20,0,24.7,0.140,22,0
-7,62,78,0,0,32.6,0.391,41,0
-5,95,72,33,0,37.7,0.370,27,0
-0,131,0,0,0,43.2,0.270,26,1
-2,112,66,22,0,25.0,0.307,24,0
-3,113,44,13,0,22.4,0.140,22,0
-2,74,0,0,0,0.0,0.102,22,0
-7,83,78,26,71,29.3,0.767,36,0
-0,101,65,28,0,24.6,0.237,22,0
-5,137,108,0,0,48.8,0.227,37,1
-2,110,74,29,125,32.4,0.698,27,0
-13,106,72,54,0,36.6,0.178,45,0
-2,100,68,25,71,38.5,0.324,26,0
-15,136,70,32,110,37.1,0.153,43,1
-1,107,68,19,0,26.5,0.165,24,0
-1,80,55,0,0,19.1,0.258,21,0
-4,123,80,15,176,32.0,0.443,34,0
-7,81,78,40,48,46.7,0.261,42,0
-4,134,72,0,0,23.8,0.277,60,1
-2,142,82,18,64,24.7,0.761,21,0
-6,144,72,27,228,33.9,0.255,40,0
-2,92,62,28,0,31.6,0.130,24,0
-1,71,48,18,76,20.4,0.323,22,0
-6,93,50,30,64,28.7,0.356,23,0
-1,122,90,51,220,49.7,0.325,31,1
-1,163,72,0,0,39.0,1.222,33,1
-1,151,60,0,0,26.1,0.179,22,0
-0,125,96,0,0,22.5,0.262,21,0
-1,81,72,18,40,26.6,0.283,24,0
-2,85,65,0,0,39.6,0.930,27,0
-1,126,56,29,152,28.7,0.801,21,0
-1,96,122,0,0,22.4,0.207,27,0
-4,144,58,28,140,29.5,0.287,37,0
-3,83,58,31,18,34.3,0.336,25,0
-0,95,85,25,36,37.4,0.247,24,1
-3,171,72,33,135,33.3,0.199,24,1
-8,155,62,26,495,34.0,0.543,46,1
-1,89,76,34,37,31.2,0.192,23,0
-4,76,62,0,0,34.0,0.391,25,0
-7,160,54,32,175,30.5,0.588,39,1
-4,146,92,0,0,31.2,0.539,61,1
-5,124,74,0,0,34.0,0.220,38,1
-5,78,48,0,0,33.7,0.654,25,0
-4,97,60,23,0,28.2,0.443,22,0
-4,99,76,15,51,23.2,0.223,21,0
-0,162,76,56,100,53.2,0.759,25,1
-6,111,64,39,0,34.2,0.260,24,0
-2,107,74,30,100,33.6,0.404,23,0
-5,132,80,0,0,26.8,0.186,69,0
-0,113,76,0,0,33.3,0.278,23,1
-1,88,30,42,99,55.0,0.496,26,1
-3,120,70,30,135,42.9,0.452,30,0
-1,118,58,36,94,33.3,0.261,23,0
-1,117,88,24,145,34.5,0.403,40,1
-0,105,84,0,0,27.9,0.741,62,1
-4,173,70,14,168,29.7,0.361,33,1
-9,122,56,0,0,33.3,1.114,33,1
-3,170,64,37,225,34.5,0.356,30,1
-8,84,74,31,0,38.3,0.457,39,0
-2,96,68,13,49,21.1,0.647,26,0
-2,125,60,20,140,33.8,0.088,31,0
-0,100,70,26,50,30.8,0.597,21,0
-0,93,60,25,92,28.7,0.532,22,0
-0,129,80,0,0,31.2,0.703,29,0
-5,105,72,29,325,36.9,0.159,28,0
-3,128,78,0,0,21.1,0.268,55,0
-5,106,82,30,0,39.5,0.286,38,0
-2,108,52,26,63,32.5,0.318,22,0
-10,108,66,0,0,32.4,0.272,42,1
-4,154,62,31,284,32.8,0.237,23,0
-0,102,75,23,0,0.0,0.572,21,0
-9,57,80,37,0,32.8,0.096,41,0
-2,106,64,35,119,30.5,1.400,34,0
-5,147,78,0,0,33.7,0.218,65,0
-2,90,70,17,0,27.3,0.085,22,0
-1,136,74,50,204,37.4,0.399,24,0
-4,114,65,0,0,21.9,0.432,37,0
-9,156,86,28,155,34.3,1.189,42,1
-1,153,82,42,485,40.6,0.687,23,0
-8,188,78,0,0,47.9,0.137,43,1
-7,152,88,44,0,50.0,0.337,36,1
-2,99,52,15,94,24.6,0.637,21,0
-1,109,56,21,135,25.2,0.833,23,0
-2,88,74,19,53,29.0,0.229,22,0
-17,163,72,41,114,40.9,0.817,47,1
-4,151,90,38,0,29.7,0.294,36,0
-7,102,74,40,105,37.2,0.204,45,0
-0,114,80,34,285,44.2,0.167,27,0
-2,100,64,23,0,29.7,0.368,21,0
-0,131,88,0,0,31.6,0.743,32,1
-6,104,74,18,156,29.9,0.722,41,1
-3,148,66,25,0,32.5,0.256,22,0
-4,120,68,0,0,29.6,0.709,34,0
-4,110,66,0,0,31.9,0.471,29,0
-3,111,90,12,78,28.4,0.495,29,0
-6,102,82,0,0,30.8,0.180,36,1
-6,134,70,23,130,35.4,0.542,29,1
-2,87,0,23,0,28.9,0.773,25,0
-1,79,60,42,48,43.5,0.678,23,0
-2,75,64,24,55,29.7,0.370,33,0
-8,179,72,42,130,32.7,0.719,36,1
-6,85,78,0,0,31.2,0.382,42,0
-0,129,110,46,130,67.1,0.319,26,1
-5,143,78,0,0,45.0,0.190,47,0
-5,130,82,0,0,39.1,0.956,37,1
-6,87,80,0,0,23.2,0.084,32,0
-0,119,64,18,92,34.9,0.725,23,0
-1,0,74,20,23,27.7,0.299,21,0
-5,73,60,0,0,26.8,0.268,27,0
-4,141,74,0,0,27.6,0.244,40,0
-7,194,68,28,0,35.9,0.745,41,1
-8,181,68,36,495,30.1,0.615,60,1
-1,128,98,41,58,32.0,1.321,33,1
-8,109,76,39,114,27.9,0.640,31,1
-5,139,80,35,160,31.6,0.361,25,1
-3,111,62,0,0,22.6,0.142,21,0
-9,123,70,44,94,33.1,0.374,40,0
-7,159,66,0,0,30.4,0.383,36,1
-11,135,0,0,0,52.3,0.578,40,1
-8,85,55,20,0,24.4,0.136,42,0
-5,158,84,41,210,39.4,0.395,29,1
-1,105,58,0,0,24.3,0.187,21,0
-3,107,62,13,48,22.9,0.678,23,1
-4,109,64,44,99,34.8,0.905,26,1
-4,148,60,27,318,30.9,0.150,29,1
-0,113,80,16,0,31.0,0.874,21,0
-1,138,82,0,0,40.1,0.236,28,0
-0,108,68,20,0,27.3,0.787,32,0
-2,99,70,16,44,20.4,0.235,27,0
-6,103,72,32,190,37.7,0.324,55,0
-5,111,72,28,0,23.9,0.407,27,0
-8,196,76,29,280,37.5,0.605,57,1
-5,162,104,0,0,37.7,0.151,52,1
-1,96,64,27,87,33.2,0.289,21,0
-7,184,84,33,0,35.5,0.355,41,1
-2,81,60,22,0,27.7,0.290,25,0
-0,147,85,54,0,42.8,0.375,24,0
-7,179,95,31,0,34.2,0.164,60,0
-0,140,65,26,130,42.6,0.431,24,1
-9,112,82,32,175,34.2,0.260,36,1
-12,151,70,40,271,41.8,0.742,38,1
-5,109,62,41,129,35.8,0.514,25,1
-6,125,68,30,120,30.0,0.464,32,0
-5,85,74,22,0,29.0,1.224,32,1
-5,112,66,0,0,37.8,0.261,41,1
-0,177,60,29,478,34.6,1.072,21,1
-2,158,90,0,0,31.6,0.805,66,1
-7,119,0,0,0,25.2,0.209,37,0
-7,142,60,33,190,28.8,0.687,61,0
-1,100,66,15,56,23.6,0.666,26,0
-1,87,78,27,32,34.6,0.101,22,0
-0,101,76,0,0,35.7,0.198,26,0
-3,162,52,38,0,37.2,0.652,24,1
-4,197,70,39,744,36.7,2.329,31,0
-0,117,80,31,53,45.2,0.089,24,0
-4,142,86,0,0,44.0,0.645,22,1
-6,134,80,37,370,46.2,0.238,46,1
-1,79,80,25,37,25.4,0.583,22,0
-4,122,68,0,0,35.0,0.394,29,0
-3,74,68,28,45,29.7,0.293,23,0
-4,171,72,0,0,43.6,0.479,26,1
-7,181,84,21,192,35.9,0.586,51,1
-0,179,90,27,0,44.1,0.686,23,1
-9,164,84,21,0,30.8,0.831,32,1
-0,104,76,0,0,18.4,0.582,27,0
-1,91,64,24,0,29.2,0.192,21,0
-4,91,70,32,88,33.1,0.446,22,0
-3,139,54,0,0,25.6,0.402,22,1
-6,119,50,22,176,27.1,1.318,33,1
-2,146,76,35,194,38.2,0.329,29,0
-9,184,85,15,0,30.0,1.213,49,1
-10,122,68,0,0,31.2,0.258,41,0
-0,165,90,33,680,52.3,0.427,23,0
-9,124,70,33,402,35.4,0.282,34,0
-1,111,86,19,0,30.1,0.143,23,0
-9,106,52,0,0,31.2,0.380,42,0
-2,129,84,0,0,28.0,0.284,27,0
-2,90,80,14,55,24.4,0.249,24,0
-0,86,68,32,0,35.8,0.238,25,0
-12,92,62,7,258,27.6,0.926,44,1
-1,113,64,35,0,33.6,0.543,21,1
-3,111,56,39,0,30.1,0.557,30,0
-2,114,68,22,0,28.7,0.092,25,0
-1,193,50,16,375,25.9,0.655,24,0
-11,155,76,28,150,33.3,1.353,51,1
-3,191,68,15,130,30.9,0.299,34,0
-3,141,0,0,0,30.0,0.761,27,1
-4,95,70,32,0,32.1,0.612,24,0
-3,142,80,15,0,32.4,0.200,63,0
-4,123,62,0,0,32.0,0.226,35,1
-5,96,74,18,67,33.6,0.997,43,0
-0,138,0,0,0,36.3,0.933,25,1
-2,128,64,42,0,40.0,1.101,24,0
-0,102,52,0,0,25.1,0.078,21,0
-2,146,0,0,0,27.5,0.240,28,1
-10,101,86,37,0,45.6,1.136,38,1
-2,108,62,32,56,25.2,0.128,21,0
-3,122,78,0,0,23.0,0.254,40,0
-1,71,78,50,45,33.2,0.422,21,0
-13,106,70,0,0,34.2,0.251,52,0
-2,100,70,52,57,40.5,0.677,25,0
-7,106,60,24,0,26.5,0.296,29,1
-0,104,64,23,116,27.8,0.454,23,0
-5,114,74,0,0,24.9,0.744,57,0
-2,108,62,10,278,25.3,0.881,22,0
-0,146,70,0,0,37.9,0.334,28,1
-10,129,76,28,122,35.9,0.280,39,0
-7,133,88,15,155,32.4,0.262,37,0
-7,161,86,0,0,30.4,0.165,47,1
-2,108,80,0,0,27.0,0.259,52,1
-7,136,74,26,135,26.0,0.647,51,0
-5,155,84,44,545,38.7,0.619,34,0
-1,119,86,39,220,45.6,0.808,29,1
-4,96,56,17,49,20.8,0.340,26,0
-5,108,72,43,75,36.1,0.263,33,0
-0,78,88,29,40,36.9,0.434,21,0
-0,107,62,30,74,36.6,0.757,25,1
-2,128,78,37,182,43.3,1.224,31,1
-1,128,48,45,194,40.5,0.613,24,1
-0,161,50,0,0,21.9,0.254,65,0
-6,151,62,31,120,35.5,0.692,28,0
-2,146,70,38,360,28.0,0.337,29,1
-0,126,84,29,215,30.7,0.520,24,0
-14,100,78,25,184,36.6,0.412,46,1
-8,112,72,0,0,23.6,0.840,58,0
-0,167,0,0,0,32.3,0.839,30,1
-2,144,58,33,135,31.6,0.422,25,1
-5,77,82,41,42,35.8,0.156,35,0
-5,115,98,0,0,52.9,0.209,28,1
-3,150,76,0,0,21.0,0.207,37,0
-2,120,76,37,105,39.7,0.215,29,0
-10,161,68,23,132,25.5,0.326,47,1
-0,137,68,14,148,24.8,0.143,21,0
-0,128,68,19,180,30.5,1.391,25,1
-2,124,68,28,205,32.9,0.875,30,1
-6,80,66,30,0,26.2,0.313,41,0
-0,106,70,37,148,39.4,0.605,22,0
-2,155,74,17,96,26.6,0.433,27,1
-3,113,50,10,85,29.5,0.626,25,0
-7,109,80,31,0,35.9,1.127,43,1
-2,112,68,22,94,34.1,0.315,26,0
-3,99,80,11,64,19.3,0.284,30,0
-3,182,74,0,0,30.5,0.345,29,1
-3,115,66,39,140,38.1,0.150,28,0
-6,194,78,0,0,23.5,0.129,59,1
-4,129,60,12,231,27.5,0.527,31,0
-3,112,74,30,0,31.6,0.197,25,1
-0,124,70,20,0,27.4,0.254,36,1
-13,152,90,33,29,26.8,0.731,43,1
-2,112,75,32,0,35.7,0.148,21,0
-1,157,72,21,168,25.6,0.123,24,0
-1,122,64,32,156,35.1,0.692,30,1
-10,179,70,0,0,35.1,0.200,37,0
-2,102,86,36,120,45.5,0.127,23,1
-6,105,70,32,68,30.8,0.122,37,0
-8,118,72,19,0,23.1,1.476,46,0
-2,87,58,16,52,32.7,0.166,25,0
-1,180,0,0,0,43.3,0.282,41,1
-12,106,80,0,0,23.6,0.137,44,0
-1,95,60,18,58,23.9,0.260,22,0
-0,165,76,43,255,47.9,0.259,26,0
-0,117,0,0,0,33.8,0.932,44,0
-5,115,76,0,0,31.2,0.343,44,1
-9,152,78,34,171,34.2,0.893,33,1
-7,178,84,0,0,39.9,0.331,41,1
-1,130,70,13,105,25.9,0.472,22,0
-1,95,74,21,73,25.9,0.673,36,0
-1,0,68,35,0,32.0,0.389,22,0
-5,122,86,0,0,34.7,0.290,33,0
-8,95,72,0,0,36.8,0.485,57,0
-8,126,88,36,108,38.5,0.349,49,0
-1,139,46,19,83,28.7,0.654,22,0
-3,116,0,0,0,23.5,0.187,23,0
-3,99,62,19,74,21.8,0.279,26,0
-5,0,80,32,0,41.0,0.346,37,1
-4,92,80,0,0,42.2,0.237,29,0
-4,137,84,0,0,31.2,0.252,30,0
-3,61,82,28,0,34.4,0.243,46,0
-1,90,62,12,43,27.2,0.580,24,0
-3,90,78,0,0,42.7,0.559,21,0
-9,165,88,0,0,30.4,0.302,49,1
-1,125,50,40,167,33.3,0.962,28,1
-13,129,0,30,0,39.9,0.569,44,1
-12,88,74,40,54,35.3,0.378,48,0
-1,196,76,36,249,36.5,0.875,29,1
-5,189,64,33,325,31.2,0.583,29,1
-5,158,70,0,0,29.8,0.207,63,0
-5,103,108,37,0,39.2,0.305,65,0
-4,146,78,0,0,38.5,0.520,67,1
-4,147,74,25,293,34.9,0.385,30,0
-5,99,54,28,83,34.0,0.499,30,0
-6,124,72,0,0,27.6,0.368,29,1
-0,101,64,17,0,21.0,0.252,21,0
-3,81,86,16,66,27.5,0.306,22,0
-1,133,102,28,140,32.8,0.234,45,1
-3,173,82,48,465,38.4,2.137,25,1
-0,118,64,23,89,0.0,1.731,21,0
-0,84,64,22,66,35.8,0.545,21,0
-2,105,58,40,94,34.9,0.225,25,0
-2,122,52,43,158,36.2,0.816,28,0
-12,140,82,43,325,39.2,0.528,58,1
-0,98,82,15,84,25.2,0.299,22,0
-1,87,60,37,75,37.2,0.509,22,0
-4,156,75,0,0,48.3,0.238,32,1
-0,93,100,39,72,43.4,1.021,35,0
-1,107,72,30,82,30.8,0.821,24,0
-0,105,68,22,0,20.0,0.236,22,0
-1,109,60,8,182,25.4,0.947,21,0
-1,90,62,18,59,25.1,1.268,25,0
-1,125,70,24,110,24.3,0.221,25,0
-1,119,54,13,50,22.3,0.205,24,0
-5,116,74,29,0,32.3,0.660,35,1
-8,105,100,36,0,43.3,0.239,45,1
-5,144,82,26,285,32.0,0.452,58,1
-3,100,68,23,81,31.6,0.949,28,0
-1,100,66,29,196,32.0,0.444,42,0
-5,166,76,0,0,45.7,0.340,27,1
-1,131,64,14,415,23.7,0.389,21,0
-4,116,72,12,87,22.1,0.463,37,0
-4,158,78,0,0,32.9,0.803,31,1
-2,127,58,24,275,27.7,1.600,25,0
-3,96,56,34,115,24.7,0.944,39,0
-0,131,66,40,0,34.3,0.196,22,1
-3,82,70,0,0,21.1,0.389,25,0
-3,193,70,31,0,34.9,0.241,25,1
-4,95,64,0,0,32.0,0.161,31,1
-6,137,61,0,0,24.2,0.151,55,0
-5,136,84,41,88,35.0,0.286,35,1
-9,72,78,25,0,31.6,0.280,38,0
-5,168,64,0,0,32.9,0.135,41,1
-2,123,48,32,165,42.1,0.520,26,0
-4,115,72,0,0,28.9,0.376,46,1
-0,101,62,0,0,21.9,0.336,25,0
-8,197,74,0,0,25.9,1.191,39,1
-1,172,68,49,579,42.4,0.702,28,1
-6,102,90,39,0,35.7,0.674,28,0
-1,112,72,30,176,34.4,0.528,25,0
-1,143,84,23,310,42.4,1.076,22,0
-1,143,74,22,61,26.2,0.256,21,0
-0,138,60,35,167,34.6,0.534,21,1
-3,173,84,33,474,35.7,0.258,22,1
-1,97,68,21,0,27.2,1.095,22,0
-4,144,82,32,0,38.5,0.554,37,1
-1,83,68,0,0,18.2,0.624,27,0
-3,129,64,29,115,26.4,0.219,28,1
-1,119,88,41,170,45.3,0.507,26,0
-2,94,68,18,76,26.0,0.561,21,0
-0,102,64,46,78,40.6,0.496,21,0
-2,115,64,22,0,30.8,0.421,21,0
-8,151,78,32,210,42.9,0.516,36,1
-4,184,78,39,277,37.0,0.264,31,1
-0,94,0,0,0,0.0,0.256,25,0
-1,181,64,30,180,34.1,0.328,38,1
-0,135,94,46,145,40.6,0.284,26,0
-1,95,82,25,180,35.0,0.233,43,1
-2,99,0,0,0,22.2,0.108,23,0
-3,89,74,16,85,30.4,0.551,38,0
-1,80,74,11,60,30.0,0.527,22,0
-2,139,75,0,0,25.6,0.167,29,0
-1,90,68,8,0,24.5,1.138,36,0
-0,141,0,0,0,42.4,0.205,29,1
-12,140,85,33,0,37.4,0.244,41,0
-5,147,75,0,0,29.9,0.434,28,0
-1,97,70,15,0,18.2,0.147,21,0
-6,107,88,0,0,36.8,0.727,31,0
-0,189,104,25,0,34.3,0.435,41,1
-2,83,66,23,50,32.2,0.497,22,0
-4,117,64,27,120,33.2,0.230,24,0
-8,108,70,0,0,30.5,0.955,33,1
-4,117,62,12,0,29.7,0.380,30,1
-0,180,78,63,14,59.4,2.420,25,1
-1,100,72,12,70,25.3,0.658,28,0
-0,95,80,45,92,36.5,0.330,26,0
-0,104,64,37,64,33.6,0.510,22,1
-0,120,74,18,63,30.5,0.285,26,0
-1,82,64,13,95,21.2,0.415,23,0
-2,134,70,0,0,28.9,0.542,23,1
-0,91,68,32,210,39.9,0.381,25,0
-2,119,0,0,0,19.6,0.832,72,0
-2,100,54,28,105,37.8,0.498,24,0
-14,175,62,30,0,33.6,0.212,38,1
-1,135,54,0,0,26.7,0.687,62,0
-5,86,68,28,71,30.2,0.364,24,0
-10,148,84,48,237,37.6,1.001,51,1
-9,134,74,33,60,25.9,0.460,81,0
-9,120,72,22,56,20.8,0.733,48,0
-1,71,62,0,0,21.8,0.416,26,0
-8,74,70,40,49,35.3,0.705,39,0
-5,88,78,30,0,27.6,0.258,37,0
-10,115,98,0,0,24.0,1.022,34,0
-0,124,56,13,105,21.8,0.452,21,0
-0,74,52,10,36,27.8,0.269,22,0
-0,97,64,36,100,36.8,0.600,25,0
-8,120,0,0,0,30.0,0.183,38,1
-6,154,78,41,140,46.1,0.571,27,0
-1,144,82,40,0,41.3,0.607,28,0
-0,137,70,38,0,33.2,0.170,22,0
-0,119,66,27,0,38.8,0.259,22,0
-7,136,90,0,0,29.9,0.210,50,0
-4,114,64,0,0,28.9,0.126,24,0
-0,137,84,27,0,27.3,0.231,59,0
-2,105,80,45,191,33.7,0.711,29,1
-7,114,76,17,110,23.8,0.466,31,0
-8,126,74,38,75,25.9,0.162,39,0
-4,132,86,31,0,28.0,0.419,63,0
-3,158,70,30,328,35.5,0.344,35,1
-0,123,88,37,0,35.2,0.197,29,0
-4,85,58,22,49,27.8,0.306,28,0
-0,84,82,31,125,38.2,0.233,23,0
-0,145,0,0,0,44.2,0.630,31,1
-0,135,68,42,250,42.3,0.365,24,1
-1,139,62,41,480,40.7,0.536,21,0
-0,173,78,32,265,46.5,1.159,58,0
-4,99,72,17,0,25.6,0.294,28,0
-8,194,80,0,0,26.1,0.551,67,0
-2,83,65,28,66,36.8,0.629,24,0
-2,89,90,30,0,33.5,0.292,42,0
-4,99,68,38,0,32.8,0.145,33,0
-4,125,70,18,122,28.9,1.144,45,1
-3,80,0,0,0,0.0,0.174,22,0
-6,166,74,0,0,26.6,0.304,66,0
-5,110,68,0,0,26.0,0.292,30,0
-2,81,72,15,76,30.1,0.547,25,0
-7,195,70,33,145,25.1,0.163,55,1
-6,154,74,32,193,29.3,0.839,39,0
-2,117,90,19,71,25.2,0.313,21,0
-3,84,72,32,0,37.2,0.267,28,0
-6,0,68,41,0,39.0,0.727,41,1
-7,94,64,25,79,33.3,0.738,41,0
-3,96,78,39,0,37.3,0.238,40,0
-10,75,82,0,0,33.3,0.263,38,0
-0,180,90,26,90,36.5,0.314,35,1
-1,130,60,23,170,28.6,0.692,21,0
-2,84,50,23,76,30.4,0.968,21,0
-8,120,78,0,0,25.0,0.409,64,0
-12,84,72,31,0,29.7,0.297,46,1
-0,139,62,17,210,22.1,0.207,21,0
-9,91,68,0,0,24.2,0.200,58,0
-2,91,62,0,0,27.3,0.525,22,0
-3,99,54,19,86,25.6,0.154,24,0
-3,163,70,18,105,31.6,0.268,28,1
-9,145,88,34,165,30.3,0.771,53,1
-7,125,86,0,0,37.6,0.304,51,0
-13,76,60,0,0,32.8,0.180,41,0
-6,129,90,7,326,19.6,0.582,60,0
-2,68,70,32,66,25.0,0.187,25,0
-3,124,80,33,130,33.2,0.305,26,0
-6,114,0,0,0,0.0,0.189,26,0
-9,130,70,0,0,34.2,0.652,45,1
-3,125,58,0,0,31.6,0.151,24,0
-3,87,60,18,0,21.8,0.444,21,0
-1,97,64,19,82,18.2,0.299,21,0
-3,116,74,15,105,26.3,0.107,24,0
-0,117,66,31,188,30.8,0.493,22,0
-0,111,65,0,0,24.6,0.660,31,0
-2,122,60,18,106,29.8,0.717,22,0
-0,107,76,0,0,45.3,0.686,24,0
-1,86,66,52,65,41.3,0.917,29,0
-6,91,0,0,0,29.8,0.501,31,0
-1,77,56,30,56,33.3,1.251,24,0
-4,132,0,0,0,32.9,0.302,23,1
-0,105,90,0,0,29.6,0.197,46,0
-0,57,60,0,0,21.7,0.735,67,0
-0,127,80,37,210,36.3,0.804,23,0
-3,129,92,49,155,36.4,0.968,32,1
-8,100,74,40,215,39.4,0.661,43,1
-3,128,72,25,190,32.4,0.549,27,1
-10,90,85,32,0,34.9,0.825,56,1
-4,84,90,23,56,39.5,0.159,25,0
-1,88,78,29,76,32.0,0.365,29,0
-8,186,90,35,225,34.5,0.423,37,1
-5,187,76,27,207,43.6,1.034,53,1
-4,131,68,21,166,33.1,0.160,28,0
-1,164,82,43,67,32.8,0.341,50,0
-4,189,110,31,0,28.5,0.680,37,0
-1,116,70,28,0,27.4,0.204,21,0
-3,84,68,30,106,31.9,0.591,25,0
-6,114,88,0,0,27.8,0.247,66,0
-1,88,62,24,44,29.9,0.422,23,0
-1,84,64,23,115,36.9,0.471,28,0
-7,124,70,33,215,25.5,0.161,37,0
-1,97,70,40,0,38.1,0.218,30,0
-8,110,76,0,0,27.8,0.237,58,0
-11,103,68,40,0,46.2,0.126,42,0
-11,85,74,0,0,30.1,0.300,35,0
-6,125,76,0,0,33.8,0.121,54,1
-0,198,66,32,274,41.3,0.502,28,1
-1,87,68,34,77,37.6,0.401,24,0
-6,99,60,19,54,26.9,0.497,32,0
-0,91,80,0,0,32.4,0.601,27,0
-2,95,54,14,88,26.1,0.748,22,0
-1,99,72,30,18,38.6,0.412,21,0
-6,92,62,32,126,32.0,0.085,46,0
-4,154,72,29,126,31.3,0.338,37,0
-0,121,66,30,165,34.3,0.203,33,1
-3,78,70,0,0,32.5,0.270,39,0
-2,130,96,0,0,22.6,0.268,21,0
-3,111,58,31,44,29.5,0.430,22,0
-2,98,60,17,120,34.7,0.198,22,0
-1,143,86,30,330,30.1,0.892,23,0
-1,119,44,47,63,35.5,0.280,25,0
-6,108,44,20,130,24.0,0.813,35,0
-2,118,80,0,0,42.9,0.693,21,1
-10,133,68,0,0,27.0,0.245,36,0
-2,197,70,99,0,34.7,0.575,62,1
-0,151,90,46,0,42.1,0.371,21,1
-6,109,60,27,0,25.0,0.206,27,0
-12,121,78,17,0,26.5,0.259,62,0
-8,100,76,0,0,38.7,0.190,42,0
-8,124,76,24,600,28.7,0.687,52,1
-1,93,56,11,0,22.5,0.417,22,0
-8,143,66,0,0,34.9,0.129,41,1
-6,103,66,0,0,24.3,0.249,29,0
-3,176,86,27,156,33.3,1.154,52,1
-0,73,0,0,0,21.1,0.342,25,0
-11,111,84,40,0,46.8,0.925,45,1
-2,112,78,50,140,39.4,0.175,24,0
-3,132,80,0,0,34.4,0.402,44,1
-2,82,52,22,115,28.5,1.699,25,0
-6,123,72,45,230,33.6,0.733,34,0
-0,188,82,14,185,32.0,0.682,22,1
-0,67,76,0,0,45.3,0.194,46,0
-1,89,24,19,25,27.8,0.559,21,0
-1,173,74,0,0,36.8,0.088,38,1
-1,109,38,18,120,23.1,0.407,26,0
-1,108,88,19,0,27.1,0.400,24,0
-6,96,0,0,0,23.7,0.190,28,0
-1,124,74,36,0,27.8,0.100,30,0
-7,150,78,29,126,35.2,0.692,54,1
-4,183,0,0,0,28.4,0.212,36,1
-1,124,60,32,0,35.8,0.514,21,0
-1,181,78,42,293,40.0,1.258,22,1
-1,92,62,25,41,19.5,0.482,25,0
-0,152,82,39,272,41.5,0.270,27,0
-1,111,62,13,182,24.0,0.138,23,0
-3,106,54,21,158,30.9,0.292,24,0
-3,174,58,22,194,32.9,0.593,36,1
-7,168,88,42,321,38.2,0.787,40,1
-6,105,80,28,0,32.5,0.878,26,0
-11,138,74,26,144,36.1,0.557,50,1
-3,106,72,0,0,25.8,0.207,27,0
-6,117,96,0,0,28.7,0.157,30,0
-2,68,62,13,15,20.1,0.257,23,0
-9,112,82,24,0,28.2,1.282,50,1
-0,119,0,0,0,32.4,0.141,24,1
-2,112,86,42,160,38.4,0.246,28,0
-2,92,76,20,0,24.2,1.698,28,0
-6,183,94,0,0,40.8,1.461,45,0
-0,94,70,27,115,43.5,0.347,21,0
-2,108,64,0,0,30.8,0.158,21,0
-4,90,88,47,54,37.7,0.362,29,0
-0,125,68,0,0,24.7,0.206,21,0
-0,132,78,0,0,32.4,0.393,21,0
-5,128,80,0,0,34.6,0.144,45,0
-4,94,65,22,0,24.7,0.148,21,0
-7,114,64,0,0,27.4,0.732,34,1
-0,102,78,40,90,34.5,0.238,24,0
-2,111,60,0,0,26.2,0.343,23,0
-1,128,82,17,183,27.5,0.115,22,0
-10,92,62,0,0,25.9,0.167,31,0
-13,104,72,0,0,31.2,0.465,38,1
-5,104,74,0,0,28.8,0.153,48,0
-2,94,76,18,66,31.6,0.649,23,0
-7,97,76,32,91,40.9,0.871,32,1
-1,100,74,12,46,19.5,0.149,28,0
-0,102,86,17,105,29.3,0.695,27,0
-4,128,70,0,0,34.3,0.303,24,0
-6,147,80,0,0,29.5,0.178,50,1
-4,90,0,0,0,28.0,0.610,31,0
-3,103,72,30,152,27.6,0.730,27,0
-2,157,74,35,440,39.4,0.134,30,0
-1,167,74,17,144,23.4,0.447,33,1
-0,179,50,36,159,37.8,0.455,22,1
-11,136,84,35,130,28.3,0.260,42,1
-0,107,60,25,0,26.4,0.133,23,0
-1,91,54,25,100,25.2,0.234,23,0
-1,117,60,23,106,33.8,0.466,27,0
-5,123,74,40,77,34.1,0.269,28,0
-2,120,54,0,0,26.8,0.455,27,0
-1,106,70,28,135,34.2,0.142,22,0
-2,155,52,27,540,38.7,0.240,25,1
-2,101,58,35,90,21.8,0.155,22,0
-1,120,80,48,200,38.9,1.162,41,0
-11,127,106,0,0,39.0,0.190,51,0
-3,80,82,31,70,34.2,1.292,27,1
-10,162,84,0,0,27.7,0.182,54,0
-1,199,76,43,0,42.9,1.394,22,1
-8,167,106,46,231,37.6,0.165,43,1
-9,145,80,46,130,37.9,0.637,40,1
-6,115,60,39,0,33.7,0.245,40,1
-1,112,80,45,132,34.8,0.217,24,0
-4,145,82,18,0,32.5,0.235,70,1
-10,111,70,27,0,27.5,0.141,40,1
-6,98,58,33,190,34.0,0.430,43,0
-9,154,78,30,100,30.9,0.164,45,0
-6,165,68,26,168,33.6,0.631,49,0
-1,99,58,10,0,25.4,0.551,21,0
-10,68,106,23,49,35.5,0.285,47,0
-3,123,100,35,240,57.3,0.880,22,0
-8,91,82,0,0,35.6,0.587,68,0
-6,195,70,0,0,30.9,0.328,31,1
-9,156,86,0,0,24.8,0.230,53,1
-0,93,60,0,0,35.3,0.263,25,0
-3,121,52,0,0,36.0,0.127,25,1
-2,101,58,17,265,24.2,0.614,23,0
-2,56,56,28,45,24.2,0.332,22,0
-0,162,76,36,0,49.6,0.364,26,1
-0,95,64,39,105,44.6,0.366,22,0
-4,125,80,0,0,32.3,0.536,27,1
-5,136,82,0,0,0.0,0.640,69,0
-2,129,74,26,205,33.2,0.591,25,0
-3,130,64,0,0,23.1,0.314,22,0
-1,107,50,19,0,28.3,0.181,29,0
-1,140,74,26,180,24.1,0.828,23,0
-1,144,82,46,180,46.1,0.335,46,1
-8,107,80,0,0,24.6,0.856,34,0
-13,158,114,0,0,42.3,0.257,44,1
-2,121,70,32,95,39.1,0.886,23,0
-7,129,68,49,125,38.5,0.439,43,1
-2,90,60,0,0,23.5,0.191,25,0
-7,142,90,24,480,30.4,0.128,43,1
-3,169,74,19,125,29.9,0.268,31,1
-0,99,0,0,0,25.0,0.253,22,0
-4,127,88,11,155,34.5,0.598,28,0
-4,118,70,0,0,44.5,0.904,26,0
-2,122,76,27,200,35.9,0.483,26,0
-6,125,78,31,0,27.6,0.565,49,1
-1,168,88,29,0,35.0,0.905,52,1
-2,129,0,0,0,38.5,0.304,41,0
-4,110,76,20,100,28.4,0.118,27,0
-6,80,80,36,0,39.8,0.177,28,0
-10,115,0,0,0,0.0,0.261,30,1
-2,127,46,21,335,34.4,0.176,22,0
-9,164,78,0,0,32.8,0.148,45,1
-2,93,64,32,160,38.0,0.674,23,1
-3,158,64,13,387,31.2,0.295,24,0
-5,126,78,27,22,29.6,0.439,40,0
-10,129,62,36,0,41.2,0.441,38,1
-0,134,58,20,291,26.4,0.352,21,0
-3,102,74,0,0,29.5,0.121,32,0
-7,187,50,33,392,33.9,0.826,34,1
-3,173,78,39,185,33.8,0.970,31,1
-10,94,72,18,0,23.1,0.595,56,0
-1,108,60,46,178,35.5,0.415,24,0
-5,97,76,27,0,35.6,0.378,52,1
-4,83,86,19,0,29.3,0.317,34,0
-1,114,66,36,200,38.1,0.289,21,0
-1,149,68,29,127,29.3,0.349,42,1
-5,117,86,30,105,39.1,0.251,42,0
-1,111,94,0,0,32.8,0.265,45,0
-4,112,78,40,0,39.4,0.236,38,0
-1,116,78,29,180,36.1,0.496,25,0
-0,141,84,26,0,32.4,0.433,22,0
-2,175,88,0,0,22.9,0.326,22,0
-2,92,52,0,0,30.1,0.141,22,0
-3,130,78,23,79,28.4,0.323,34,1
-8,120,86,0,0,28.4,0.259,22,1
-2,174,88,37,120,44.5,0.646,24,1
-2,106,56,27,165,29.0,0.426,22,0
-2,105,75,0,0,23.3,0.560,53,0
-4,95,60,32,0,35.4,0.284,28,0
-0,126,86,27,120,27.4,0.515,21,0
-8,65,72,23,0,32.0,0.600,42,0
-2,99,60,17,160,36.6,0.453,21,0
-1,102,74,0,0,39.5,0.293,42,1
-11,120,80,37,150,42.3,0.785,48,1
-3,102,44,20,94,30.8,0.400,26,0
-1,109,58,18,116,28.5,0.219,22,0
-9,140,94,0,0,32.7,0.734,45,1
-13,153,88,37,140,40.6,1.174,39,0
-12,100,84,33,105,30.0,0.488,46,0
-1,147,94,41,0,49.3,0.358,27,1
-1,81,74,41,57,46.3,1.096,32,0
-3,187,70,22,200,36.4,0.408,36,1
-6,162,62,0,0,24.3,0.178,50,1
-4,136,70,0,0,31.2,1.182,22,1
-1,121,78,39,74,39.0,0.261,28,0
-3,108,62,24,0,26.0,0.223,25,0
-0,181,88,44,510,43.3,0.222,26,1
-8,154,78,32,0,32.4,0.443,45,1
-1,128,88,39,110,36.5,1.057,37,1
-7,137,90,41,0,32.0,0.391,39,0
-0,123,72,0,0,36.3,0.258,52,1
-1,106,76,0,0,37.5,0.197,26,0
-6,190,92,0,0,35.5,0.278,66,1
-2,88,58,26,16,28.4,0.766,22,0
-9,170,74,31,0,44.0,0.403,43,1
-9,89,62,0,0,22.5,0.142,33,0
-10,101,76,48,180,32.9,0.171,63,0
-2,122,70,27,0,36.8,0.340,27,0
-5,121,72,23,112,26.2,0.245,30,0
-1,126,60,0,0,30.1,0.349,47,1
-1,93,70,31,0,30.4,0.315,23,0
-

GPy/util/datasets/heart.dat

@@ -1,271 +0,0 @@
-70.0 1.0 4.0 130.0 322.0 0.0 2.0 109.0 0.0 2.4 2.0 3.0 3.0 2
-67.0 0.0 3.0 115.0 564.0 0.0 2.0 160.0 0.0 1.6 2.0 0.0 7.0 1
-57.0 1.0 2.0 124.0 261.0 0.0 0.0 141.0 0.0 0.3 1.0 0.0 7.0 2
-64.0 1.0 4.0 128.0 263.0 0.0 0.0 105.0 1.0 0.2 2.0 1.0 7.0 1
-74.0 0.0 2.0 120.0 269.0 0.0 2.0 121.0 1.0 0.2 1.0 1.0 3.0 1
-65.0 1.0 4.0 120.0 177.0 0.0 0.0 140.0 0.0 0.4 1.0 0.0 7.0 1
-56.0 1.0 3.0 130.0 256.0 1.0 2.0 142.0 1.0 0.6 2.0 1.0 6.0 2
-59.0 1.0 4.0 110.0 239.0 0.0 2.0 142.0 1.0 1.2 2.0 1.0 7.0 2
-60.0 1.0 4.0 140.0 293.0 0.0 2.0 170.0 0.0 1.2 2.0 2.0 7.0 2
-63.0 0.0 4.0 150.0 407.0 0.0 2.0 154.0 0.0 4.0 2.0 3.0 7.0 2
-59.0 1.0 4.0 135.0 234.0 0.0 0.0 161.0 0.0 0.5 2.0 0.0 7.0 1
-53.0 1.0 4.0 142.0 226.0 0.0 2.0 111.0 1.0 0.0 1.0 0.0 7.0 1
-44.0 1.0 3.0 140.0 235.0 0.0 2.0 180.0 0.0 0.0 1.0 0.0 3.0 1
-61.0 1.0 1.0 134.0 234.0 0.0 0.0 145.0 0.0 2.6 2.0 2.0 3.0 2
-57.0 0.0 4.0 128.0 303.0 0.0 2.0 159.0 0.0 0.0 1.0 1.0 3.0 1
-71.0 0.0 4.0 112.0 149.0 0.0 0.0 125.0 0.0 1.6 2.0 0.0 3.0 1
-46.0 1.0 4.0 140.0 311.0 0.0 0.0 120.0 1.0 1.8 2.0 2.0 7.0 2
-53.0 1.0 4.0 140.0 203.0 1.0 2.0 155.0 1.0 3.1 3.0 0.0 7.0 2
-64.0 1.0 1.0 110.0 211.0 0.0 2.0 144.0 1.0 1.8 2.0 0.0 3.0 1
-40.0 1.0 1.0 140.0 199.0 0.0 0.0 178.0 1.0 1.4 1.0 0.0 7.0 1
-67.0 1.0 4.0 120.0 229.0 0.0 2.0 129.0 1.0 2.6 2.0 2.0 7.0 2
-48.0 1.0 2.0 130.0 245.0 0.0 2.0 180.0 0.0 0.2 2.0 0.0 3.0 1
-43.0 1.0 4.0 115.0 303.0 0.0 0.0 181.0 0.0 1.2 2.0 0.0 3.0 1
-47.0 1.0 4.0 112.0 204.0 0.0 0.0 143.0 0.0 0.1 1.0 0.0 3.0 1
-54.0 0.0 2.0 132.0 288.0 1.0 2.0 159.0 1.0 0.0 1.0 1.0 3.0 1
-48.0 0.0 3.0 130.0 275.0 0.0 0.0 139.0 0.0 0.2 1.0 0.0 3.0 1
-46.0 0.0 4.0 138.0 243.0 0.0 2.0 152.0 1.0 0.0 2.0 0.0 3.0 1
-51.0 0.0 3.0 120.0 295.0 0.0 2.0 157.0 0.0 0.6 1.0 0.0 3.0 1
-58.0 1.0 3.0 112.0 230.0 0.0 2.0 165.0 0.0 2.5 2.0 1.0 7.0 2
-71.0 0.0 3.0 110.0 265.0 1.0 2.0 130.0 0.0 0.0 1.0 1.0 3.0 1
-57.0 1.0 3.0 128.0 229.0 0.0 2.0 150.0 0.0 0.4 2.0 1.0 7.0 2
-66.0 1.0 4.0 160.0 228.0 0.0 2.0 138.0 0.0 2.3 1.0 0.0 6.0 1
-37.0 0.0 3.0 120.0 215.0 0.0 0.0 170.0 0.0 0.0 1.0 0.0 3.0 1
-59.0 1.0 4.0 170.0 326.0 0.0 2.0 140.0 1.0 3.4 3.0 0.0 7.0 2
-50.0 1.0 4.0 144.0 200.0 0.0 2.0 126.0 1.0 0.9 2.0 0.0 7.0 2
-48.0 1.0 4.0 130.0 256.0 1.0 2.0 150.0 1.0 0.0 1.0 2.0 7.0 2
-61.0 1.0 4.0 140.0 207.0 0.0 2.0 138.0 1.0 1.9 1.0 1.0 7.0 2
-59.0 1.0 1.0 160.0 273.0 0.0 2.0 125.0 0.0 0.0 1.0 0.0 3.0 2
-42.0 1.0 3.0 130.0 180.0 0.0 0.0 150.0 0.0 0.0 1.0 0.0 3.0 1
-48.0 1.0 4.0 122.0 222.0 0.0 2.0 186.0 0.0 0.0 1.0 0.0 3.0 1
-40.0 1.0 4.0 152.0 223.0 0.0 0.0 181.0 0.0 0.0 1.0 0.0 7.0 2
-62.0 0.0 4.0 124.0 209.0 0.0 0.0 163.0 0.0 0.0 1.0 0.0 3.0 1
-44.0 1.0 3.0 130.0 233.0 0.0 0.0 179.0 1.0 0.4 1.0 0.0 3.0 1
-46.0 1.0 2.0 101.0 197.0 1.0 0.0 156.0 0.0 0.0 1.0 0.0 7.0 1
-59.0 1.0 3.0 126.0 218.0 1.0 0.0 134.0 0.0 2.2 2.0 1.0 6.0 2
-58.0 1.0 3.0 140.0 211.0 1.0 2.0 165.0 0.0 0.0 1.0 0.0 3.0 1
-49.0 1.0 3.0 118.0 149.0 0.0 2.0 126.0 0.0 0.8 1.0 3.0 3.0 2
-44.0 1.0 4.0 110.0 197.0 0.0 2.0 177.0 0.0 0.0 1.0 1.0 3.0 2
-66.0 1.0 2.0 160.0 246.0 0.0 0.0 120.0 1.0 0.0 2.0 3.0 6.0 2
-65.0 0.0 4.0 150.0 225.0 0.0 2.0 114.0 0.0 1.0 2.0 3.0 7.0 2
-42.0 1.0 4.0 136.0 315.0 0.0 0.0 125.0 1.0 1.8 2.0 0.0 6.0 2
-52.0 1.0 2.0 128.0 205.0 1.0 0.0 184.0 0.0 0.0 1.0 0.0 3.0 1
-65.0 0.0 3.0 140.0 417.0 1.0 2.0 157.0 0.0 0.8 1.0 1.0 3.0 1
-63.0 0.0 2.0 140.0 195.0 0.0 0.0 179.0 0.0 0.0 1.0 2.0 3.0 1
-45.0 0.0 2.0 130.0 234.0 0.0 2.0 175.0 0.0 0.6 2.0 0.0 3.0 1
-41.0 0.0 2.0 105.0 198.0 0.0 0.0 168.0 0.0 0.0 1.0 1.0 3.0 1
-61.0 1.0 4.0 138.0 166.0 0.0 2.0 125.0 1.0 3.6 2.0 1.0 3.0 2
-60.0 0.0 3.0 120.0 178.0 1.0 0.0 96.0 0.0 0.0 1.0 0.0 3.0 1
-59.0 0.0 4.0 174.0 249.0 0.0 0.0 143.0 1.0 0.0 2.0 0.0 3.0 2
-62.0 1.0 2.0 120.0 281.0 0.0 2.0 103.0 0.0 1.4 2.0 1.0 7.0 2
-57.0 1.0 3.0 150.0 126.0 1.0 0.0 173.0 0.0 0.2 1.0 1.0 7.0 1
-51.0 0.0 4.0 130.0 305.0 0.0 0.0 142.0 1.0 1.2 2.0 0.0 7.0 2
-44.0 1.0 3.0 120.0 226.0 0.0 0.0 169.0 0.0 0.0 1.0 0.0 3.0 1
-60.0 0.0 1.0 150.0 240.0 0.0 0.0 171.0 0.0 0.9 1.0 0.0 3.0 1
-63.0 1.0 1.0 145.0 233.0 1.0 2.0 150.0 0.0 2.3 3.0 0.0 6.0 1
-57.0 1.0 4.0 150.0 276.0 0.0 2.0 112.0 1.0 0.6 2.0 1.0 6.0 2
-51.0 1.0 4.0 140.0 261.0 0.0 2.0 186.0 1.0 0.0 1.0 0.0 3.0 1
-58.0 0.0 2.0 136.0 319.0 1.0 2.0 152.0 0.0 0.0 1.0 2.0 3.0 2
-44.0 0.0 3.0 118.0 242.0 0.0 0.0 149.0 0.0 0.3 2.0 1.0 3.0 1
-47.0 1.0 3.0 108.0 243.0 0.0 0.0 152.0 0.0 0.0 1.0 0.0 3.0 2
-61.0 1.0 4.0 120.0 260.0 0.0 0.0 140.0 1.0 3.6 2.0 1.0 7.0 2
-57.0 0.0 4.0 120.0 354.0 0.0 0.0 163.0 1.0 0.6 1.0 0.0 3.0 1
-70.0 1.0 2.0 156.0 245.0 0.0 2.0 143.0 0.0 0.0 1.0 0.0 3.0 1
-76.0 0.0 3.0 140.0 197.0 0.0 1.0 116.0 0.0 1.1 2.0 0.0 3.0 1
-67.0 0.0 4.0 106.0 223.0 0.0 0.0 142.0 0.0 0.3 1.0 2.0 3.0 1
-45.0 1.0 4.0 142.0 309.0 0.0 2.0 147.0 1.0 0.0 2.0 3.0 7.0 2
-45.0 1.0 4.0 104.0 208.0 0.0 2.0 148.0 1.0 3.0 2.0 0.0 3.0 1
-39.0 0.0 3.0 94.0 199.0 0.0 0.0 179.0 0.0 0.0 1.0 0.0 3.0 1
-42.0 0.0 3.0 120.0 209.0 0.0 0.0 173.0 0.0 0.0 2.0 0.0 3.0 1
-56.0 1.0 2.0 120.0 236.0 0.0 0.0 178.0 0.0 0.8 1.0 0.0 3.0 1
-58.0 1.0 4.0 146.0 218.0 0.0 0.0 105.0 0.0 2.0 2.0 1.0 7.0 2
-35.0 1.0 4.0 120.0 198.0 0.0 0.0 130.0 1.0 1.6 2.0 0.0 7.0 2
-58.0 1.0 4.0 150.0 270.0 0.0 2.0 111.0 1.0 0.8 1.0 0.0 7.0 2
-41.0 1.0 3.0 130.0 214.0 0.0 2.0 168.0 0.0 2.0 2.0 0.0 3.0 1
-57.0 1.0 4.0 110.0 201.0 0.0 0.0 126.0 1.0 1.5 2.0 0.0 6.0 1
-42.0 1.0 1.0 148.0 244.0 0.0 2.0 178.0 0.0 0.8 1.0 2.0 3.0 1
-62.0 1.0 2.0 128.0 208.0 1.0 2.0 140.0 0.0 0.0 1.0 0.0 3.0 1
-59.0 1.0 1.0 178.0 270.0 0.0 2.0 145.0 0.0 4.2 3.0 0.0 7.0 1
-41.0 0.0 2.0 126.0 306.0 0.0 0.0 163.0 0.0 0.0 1.0 0.0 3.0 1
-50.0 1.0 4.0 150.0 243.0 0.0 2.0 128.0 0.0 2.6 2.0 0.0 7.0 2
-59.0 1.0 2.0 140.0 221.0 0.0 0.0 164.0 1.0 0.0 1.0 0.0 3.0 1
-61.0 0.0 4.0 130.0 330.0 0.0 2.0 169.0 0.0 0.0 1.0 0.0 3.0 2
-54.0 1.0 4.0 124.0 266.0 0.0 2.0 109.0 1.0 2.2 2.0 1.0 7.0 2
-54.0 1.0 4.0 110.0 206.0 0.0 2.0 108.0 1.0 0.0 2.0 1.0 3.0 2
-52.0 1.0 4.0 125.0 212.0 0.0 0.0 168.0 0.0 1.0 1.0 2.0 7.0 2
-47.0 1.0 4.0 110.0 275.0 0.0 2.0 118.0 1.0 1.0 2.0 1.0 3.0 2
-66.0 1.0 4.0 120.0 302.0 0.0 2.0 151.0 0.0 0.4 2.0 0.0 3.0 1
-58.0 1.0 4.0 100.0 234.0 0.0 0.0 156.0 0.0 0.1 1.0 1.0 7.0 2
-64.0 0.0 3.0 140.0 313.0 0.0 0.0 133.0 0.0 0.2 1.0 0.0 7.0 1
-50.0 0.0 2.0 120.0 244.0 0.0 0.0 162.0 0.0 1.1 1.0 0.0 3.0 1
-44.0 0.0 3.0 108.0 141.0 0.0 0.0 175.0 0.0 0.6 2.0 0.0 3.0 1
-67.0 1.0 4.0 120.0 237.0 0.0 0.0 71.0 0.0 1.0 2.0 0.0 3.0 2
-49.0 0.0 4.0 130.0 269.0 0.0 0.0 163.0 0.0 0.0 1.0 0.0 3.0 1
-57.0 1.0 4.0 165.0 289.0 1.0 2.0 124.0 0.0 1.0 2.0 3.0 7.0 2
-63.0 1.0 4.0 130.0 254.0 0.0 2.0 147.0 0.0 1.4 2.0 1.0 7.0 2
-48.0 1.0 4.0 124.0 274.0 0.0 2.0 166.0 0.0 0.5 2.0 0.0 7.0 2
-51.0 1.0 3.0 100.0 222.0 0.0 0.0 143.0 1.0 1.2 2.0 0.0 3.0 1
-60.0 0.0 4.0 150.0 258.0 0.0 2.0 157.0 0.0 2.6 2.0 2.0 7.0 2
-59.0 1.0 4.0 140.0 177.0 0.0 0.0 162.0 1.0 0.0 1.0 1.0 7.0 2
-45.0 0.0 2.0 112.0 160.0 0.0 0.0 138.0 0.0 0.0 2.0 0.0 3.0 1
-55.0 0.0 4.0 180.0 327.0 0.0 1.0 117.0 1.0 3.4 2.0 0.0 3.0 2
-41.0 1.0 2.0 110.0 235.0 0.0 0.0 153.0 0.0 0.0 1.0 0.0 3.0 1
-60.0 0.0 4.0 158.0 305.0 0.0 2.0 161.0 0.0 0.0 1.0 0.0 3.0 2
-54.0 0.0 3.0 135.0 304.0 1.0 0.0 170.0 0.0 0.0 1.0 0.0 3.0 1
-42.0 1.0 2.0 120.0 295.0 0.0 0.0 162.0 0.0 0.0 1.0 0.0 3.0 1
-49.0 0.0 2.0 134.0 271.0 0.0 0.0 162.0 0.0 0.0 2.0 0.0 3.0 1
-46.0 1.0 4.0 120.0 249.0 0.0 2.0 144.0 0.0 0.8 1.0 0.0 7.0 2
-56.0 0.0 4.0 200.0 288.0 1.0 2.0 133.0 1.0 4.0 3.0 2.0 7.0 2
-66.0 0.0 1.0 150.0 226.0 0.0 0.0 114.0 0.0 2.6 3.0 0.0 3.0 1
-56.0 1.0 4.0 130.0 283.0 1.0 2.0 103.0 1.0 1.6 3.0 0.0 7.0 2
-49.0 1.0 3.0 120.0 188.0 0.0 0.0 139.0 0.0 2.0 2.0 3.0 7.0 2
-54.0 1.0 4.0 122.0 286.0 0.0 2.0 116.0 1.0 3.2 2.0 2.0 3.0 2
-57.0 1.0 4.0 152.0 274.0 0.0 0.0 88.0 1.0 1.2 2.0 1.0 7.0 2
-65.0 0.0 3.0 160.0 360.0 0.0 2.0 151.0 0.0 0.8 1.0 0.0 3.0 1
-54.0 1.0 3.0 125.0 273.0 0.0 2.0 152.0 0.0 0.5 3.0 1.0 3.0 1
-54.0 0.0 3.0 160.0 201.0 0.0 0.0 163.0 0.0 0.0 1.0 1.0 3.0 1
-62.0 1.0 4.0 120.0 267.0 0.0 0.0 99.0 1.0 1.8 2.0 2.0 7.0 2
-52.0 0.0 3.0 136.0 196.0 0.0 2.0 169.0 0.0 0.1 2.0 0.0 3.0 1
-52.0 1.0 2.0 134.0 201.0 0.0 0.0 158.0 0.0 0.8 1.0 1.0 3.0 1
-60.0 1.0 4.0 117.0 230.0 1.0 0.0 160.0 1.0 1.4 1.0 2.0 7.0 2
-63.0 0.0 4.0 108.0 269.0 0.0 0.0 169.0 1.0 1.8 2.0 2.0 3.0 2
-66.0 1.0 4.0 112.0 212.0 0.0 2.0 132.0 1.0 0.1 1.0 1.0 3.0 2
-42.0 1.0 4.0 140.0 226.0 0.0 0.0 178.0 0.0 0.0 1.0 0.0 3.0 1
-64.0 1.0 4.0 120.0 246.0 0.0 2.0 96.0 1.0 2.2 3.0 1.0 3.0 2
-54.0 1.0 3.0 150.0 232.0 0.0 2.0 165.0 0.0 1.6 1.0 0.0 7.0 1
-46.0 0.0 3.0 142.0 177.0 0.0 2.0 160.0 1.0 1.4 3.0 0.0 3.0 1
-67.0 0.0 3.0 152.0 277.0 0.0 0.0 172.0 0.0 0.0 1.0 1.0 3.0 1
-56.0 1.0 4.0 125.0 249.0 1.0 2.0 144.0 1.0 1.2 2.0 1.0 3.0 2
-34.0 0.0 2.0 118.0 210.0 0.0 0.0 192.0 0.0 0.7 1.0 0.0 3.0 1
-57.0 1.0 4.0 132.0 207.0 0.0 0.0 168.0 1.0 0.0 1.0 0.0 7.0 1
-64.0 1.0 4.0 145.0 212.0 0.0 2.0 132.0 0.0 2.0 2.0 2.0 6.0 2
-59.0 1.0 4.0 138.0 271.0 0.0 2.0 182.0 0.0 0.0 1.0 0.0 3.0 1
-50.0 1.0 3.0 140.0 233.0 0.0 0.0 163.0 0.0 0.6 2.0 1.0 7.0 2
-51.0 1.0 1.0 125.0 213.0 0.0 2.0 125.0 1.0 1.4 1.0 1.0 3.0 1
-54.0 1.0 2.0 192.0 283.0 0.0 2.0 195.0 0.0 0.0 1.0 1.0 7.0 2
-53.0 1.0 4.0 123.0 282.0 0.0 0.0 95.0 1.0 2.0 2.0 2.0 7.0 2
-52.0 1.0 4.0 112.0 230.0 0.0 0.0 160.0 0.0 0.0 1.0 1.0 3.0 2
-40.0 1.0 4.0 110.0 167.0 0.0 2.0 114.0 1.0 2.0 2.0 0.0 7.0 2
-58.0 1.0 3.0 132.0 224.0 0.0 2.0 173.0 0.0 3.2 1.0 2.0 7.0 2
-41.0 0.0 3.0 112.0 268.0 0.0 2.0 172.0 1.0 0.0 1.0 0.0 3.0 1
-41.0 1.0 3.0 112.0 250.0 0.0 0.0 179.0 0.0 0.0 1.0 0.0 3.0 1
-50.0 0.0 3.0 120.0 219.0 0.0 0.0 158.0 0.0 1.6 2.0 0.0 3.0 1
-54.0 0.0 3.0 108.0 267.0 0.0 2.0 167.0 0.0 0.0 1.0 0.0 3.0 1
-64.0 0.0 4.0 130.0 303.0 0.0 0.0 122.0 0.0 2.0 2.0 2.0 3.0 1
-51.0 0.0 3.0 130.0 256.0 0.0 2.0 149.0 0.0 0.5 1.0 0.0 3.0 1
-46.0 0.0 2.0 105.0 204.0 0.0 0.0 172.0 0.0 0.0 1.0 0.0 3.0 1
-55.0 1.0 4.0 140.0 217.0 0.0 0.0 111.0 1.0 5.6 3.0 0.0 7.0 2
-45.0 1.0 2.0 128.0 308.0 0.0 2.0 170.0 0.0 0.0 1.0 0.0 3.0 1
-56.0 1.0 1.0 120.0 193.0 0.0 2.0 162.0 0.0 1.9 2.0 0.0 7.0 1
-66.0 0.0 4.0 178.0 228.0 1.0 0.0 165.0 1.0 1.0 2.0 2.0 7.0 2
-38.0 1.0 1.0 120.0 231.0 0.0 0.0 182.0 1.0 3.8 2.0 0.0 7.0 2
-62.0 0.0 4.0 150.0 244.0 0.0 0.0 154.0 1.0 1.4 2.0 0.0 3.0 2
-55.0 1.0 2.0 130.0 262.0 0.0 0.0 155.0 0.0 0.0 1.0 0.0 3.0 1
-58.0 1.0 4.0 128.0 259.0 0.0 2.0 130.0 1.0 3.0 2.0 2.0 7.0 2
-43.0 1.0 4.0 110.0 211.0 0.0 0.0 161.0 0.0 0.0 1.0 0.0 7.0 1
-64.0 0.0 4.0 180.0 325.0 0.0 0.0 154.0 1.0 0.0 1.0 0.0 3.0 1
-50.0 0.0 4.0 110.0 254.0 0.0 2.0 159.0 0.0 0.0 1.0 0.0 3.0 1
-53.0 1.0 3.0 130.0 197.0 1.0 2.0 152.0 0.0 1.2 3.0 0.0 3.0 1
-45.0 0.0 4.0 138.0 236.0 0.0 2.0 152.0 1.0 0.2 2.0 0.0 3.0 1
-65.0 1.0 1.0 138.0 282.0 1.0 2.0 174.0 0.0 1.4 2.0 1.0 3.0 2
-69.0 1.0 1.0 160.0 234.0 1.0 2.0 131.0 0.0 0.1 2.0 1.0 3.0 1
-69.0 1.0 3.0 140.0 254.0 0.0 2.0 146.0 0.0 2.0 2.0 3.0 7.0 2
-67.0 1.0 4.0 100.0 299.0 0.0 2.0 125.0 1.0 0.9 2.0 2.0 3.0 2
-68.0 0.0 3.0 120.0 211.0 0.0 2.0 115.0 0.0 1.5 2.0 0.0 3.0 1
-34.0 1.0 1.0 118.0 182.0 0.0 2.0 174.0 0.0 0.0 1.0 0.0 3.0 1
-62.0 0.0 4.0 138.0 294.0 1.0 0.0 106.0 0.0 1.9 2.0 3.0 3.0 2
-51.0 1.0 4.0 140.0 298.0 0.0 0.0 122.0 1.0 4.2 2.0 3.0 7.0 2
-46.0 1.0 3.0 150.0 231.0 0.0 0.0 147.0 0.0 3.6 2.0 0.0 3.0 2
-67.0 1.0 4.0 125.0 254.0 1.0 0.0 163.0 0.0 0.2 2.0 2.0 7.0 2
-50.0 1.0 3.0 129.0 196.0 0.0 0.0 163.0 0.0 0.0 1.0 0.0 3.0 1
-42.0 1.0 3.0 120.0 240.0 1.0 0.0 194.0 0.0 0.8 3.0 0.0 7.0 1
-56.0 0.0 4.0 134.0 409.0 0.0 2.0 150.0 1.0 1.9 2.0 2.0 7.0 2
-41.0 1.0 4.0 110.0 172.0 0.0 2.0 158.0 0.0 0.0 1.0 0.0 7.0 2
-42.0 0.0 4.0 102.0 265.0 0.0 2.0 122.0 0.0 0.6 2.0 0.0 3.0 1
-53.0 1.0 3.0 130.0 246.0 1.0 2.0 173.0 0.0 0.0 1.0 3.0 3.0 1
-43.0 1.0 3.0 130.0 315.0 0.0 0.0 162.0 0.0 1.9 1.0 1.0 3.0 1
-56.0 1.0 4.0 132.0 184.0 0.0 2.0 105.0 1.0 2.1 2.0 1.0 6.0 2
-52.0 1.0 4.0 108.0 233.0 1.0 0.0 147.0 0.0 0.1 1.0 3.0 7.0 1
-62.0 0.0 4.0 140.0 394.0 0.0 2.0 157.0 0.0 1.2 2.0 0.0 3.0 1
-70.0 1.0 3.0 160.0 269.0 0.0 0.0 112.0 1.0 2.9 2.0 1.0 7.0 2
-54.0 1.0 4.0 140.0 239.0 0.0 0.0 160.0 0.0 1.2 1.0 0.0 3.0 1
-70.0 1.0 4.0 145.0 174.0 0.0 0.0 125.0 1.0 2.6 3.0 0.0 7.0 2
-54.0 1.0 2.0 108.0 309.0 0.0 0.0 156.0 0.0 0.0 1.0 0.0 7.0 1
-35.0 1.0 4.0 126.0 282.0 0.0 2.0 156.0 1.0 0.0 1.0 0.0 7.0 2
-48.0 1.0 3.0 124.0 255.0 1.0 0.0 175.0 0.0 0.0 1.0 2.0 3.0 1
-55.0 0.0 2.0 135.0 250.0 0.0 2.0 161.0 0.0 1.4 2.0 0.0 3.0 1
-58.0 0.0 4.0 100.0 248.0 0.0 2.0 122.0 0.0 1.0 2.0 0.0 3.0 1
-54.0 0.0 3.0 110.0 214.0 0.0 0.0 158.0 0.0 1.6 2.0 0.0 3.0 1
-69.0 0.0 1.0 140.0 239.0 0.0 0.0 151.0 0.0 1.8 1.0 2.0 3.0 1
-77.0 1.0 4.0 125.0 304.0 0.0 2.0 162.0 1.0 0.0 1.0 3.0 3.0 2
-68.0 1.0 3.0 118.0 277.0 0.0 0.0 151.0 0.0 1.0 1.0 1.0 7.0 1
-58.0 1.0 4.0 125.0 300.0 0.0 2.0 171.0 0.0 0.0 1.0 2.0 7.0 2
-60.0 1.0 4.0 125.0 258.0 0.0 2.0 141.0 1.0 2.8 2.0 1.0 7.0 2
-51.0 1.0 4.0 140.0 299.0 0.0 0.0 173.0 1.0 1.6 1.0 0.0 7.0 2
-55.0 1.0 4.0 160.0 289.0 0.0 2.0 145.0 1.0 0.8 2.0 1.0 7.0 2
-52.0 1.0 1.0 152.0 298.0 1.0 0.0 178.0 0.0 1.2 2.0 0.0 7.0 1
-60.0 0.0 3.0 102.0 318.0 0.0 0.0 160.0 0.0 0.0 1.0 1.0 3.0 1
-58.0 1.0 3.0 105.0 240.0 0.0 2.0 154.0 1.0 0.6 2.0 0.0 7.0 1
-64.0 1.0 3.0 125.0 309.0 0.0 0.0 131.0 1.0 1.8 2.0 0.0 7.0 2
-37.0 1.0 3.0 130.0 250.0 0.0 0.0 187.0 0.0 3.5 3.0 0.0 3.0 1
-59.0 1.0 1.0 170.0 288.0 0.0 2.0 159.0 0.0 0.2 2.0 0.0 7.0 2
-51.0 1.0 3.0 125.0 245.0 1.0 2.0 166.0 0.0 2.4 2.0 0.0 3.0 1
-43.0 0.0 3.0 122.0 213.0 0.0 0.0 165.0 0.0 0.2 2.0 0.0 3.0 1
-58.0 1.0 4.0 128.0 216.0 0.0 2.0 131.0 1.0 2.2 2.0 3.0 7.0 2
-29.0 1.0 2.0 130.0 204.0 0.0 2.0 202.0 0.0 0.0 1.0 0.0 3.0 1
-41.0 0.0 2.0 130.0 204.0 0.0 2.0 172.0 0.0 1.4 1.0 0.0 3.0 1
-63.0 0.0 3.0 135.0 252.0 0.0 2.0 172.0 0.0 0.0 1.0 0.0 3.0 1
-51.0 1.0 3.0 94.0 227.0 0.0 0.0 154.0 1.0 0.0 1.0 1.0 7.0 1
-54.0 1.0 3.0 120.0 258.0 0.0 2.0 147.0 0.0 0.4 2.0 0.0 7.0 1
-44.0 1.0 2.0 120.0 220.0 0.0 0.0 170.0 0.0 0.0 1.0 0.0 3.0 1
-54.0 1.0 4.0 110.0 239.0 0.0 0.0 126.0 1.0 2.8 2.0 1.0 7.0 2
-65.0 1.0 4.0 135.0 254.0 0.0 2.0 127.0 0.0 2.8 2.0 1.0 7.0 2
-57.0 1.0 3.0 150.0 168.0 0.0 0.0 174.0 0.0 1.6 1.0 0.0 3.0 1
-63.0 1.0 4.0 130.0 330.0 1.0 2.0 132.0 1.0 1.8 1.0 3.0 7.0 2
-35.0 0.0 4.0 138.0 183.0 0.0 0.0 182.0 0.0 1.4 1.0 0.0 3.0 1
-41.0 1.0 2.0 135.0 203.0 0.0 0.0 132.0 0.0 0.0 2.0 0.0 6.0 1
-62.0 0.0 3.0 130.0 263.0 0.0 0.0 97.0 0.0 1.2 2.0 1.0 7.0 2
-43.0 0.0 4.0 132.0 341.0 1.0 2.0 136.0 1.0 3.0 2.0 0.0 7.0 2
-58.0 0.0 1.0 150.0 283.0 1.0 2.0 162.0 0.0 1.0 1.0 0.0 3.0 1
-52.0 1.0 1.0 118.0 186.0 0.0 2.0 190.0 0.0 0.0 2.0 0.0 6.0 1
-61.0 0.0 4.0 145.0 307.0 0.0 2.0 146.0 1.0 1.0 2.0 0.0 7.0 2
-39.0 1.0 4.0 118.0 219.0 0.0 0.0 140.0 0.0 1.2 2.0 0.0 7.0 2
-45.0 1.0 4.0 115.0 260.0 0.0 2.0 185.0 0.0 0.0 1.0 0.0 3.0 1
-52.0 1.0 4.0 128.0 255.0 0.0 0.0 161.0 1.0 0.0 1.0 1.0 7.0 2
-62.0 1.0 3.0 130.0 231.0 0.0 0.0 146.0 0.0 1.8 2.0 3.0 7.0 1
-62.0 0.0 4.0 160.0 164.0 0.0 2.0 145.0 0.0 6.2 3.0 3.0 7.0 2
-53.0 0.0 4.0 138.0 234.0 0.0 2.0 160.0 0.0 0.0 1.0 0.0 3.0 1
-43.0 1.0 4.0 120.0 177.0 0.0 2.0 120.0 1.0 2.5 2.0 0.0 7.0 2
-47.0 1.0 3.0 138.0 257.0 0.0 2.0 156.0 0.0 0.0 1.0 0.0 3.0 1
-52.0 1.0 2.0 120.0 325.0 0.0 0.0 172.0 0.0 0.2 1.0 0.0 3.0 1
-68.0 1.0 3.0 180.0 274.0 1.0 2.0 150.0 1.0 1.6 2.0 0.0 7.0 2
-39.0 1.0 3.0 140.0 321.0 0.0 2.0 182.0 0.0 0.0 1.0 0.0 3.0 1
-53.0 0.0 4.0 130.0 264.0 0.0 2.0 143.0 0.0 0.4 2.0 0.0 3.0 1
-62.0 0.0 4.0 140.0 268.0 0.0 2.0 160.0 0.0 3.6 3.0 2.0 3.0 2
-51.0 0.0 3.0 140.0 308.0 0.0 2.0 142.0 0.0 1.5 1.0 1.0 3.0 1
-60.0 1.0 4.0 130.0 253.0 0.0 0.0 144.0 1.0 1.4 1.0 1.0 7.0 2
-65.0 1.0 4.0 110.0 248.0 0.0 2.0 158.0 0.0 0.6 1.0 2.0 6.0 2
-65.0 0.0 3.0 155.0 269.0 0.0 0.0 148.0 0.0 0.8 1.0 0.0 3.0 1
-60.0 1.0 3.0 140.0 185.0 0.0 2.0 155.0 0.0 3.0 2.0 0.0 3.0 2
-60.0 1.0 4.0 145.0 282.0 0.0 2.0 142.0 1.0 2.8 2.0 2.0 7.0 2
-54.0 1.0 4.0 120.0 188.0 0.0 0.0 113.0 0.0 1.4 2.0 1.0 7.0 2
-44.0 1.0 2.0 130.0 219.0 0.0 2.0 188.0 0.0 0.0 1.0 0.0 3.0 1
-44.0 1.0 4.0 112.0 290.0 0.0 2.0 153.0 0.0 0.0 1.0 1.0 3.0 2
-51.0 1.0 3.0 110.0 175.0 0.0 0.0 123.0 0.0 0.6 1.0 0.0 3.0 1
-59.0 1.0 3.0 150.0 212.0 1.0 0.0 157.0 0.0 1.6 1.0 0.0 3.0 1
-71.0 0.0 2.0 160.0 302.0 0.0 0.0 162.0 0.0 0.4 1.0 2.0 3.0 1
-61.0 1.0 3.0 150.0 243.0 1.0 0.0 137.0 1.0 1.0 2.0 0.0 3.0 1
-55.0 1.0 4.0 132.0 353.0 0.0 0.0 132.0 1.0 1.2 2.0 1.0 7.0 2
-64.0 1.0 3.0 140.0 335.0 0.0 0.0 158.0 0.0 0.0 1.0 0.0 3.0 2
-43.0 1.0 4.0 150.0 247.0 0.0 0.0 171.0 0.0 1.5 1.0 0.0 3.0 1
-58.0 0.0 3.0 120.0 340.0 0.0 0.0 172.0 0.0 0.0 1.0 0.0 3.0 1
-60.0 1.0 4.0 130.0 206.0 0.0 2.0 132.0 1.0 2.4 2.0 2.0 7.0 2
-58.0 1.0 2.0 120.0 284.0 0.0 2.0 160.0 0.0 1.8 2.0 0.0 3.0 2
-49.0 1.0 2.0 130.0 266.0 0.0 0.0 171.0 0.0 0.6 1.0 0.0 3.0 1
-48.0 1.0 2.0 110.0 229.0 0.0 0.0 168.0 0.0 1.0 3.0 0.0 7.0 2
-52.0 1.0 3.0 172.0 199.0 1.0 0.0 162.0 0.0 0.5 1.0 0.0 7.0 1
-44.0 1.0 2.0 120.0 263.0 0.0 0.0 173.0 0.0 0.0 1.0 0.0 7.0 1
-56.0 0.0 2.0 140.0 294.0 0.0 2.0 153.0 0.0 1.3 2.0 0.0 3.0 1
-57.0 1.0 4.0 140.0 192.0 0.0 0.0 148.0 0.0 0.4 2.0 0.0 6.0 1
-67.0 1.0 4.0 160.0 286.0 0.0 2.0 108.0 1.0 1.5 2.0 3.0 3.0 2
-

GPy/util/datasets/mocap/cmu/README.txt

@@ -1 +0,0 @@
-this otherwise empty directory is for storing mnocap data, which we don;t distribute

GPy/util/datasets/sonar_data

@@ -1,209 +0,0 @@
-0.0200,0.0371,0.0428,0.0207,0.0954,0.0986,0.1539,0.1601,0.3109,0.2111,0.1609,0.1582,0.2238,0.0645,0.0660,0.2273,0.3100,0.2999,0.5078,0.4797,0.5783,0.5071,0.4328,0.5550,0.6711,0.6415,0.7104,0.8080,0.6791,0.3857,0.1307,0.2604,0.5121,0.7547,0.8537,0.8507,0.6692,0.6097,0.4943,0.2744,0.0510,0.2834,0.2825,0.4256,0.2641,0.1386,0.1051,0.1343,0.0383,0.0324,0.0232,0.0027,0.0065,0.0159,0.0072,0.0167,0.0180,0.0084,0.0090,0.0032,1
-0.0453,0.0523,0.0843,0.0689,0.1183,0.2583,0.2156,0.3481,0.3337,0.2872,0.4918,0.6552,0.6919,0.7797,0.7464,0.9444,1.0000,0.8874,0.8024,0.7818,0.5212,0.4052,0.3957,0.3914,0.3250,0.3200,0.3271,0.2767,0.4423,0.2028,0.3788,0.2947,0.1984,0.2341,0.1306,0.4182,0.3835,0.1057,0.1840,0.1970,0.1674,0.0583,0.1401,0.1628,0.0621,0.0203,0.0530,0.0742,0.0409,0.0061,0.0125,0.0084,0.0089,0.0048,0.0094,0.0191,0.0140,0.0049,0.0052,0.0044,1
-0.0262,0.0582,0.1099,0.1083,0.0974,0.2280,0.2431,0.3771,0.5598,0.6194,0.6333,0.7060,0.5544,0.5320,0.6479,0.6931,0.6759,0.7551,0.8929,0.8619,0.7974,0.6737,0.4293,0.3648,0.5331,0.2413,0.5070,0.8533,0.6036,0.8514,0.8512,0.5045,0.1862,0.2709,0.4232,0.3043,0.6116,0.6756,0.5375,0.4719,0.4647,0.2587,0.2129,0.2222,0.2111,0.0176,0.1348,0.0744,0.0130,0.0106,0.0033,0.0232,0.0166,0.0095,0.0180,0.0244,0.0316,0.0164,0.0095,0.0078,1
-0.0100,0.0171,0.0623,0.0205,0.0205,0.0368,0.1098,0.1276,0.0598,0.1264,0.0881,0.1992,0.0184,0.2261,0.1729,0.2131,0.0693,0.2281,0.4060,0.3973,0.2741,0.3690,0.5556,0.4846,0.3140,0.5334,0.5256,0.2520,0.2090,0.3559,0.6260,0.7340,0.6120,0.3497,0.3953,0.3012,0.5408,0.8814,0.9857,0.9167,0.6121,0.5006,0.3210,0.3202,0.4295,0.3654,0.2655,0.1576,0.0681,0.0294,0.0241,0.0121,0.0036,0.0150,0.0085,0.0073,0.0050,0.0044,0.0040,0.0117,1
-0.0762,0.0666,0.0481,0.0394,0.0590,0.0649,0.1209,0.2467,0.3564,0.4459,0.4152,0.3952,0.4256,0.4135,0.4528,0.5326,0.7306,0.6193,0.2032,0.4636,0.4148,0.4292,0.5730,0.5399,0.3161,0.2285,0.6995,1.0000,0.7262,0.4724,0.5103,0.5459,0.2881,0.0981,0.1951,0.4181,0.4604,0.3217,0.2828,0.2430,0.1979,0.2444,0.1847,0.0841,0.0692,0.0528,0.0357,0.0085,0.0230,0.0046,0.0156,0.0031,0.0054,0.0105,0.0110,0.0015,0.0072,0.0048,0.0107,0.0094,1
-0.0286,0.0453,0.0277,0.0174,0.0384,0.0990,0.1201,0.1833,0.2105,0.3039,0.2988,0.4250,0.6343,0.8198,1.0000,0.9988,0.9508,0.9025,0.7234,0.5122,0.2074,0.3985,0.5890,0.2872,0.2043,0.5782,0.5389,0.3750,0.3411,0.5067,0.5580,0.4778,0.3299,0.2198,0.1407,0.2856,0.3807,0.4158,0.4054,0.3296,0.2707,0.2650,0.0723,0.1238,0.1192,0.1089,0.0623,0.0494,0.0264,0.0081,0.0104,0.0045,0.0014,0.0038,0.0013,0.0089,0.0057,0.0027,0.0051,0.0062,1
-0.0317,0.0956,0.1321,0.1408,0.1674,0.1710,0.0731,0.1401,0.2083,0.3513,0.1786,0.0658,0.0513,0.3752,0.5419,0.5440,0.5150,0.4262,0.2024,0.4233,0.7723,0.9735,0.9390,0.5559,0.5268,0.6826,0.5713,0.5429,0.2177,0.2149,0.5811,0.6323,0.2965,0.1873,0.2969,0.5163,0.6153,0.4283,0.5479,0.6133,0.5017,0.2377,0.1957,0.1749,0.1304,0.0597,0.1124,0.1047,0.0507,0.0159,0.0195,0.0201,0.0248,0.0131,0.0070,0.0138,0.0092,0.0143,0.0036,0.0103,1
-0.0519,0.0548,0.0842,0.0319,0.1158,0.0922,0.1027,0.0613,0.1465,0.2838,0.2802,0.3086,0.2657,0.3801,0.5626,0.4376,0.2617,0.1199,0.6676,0.9402,0.7832,0.5352,0.6809,0.9174,0.7613,0.8220,0.8872,0.6091,0.2967,0.1103,0.1318,0.0624,0.0990,0.4006,0.3666,0.1050,0.1915,0.3930,0.4288,0.2546,0.1151,0.2196,0.1879,0.1437,0.2146,0.2360,0.1125,0.0254,0.0285,0.0178,0.0052,0.0081,0.0120,0.0045,0.0121,0.0097,0.0085,0.0047,0.0048,0.0053,1
-0.0223,0.0375,0.0484,0.0475,0.0647,0.0591,0.0753,0.0098,0.0684,0.1487,0.1156,0.1654,0.3833,0.3598,0.1713,0.1136,0.0349,0.3796,0.7401,0.9925,0.9802,0.8890,0.6712,0.4286,0.3374,0.7366,0.9611,0.7353,0.4856,0.1594,0.3007,0.4096,0.3170,0.3305,0.3408,0.2186,0.2463,0.2726,0.1680,0.2792,0.2558,0.1740,0.2121,0.1099,0.0985,0.1271,0.1459,0.1164,0.0777,0.0439,0.0061,0.0145,0.0128,0.0145,0.0058,0.0049,0.0065,0.0093,0.0059,0.0022,1
-0.0164,0.0173,0.0347,0.0070,0.0187,0.0671,0.1056,0.0697,0.0962,0.0251,0.0801,0.1056,0.1266,0.0890,0.0198,0.1133,0.2826,0.3234,0.3238,0.4333,0.6068,0.7652,0.9203,0.9719,0.9207,0.7545,0.8289,0.8907,0.7309,0.6896,0.5829,0.4935,0.3101,0.0306,0.0244,0.1108,0.1594,0.1371,0.0696,0.0452,0.0620,0.1421,0.1597,0.1384,0.0372,0.0688,0.0867,0.0513,0.0092,0.0198,0.0118,0.0090,0.0223,0.0179,0.0084,0.0068,0.0032,0.0035,0.0056,0.0040,1
-0.0039,0.0063,0.0152,0.0336,0.0310,0.0284,0.0396,0.0272,0.0323,0.0452,0.0492,0.0996,0.1424,0.1194,0.0628,0.0907,0.1177,0.1429,0.1223,0.1104,0.1847,0.3715,0.4382,0.5707,0.6654,0.7476,0.7654,0.8555,0.9720,0.9221,0.7502,0.7209,0.7757,0.6055,0.5021,0.4499,0.3947,0.4281,0.4427,0.3749,0.1972,0.0511,0.0793,0.1269,0.1533,0.0690,0.0402,0.0534,0.0228,0.0073,0.0062,0.0062,0.0120,0.0052,0.0056,0.0093,0.0042,0.0003,0.0053,0.0036,1
-0.0123,0.0309,0.0169,0.0313,0.0358,0.0102,0.0182,0.0579,0.1122,0.0835,0.0548,0.0847,0.2026,0.2557,0.1870,0.2032,0.1463,0.2849,0.5824,0.7728,0.7852,0.8515,0.5312,0.3653,0.5973,0.8275,1.0000,0.8673,0.6301,0.4591,0.3940,0.2576,0.2817,0.2641,0.2757,0.2698,0.3994,0.4576,0.3940,0.2522,0.1782,0.1354,0.0516,0.0337,0.0894,0.0861,0.0872,0.0445,0.0134,0.0217,0.0188,0.0133,0.0265,0.0224,0.0074,0.0118,0.0026,0.0092,0.0009,0.0044,1
-0.0079,0.0086,0.0055,0.0250,0.0344,0.0546,0.0528,0.0958,0.1009,0.1240,0.1097,0.1215,0.1874,0.3383,0.3227,0.2723,0.3943,0.6432,0.7271,0.8673,0.9674,0.9847,0.9480,0.8036,0.6833,0.5136,0.3090,0.0832,0.4019,0.2344,0.1905,0.1235,0.1717,0.2351,0.2489,0.3649,0.3382,0.1589,0.0989,0.1089,0.1043,0.0839,0.1391,0.0819,0.0678,0.0663,0.1202,0.0692,0.0152,0.0266,0.0174,0.0176,0.0127,0.0088,0.0098,0.0019,0.0059,0.0058,0.0059,0.0032,1
-0.0090,0.0062,0.0253,0.0489,0.1197,0.1589,0.1392,0.0987,0.0955,0.1895,0.1896,0.2547,0.4073,0.2988,0.2901,0.5326,0.4022,0.1571,0.3024,0.3907,0.3542,0.4438,0.6414,0.4601,0.6009,0.8690,0.8345,0.7669,0.5081,0.4620,0.5380,0.5375,0.3844,0.3601,0.7402,0.7761,0.3858,0.0667,0.3684,0.6114,0.3510,0.2312,0.2195,0.3051,0.1937,0.1570,0.0479,0.0538,0.0146,0.0068,0.0187,0.0059,0.0095,0.0194,0.0080,0.0152,0.0158,0.0053,0.0189,0.0102,1
-0.0124,0.0433,0.0604,0.0449,0.0597,0.0355,0.0531,0.0343,0.1052,0.2120,0.1640,0.1901,0.3026,0.2019,0.0592,0.2390,0.3657,0.3809,0.5929,0.6299,0.5801,0.4574,0.4449,0.3691,0.6446,0.8940,0.8978,0.4980,0.3333,0.2350,0.1553,0.3666,0.4340,0.3082,0.3024,0.4109,0.5501,0.4129,0.5499,0.5018,0.3132,0.2802,0.2351,0.2298,0.1155,0.0724,0.0621,0.0318,0.0450,0.0167,0.0078,0.0083,0.0057,0.0174,0.0188,0.0054,0.0114,0.0196,0.0147,0.0062,1
-0.0298,0.0615,0.0650,0.0921,0.1615,0.2294,0.2176,0.2033,0.1459,0.0852,0.2476,0.3645,0.2777,0.2826,0.3237,0.4335,0.5638,0.4555,0.4348,0.6433,0.3932,0.1989,0.3540,0.9165,0.9371,0.4620,0.2771,0.6613,0.8028,0.4200,0.5192,0.6962,0.5792,0.8889,0.7863,0.7133,0.7615,0.4401,0.3009,0.3163,0.2809,0.2898,0.0526,0.1867,0.1553,0.1633,0.1252,0.0748,0.0452,0.0064,0.0154,0.0031,0.0153,0.0071,0.0212,0.0076,0.0152,0.0049,0.0200,0.0073,1
-0.0352,0.0116,0.0191,0.0469,0.0737,0.1185,0.1683,0.1541,0.1466,0.2912,0.2328,0.2237,0.2470,0.1560,0.3491,0.3308,0.2299,0.2203,0.2493,0.4128,0.3158,0.6191,0.5854,0.3395,0.2561,0.5599,0.8145,0.6941,0.6985,0.8660,0.5930,0.3664,0.6750,0.8697,0.7837,0.7552,0.5789,0.4713,0.1252,0.6087,0.7322,0.5977,0.3431,0.1803,0.2378,0.3424,0.2303,0.0689,0.0216,0.0469,0.0426,0.0346,0.0158,0.0154,0.0109,0.0048,0.0095,0.0015,0.0073,0.0067,1
-0.0192,0.0607,0.0378,0.0774,0.1388,0.0809,0.0568,0.0219,0.1037,0.1186,0.1237,0.1601,0.3520,0.4479,0.3769,0.5761,0.6426,0.6790,0.7157,0.5466,0.5399,0.6362,0.7849,0.7756,0.5780,0.4862,0.4181,0.2457,0.0716,0.0613,0.1816,0.4493,0.5976,0.3785,0.2495,0.5771,0.8852,0.8409,0.3570,0.3133,0.6096,0.6378,0.2709,0.1419,0.1260,0.1288,0.0790,0.0829,0.0520,0.0216,0.0360,0.0331,0.0131,0.0120,0.0108,0.0024,0.0045,0.0037,0.0112,0.0075,1
-0.0270,0.0092,0.0145,0.0278,0.0412,0.0757,0.1026,0.1138,0.0794,0.1520,0.1675,0.1370,0.1361,0.1345,0.2144,0.5354,0.6830,0.5600,0.3093,0.3226,0.4430,0.5573,0.5782,0.6173,0.8132,0.9819,0.9823,0.9166,0.7423,0.7736,0.8473,0.7352,0.6671,0.6083,0.6239,0.5972,0.5715,0.5242,0.2924,0.1536,0.2003,0.2031,0.2207,0.1778,0.1353,0.1373,0.0749,0.0472,0.0325,0.0179,0.0045,0.0084,0.0010,0.0018,0.0068,0.0039,0.0120,0.0132,0.0070,0.0088,1
-0.0126,0.0149,0.0641,0.1732,0.2565,0.2559,0.2947,0.4110,0.4983,0.5920,0.5832,0.5419,0.5472,0.5314,0.4981,0.6985,0.8292,0.7839,0.8215,0.9363,1.0000,0.9224,0.7839,0.5470,0.4562,0.5922,0.5448,0.3971,0.0882,0.2385,0.2005,0.0587,0.2544,0.2009,0.0329,0.1547,0.1212,0.2446,0.3171,0.3195,0.3051,0.0836,0.1266,0.1381,0.1136,0.0516,0.0073,0.0278,0.0372,0.0121,0.0153,0.0092,0.0035,0.0098,0.0121,0.0006,0.0181,0.0094,0.0116,0.0063,1
-0.0473,0.0509,0.0819,0.1252,0.1783,0.3070,0.3008,0.2362,0.3830,0.3759,0.3021,0.2909,0.2301,0.1411,0.1582,0.2430,0.4474,0.5964,0.6744,0.7969,0.8319,0.7813,0.8626,0.7369,0.4122,0.2596,0.3392,0.3788,0.4488,0.6281,0.7449,0.7328,0.7704,0.7870,0.6048,0.5860,0.6385,0.7279,0.6286,0.5316,0.4069,0.1791,0.1625,0.2527,0.1903,0.1643,0.0604,0.0209,0.0436,0.0175,0.0107,0.0193,0.0118,0.0064,0.0042,0.0054,0.0049,0.0082,0.0028,0.0027,1
-0.0664,0.0575,0.0842,0.0372,0.0458,0.0771,0.0771,0.1130,0.2353,0.1838,0.2869,0.4129,0.3647,0.1984,0.2840,0.4039,0.5837,0.6792,0.6086,0.4858,0.3246,0.2013,0.2082,0.1686,0.2484,0.2736,0.2984,0.4655,0.6990,0.7474,0.7956,0.7981,0.6715,0.6942,0.7440,0.8169,0.8912,1.0000,0.8753,0.7061,0.6803,0.5898,0.4618,0.3639,0.1492,0.1216,0.1306,0.1198,0.0578,0.0235,0.0135,0.0141,0.0190,0.0043,0.0036,0.0026,0.0024,0.0162,0.0109,0.0079,1
-0.0099,0.0484,0.0299,0.0297,0.0652,0.1077,0.2363,0.2385,0.0075,0.1882,0.1456,0.1892,0.3176,0.1340,0.2169,0.2458,0.2589,0.2786,0.2298,0.0656,0.1441,0.1179,0.1668,0.1783,0.2476,0.2570,0.1036,0.5356,0.7124,0.6291,0.4756,0.6015,0.7208,0.6234,0.5725,0.7523,0.8712,0.9252,0.9709,0.9297,0.8995,0.7911,0.5600,0.2838,0.4407,0.5507,0.4331,0.2905,0.1981,0.0779,0.0396,0.0173,0.0149,0.0115,0.0202,0.0139,0.0029,0.0160,0.0106,0.0134,1
-0.0115,0.0150,0.0136,0.0076,0.0211,0.1058,0.1023,0.0440,0.0931,0.0734,0.0740,0.0622,0.1055,0.1183,0.1721,0.2584,0.3232,0.3817,0.4243,0.4217,0.4449,0.4075,0.3306,0.4012,0.4466,0.5218,0.7552,0.9503,1.0000,0.9084,0.8283,0.7571,0.7262,0.6152,0.5680,0.5757,0.5324,0.3672,0.1669,0.0866,0.0646,0.1891,0.2683,0.2887,0.2341,0.1668,0.1015,0.1195,0.0704,0.0167,0.0107,0.0091,0.0016,0.0084,0.0064,0.0026,0.0029,0.0037,0.0070,0.0041,1
-0.0293,0.0644,0.0390,0.0173,0.0476,0.0816,0.0993,0.0315,0.0736,0.0860,0.0414,0.0472,0.0835,0.0938,0.1466,0.0809,0.1179,0.2179,0.3326,0.3258,0.2111,0.2302,0.3361,0.4259,0.4609,0.2606,0.0874,0.2862,0.5606,0.8344,0.8096,0.7250,0.8048,0.9435,1.0000,0.8960,0.5516,0.3037,0.2338,0.2382,0.3318,0.3821,0.1575,0.2228,0.1582,0.1433,0.1634,0.1133,0.0567,0.0133,0.0170,0.0035,0.0052,0.0083,0.0078,0.0075,0.0105,0.0160,0.0095,0.0011,1
-0.0201,0.0026,0.0138,0.0062,0.0133,0.0151,0.0541,0.0210,0.0505,0.1097,0.0841,0.0942,0.1204,0.0420,0.0031,0.0162,0.0624,0.2127,0.3436,0.3813,0.3825,0.4764,0.6313,0.7523,0.8675,0.8788,0.7901,0.8357,0.9631,0.9619,0.9236,0.8903,0.9708,0.9647,0.7892,0.5307,0.2718,0.1953,0.1374,0.3105,0.3790,0.4105,0.3355,0.2998,0.2748,0.2024,0.1043,0.0453,0.0337,0.0122,0.0072,0.0108,0.0070,0.0063,0.0030,0.0011,0.0007,0.0024,0.0057,0.0044,1
-0.0151,0.0320,0.0599,0.1050,0.1163,0.1734,0.1679,0.1119,0.0889,0.1205,0.0847,0.1518,0.2305,0.2793,0.3404,0.4527,0.6950,0.8807,0.9154,0.7542,0.6736,0.7146,0.8335,0.7701,0.6993,0.6543,0.5040,0.4926,0.4992,0.4161,0.1631,0.0404,0.0637,0.2962,0.3609,0.1866,0.0476,0.1497,0.2405,0.1980,0.3175,0.2379,0.1716,0.1559,0.1556,0.0422,0.0493,0.0476,0.0219,0.0059,0.0086,0.0061,0.0015,0.0084,0.0128,0.0054,0.0011,0.0019,0.0023,0.0062,1
-0.0177,0.0300,0.0288,0.0394,0.0630,0.0526,0.0688,0.0633,0.0624,0.0613,0.1680,0.3476,0.4561,0.5188,0.6308,0.7201,0.5153,0.3818,0.2644,0.3345,0.4865,0.6628,0.7389,0.9213,1.0000,0.7750,0.5593,0.6172,0.8635,0.6592,0.4770,0.4983,0.3330,0.3076,0.2876,0.2226,0.0794,0.0603,0.1049,0.0606,0.1530,0.0983,0.1643,0.1901,0.1107,0.1917,0.1467,0.0392,0.0356,0.0270,0.0168,0.0102,0.0122,0.0044,0.0075,0.0124,0.0099,0.0057,0.0032,0.0019,1
-0.0100,0.0275,0.0190,0.0371,0.0416,0.0201,0.0314,0.0651,0.1896,0.2668,0.3376,0.3282,0.2432,0.1268,0.1278,0.4441,0.6795,0.7051,0.7966,0.9401,0.9857,0.8193,0.5789,0.6394,0.7043,0.6875,0.4081,0.1811,0.2064,0.3917,0.3791,0.2042,0.2227,0.3341,0.3984,0.5077,0.5534,0.3352,0.2723,0.2278,0.2044,0.1986,0.0835,0.0908,0.1380,0.1948,0.1211,0.0843,0.0589,0.0247,0.0118,0.0088,0.0104,0.0036,0.0088,0.0047,0.0117,0.0020,0.0091,0.0058,1
-0.0189,0.0308,0.0197,0.0622,0.0080,0.0789,0.1440,0.1451,0.1789,0.2522,0.2607,0.3710,0.3906,0.2672,0.2716,0.4183,0.6988,0.5733,0.2226,0.2631,0.7473,0.7263,0.3393,0.2824,0.6053,0.5897,0.4967,0.8616,0.8339,0.4084,0.2268,0.1745,0.0507,0.1588,0.3040,0.1369,0.1605,0.2061,0.0734,0.0202,0.1638,0.1583,0.1830,0.1886,0.1008,0.0663,0.0183,0.0404,0.0108,0.0143,0.0091,0.0038,0.0096,0.0142,0.0190,0.0140,0.0099,0.0092,0.0052,0.0075,1
-0.0240,0.0218,0.0324,0.0569,0.0330,0.0513,0.0897,0.0713,0.0569,0.0389,0.1934,0.2434,0.2906,0.2606,0.3811,0.4997,0.3015,0.3655,0.6791,0.7307,0.5053,0.4441,0.6987,0.8133,0.7781,0.8943,0.8929,0.8913,0.8610,0.8063,0.5540,0.2446,0.3459,0.1615,0.2467,0.5564,0.4681,0.0979,0.1582,0.0751,0.3321,0.3745,0.2666,0.1078,0.1418,0.1687,0.0738,0.0634,0.0144,0.0226,0.0061,0.0162,0.0146,0.0093,0.0112,0.0094,0.0054,0.0019,0.0066,0.0023,1
-0.0084,0.0153,0.0291,0.0432,0.0951,0.0752,0.0414,0.0259,0.0692,0.1753,0.1970,0.1167,0.1683,0.0814,0.2179,0.5121,0.7231,0.7776,0.6222,0.3501,0.3733,0.2622,0.3776,0.7361,0.8673,0.8223,0.7772,0.7862,0.5652,0.3635,0.3534,0.3865,0.3370,0.1693,0.2627,0.3195,0.1388,0.1048,0.1681,0.1910,0.1174,0.0933,0.0856,0.0951,0.0986,0.0956,0.0426,0.0407,0.0106,0.0179,0.0056,0.0236,0.0114,0.0136,0.0117,0.0060,0.0058,0.0031,0.0072,0.0045,1
-0.0195,0.0213,0.0058,0.0190,0.0319,0.0571,0.1004,0.0668,0.0691,0.0242,0.0728,0.0639,0.3002,0.3854,0.4767,0.4602,0.3175,0.4160,0.6428,1.0000,0.8631,0.5212,0.3156,0.5952,0.7732,0.6042,0.4375,0.5487,0.4720,0.6235,0.3851,0.1590,0.3891,0.5294,0.3504,0.4480,0.4041,0.5031,0.6475,0.5493,0.3548,0.2028,0.1882,0.0845,0.1315,0.1590,0.0562,0.0617,0.0343,0.0370,0.0261,0.0157,0.0074,0.0271,0.0203,0.0089,0.0095,0.0095,0.0021,0.0053,1
-0.0442,0.0477,0.0049,0.0581,0.0278,0.0678,0.1664,0.1490,0.0974,0.1268,0.1109,0.2375,0.2007,0.2140,0.1109,0.2036,0.2468,0.6682,0.8345,0.8252,0.8017,0.8982,0.9664,0.8515,0.6626,0.3241,0.2054,0.5669,0.5726,0.4877,0.7532,0.7600,0.5185,0.4120,0.5560,0.5569,0.1336,0.3831,0.4611,0.4330,0.2556,0.1466,0.3489,0.2659,0.0944,0.1370,0.1344,0.0416,0.0719,0.0637,0.0210,0.0204,0.0216,0.0135,0.0055,0.0073,0.0080,0.0105,0.0059,0.0105,1
-0.0311,0.0491,0.0692,0.0831,0.0079,0.0200,0.0981,0.1016,0.2025,0.0767,0.1767,0.2555,0.2812,0.2722,0.3227,0.3463,0.5395,0.7911,0.9064,0.8701,0.7672,0.2957,0.4148,0.6043,0.3178,0.3482,0.6158,0.8049,0.6289,0.4999,0.5830,0.6660,0.4124,0.1260,0.2487,0.4676,0.5382,0.3150,0.2139,0.1848,0.1679,0.2328,0.1015,0.0713,0.0615,0.0779,0.0761,0.0845,0.0592,0.0068,0.0089,0.0087,0.0032,0.0130,0.0188,0.0101,0.0229,0.0182,0.0046,0.0038,1
-0.0206,0.0132,0.0533,0.0569,0.0647,0.1432,0.1344,0.2041,0.1571,0.1573,0.2327,0.1785,0.1507,0.1916,0.2061,0.2307,0.2360,0.1299,0.3812,0.5858,0.4497,0.4876,1.0000,0.8675,0.4718,0.5341,0.6197,0.7143,0.5605,0.3728,0.2481,0.1921,0.1386,0.3325,0.2883,0.3228,0.2607,0.2040,0.2396,0.1319,0.0683,0.0334,0.0716,0.0976,0.0787,0.0522,0.0500,0.0231,0.0221,0.0144,0.0307,0.0386,0.0147,0.0018,0.0100,0.0096,0.0077,0.0180,0.0109,0.0070,1
-0.0094,0.0166,0.0398,0.0359,0.0681,0.0706,0.1020,0.0893,0.0381,0.1328,0.1303,0.0273,0.0644,0.0712,0.1204,0.0717,0.1224,0.2349,0.3684,0.3918,0.4925,0.8793,0.9606,0.8786,0.6905,0.6937,0.5674,0.6540,0.7802,0.7575,0.5836,0.6316,0.8108,0.9039,0.8647,0.6695,0.4027,0.2370,0.2685,0.3662,0.3267,0.2200,0.2996,0.2205,0.1163,0.0635,0.0465,0.0422,0.0174,0.0172,0.0134,0.0141,0.0191,0.0145,0.0065,0.0129,0.0217,0.0087,0.0077,0.0122,1
-0.0333,0.0221,0.0270,0.0481,0.0679,0.0981,0.0843,0.1172,0.0759,0.0920,0.1475,0.0522,0.1119,0.0970,0.1174,0.1678,0.1642,0.1205,0.0494,0.1544,0.3485,0.6146,0.9146,0.9364,0.8677,0.8772,0.8553,0.8833,1.0000,0.8296,0.6601,0.5499,0.5716,0.6859,0.6825,0.5142,0.2750,0.1358,0.1551,0.2646,0.1994,0.1883,0.2746,0.1651,0.0575,0.0695,0.0598,0.0456,0.0021,0.0068,0.0036,0.0022,0.0032,0.0060,0.0054,0.0063,0.0143,0.0132,0.0051,0.0041,1
-0.0123,0.0022,0.0196,0.0206,0.0180,0.0492,0.0033,0.0398,0.0791,0.0475,0.1152,0.0520,0.1192,0.1943,0.1840,0.2077,0.1956,0.1630,0.1218,0.1017,0.1354,0.3157,0.4645,0.5906,0.6776,0.8119,0.8594,0.9228,0.8387,0.7238,0.6292,0.5181,0.4629,0.5255,0.5147,0.3929,0.1279,0.0411,0.0859,0.1131,0.1306,0.1757,0.2648,0.1955,0.0656,0.0580,0.0319,0.0301,0.0272,0.0074,0.0149,0.0125,0.0134,0.0026,0.0038,0.0018,0.0113,0.0058,0.0047,0.0071,1
-0.0091,0.0213,0.0206,0.0505,0.0657,0.0795,0.0970,0.0872,0.0743,0.0837,0.1579,0.0898,0.0309,0.1856,0.2969,0.2032,0.1264,0.1655,0.1661,0.2091,0.2310,0.4460,0.6634,0.6933,0.7663,0.8206,0.7049,0.7560,0.7466,0.6387,0.4846,0.3328,0.5356,0.8741,0.8573,0.6718,0.3446,0.3150,0.2702,0.2598,0.2742,0.3594,0.4382,0.2460,0.0758,0.0187,0.0797,0.0748,0.0367,0.0155,0.0300,0.0112,0.0112,0.0102,0.0026,0.0097,0.0098,0.0043,0.0071,0.0108,1
-0.0068,0.0232,0.0513,0.0444,0.0249,0.0637,0.0422,0.1130,0.1911,0.2475,0.1606,0.0922,0.2398,0.3220,0.4295,0.2652,0.0666,0.1442,0.2373,0.2595,0.2493,0.3903,0.6384,0.8037,0.7026,0.6874,0.6997,0.8558,1.0000,0.9621,0.8996,0.7575,0.6902,0.5686,0.4396,0.4546,0.2959,0.1587,0.1681,0.0842,0.1173,0.1754,0.2728,0.1705,0.0194,0.0213,0.0354,0.0420,0.0093,0.0204,0.0199,0.0173,0.0163,0.0055,0.0045,0.0068,0.0041,0.0052,0.0194,0.0105,1
-0.0093,0.0185,0.0056,0.0064,0.0260,0.0458,0.0470,0.0057,0.0425,0.0640,0.0888,0.1599,0.1541,0.2768,0.2176,0.2799,0.3491,0.2824,0.2479,0.3005,0.4300,0.4684,0.4520,0.5026,0.6217,0.6571,0.6632,0.7321,0.8534,1.0000,0.8448,0.6354,0.6308,0.6211,0.6976,0.5868,0.4889,0.3683,0.2043,0.1469,0.2220,0.1449,0.1490,0.1211,0.1144,0.0791,0.0365,0.0152,0.0085,0.0120,0.0022,0.0069,0.0064,0.0129,0.0114,0.0054,0.0089,0.0050,0.0058,0.0025,1
-0.0211,0.0319,0.0415,0.0286,0.0121,0.0438,0.1299,0.1390,0.0695,0.0568,0.0869,0.1935,0.1478,0.1871,0.1994,0.3283,0.6861,0.5814,0.2500,0.1734,0.3363,0.5588,0.6592,0.7012,0.8099,0.8901,0.8745,0.7887,0.8725,0.9376,0.8920,0.7508,0.6832,0.7610,0.9017,1.0000,0.9123,0.7388,0.5915,0.4057,0.3019,0.2331,0.2931,0.2298,0.2391,0.1910,0.1096,0.0300,0.0171,0.0383,0.0053,0.0090,0.0042,0.0153,0.0106,0.0020,0.0105,0.0049,0.0070,0.0080,1
-0.0093,0.0269,0.0217,0.0339,0.0305,0.1172,0.1450,0.0638,0.0740,0.1360,0.2132,0.3738,0.3738,0.2673,0.2333,0.5367,0.7312,0.7659,0.6271,0.4395,0.4330,0.4326,0.5544,0.7360,0.8589,0.8989,0.9420,0.9401,0.9379,0.8575,0.7284,0.6700,0.7547,0.8773,0.9919,0.9922,0.9419,0.8388,0.6605,0.4816,0.2917,0.1769,0.1136,0.0701,0.1578,0.1938,0.1106,0.0693,0.0176,0.0205,0.0309,0.0212,0.0091,0.0056,0.0086,0.0092,0.0070,0.0116,0.0060,0.0110,1
-0.0257,0.0447,0.0388,0.0239,0.1315,0.1323,0.1608,0.2145,0.0847,0.0561,0.0891,0.0861,0.1531,0.1524,0.1849,0.2871,0.2009,0.2748,0.5017,0.2172,0.4978,0.5265,0.3647,0.5768,0.5161,0.5715,0.4006,0.3650,0.6685,0.8659,0.8052,0.4082,0.3379,0.5092,0.6776,0.7313,0.6062,0.7040,0.8849,0.8979,0.7751,0.7247,0.7733,0.7762,0.6009,0.4514,0.3096,0.1859,0.0956,0.0206,0.0206,0.0096,0.0153,0.0096,0.0131,0.0198,0.0025,0.0199,0.0255,0.0180,1
-0.0408,0.0653,0.0397,0.0604,0.0496,0.1817,0.1178,0.1024,0.0583,0.2176,0.2459,0.3332,0.3087,0.2613,0.3232,0.3731,0.4203,0.5364,0.7062,0.8196,0.8835,0.8299,0.7609,0.7605,0.8367,0.8905,0.7652,0.5897,0.3037,0.0823,0.2787,0.7241,0.8032,0.8050,0.7676,0.7468,0.6253,0.1730,0.2916,0.5003,0.5220,0.4824,0.4004,0.3877,0.1651,0.0442,0.0663,0.0418,0.0475,0.0235,0.0066,0.0062,0.0129,0.0184,0.0069,0.0198,0.0199,0.0102,0.0070,0.0055,1
-0.0308,0.0339,0.0202,0.0889,0.1570,0.1750,0.0920,0.1353,0.1593,0.2795,0.3336,0.2940,0.1608,0.3335,0.4985,0.7295,0.7350,0.8253,0.8793,0.9657,1.0000,0.8707,0.6471,0.5973,0.8218,0.7755,0.6111,0.4195,0.2990,0.1354,0.2438,0.5624,0.5555,0.6963,0.7298,0.7022,0.5468,0.1421,0.4738,0.6410,0.4375,0.3178,0.2377,0.2808,0.1374,0.1136,0.1034,0.0688,0.0422,0.0117,0.0070,0.0167,0.0127,0.0138,0.0090,0.0051,0.0029,0.0122,0.0056,0.0020,1
-0.0373,0.0281,0.0232,0.0225,0.0179,0.0733,0.0841,0.1031,0.0993,0.0802,0.1564,0.2565,0.2624,0.1179,0.0597,0.1563,0.2241,0.3586,0.1792,0.3256,0.6079,0.6988,0.8391,0.8553,0.7710,0.6215,0.5736,0.4402,0.4056,0.4411,0.5130,0.5965,0.7272,0.6539,0.5902,0.5393,0.4897,0.4081,0.4145,0.6003,0.7196,0.6633,0.6287,0.4087,0.3212,0.2518,0.1482,0.0988,0.0317,0.0269,0.0066,0.0008,0.0045,0.0024,0.0006,0.0073,0.0096,0.0054,0.0085,0.0060,1
-0.0190,0.0038,0.0642,0.0452,0.0333,0.0690,0.0901,0.1454,0.0740,0.0349,0.1459,0.3473,0.3197,0.2823,0.0166,0.0572,0.2164,0.4563,0.3819,0.5627,0.6484,0.7235,0.8242,0.8766,1.0000,0.8582,0.6563,0.5087,0.4817,0.4530,0.4521,0.4532,0.5385,0.5308,0.5356,0.5271,0.4260,0.2436,0.1205,0.3845,0.4107,0.5067,0.4216,0.2479,0.1586,0.1124,0.0651,0.0789,0.0325,0.0070,0.0026,0.0093,0.0118,0.0112,0.0094,0.0140,0.0072,0.0022,0.0055,0.0122,1
-0.0119,0.0582,0.0623,0.0600,0.1397,0.1883,0.1422,0.1447,0.0487,0.0864,0.2143,0.3720,0.2665,0.2113,0.1103,0.1136,0.1934,0.4142,0.3279,0.6222,0.7468,0.7676,0.7867,0.8253,1.0000,0.9481,0.7539,0.6008,0.5437,0.5387,0.5619,0.5141,0.6084,0.5621,0.5956,0.6078,0.5025,0.2829,0.0477,0.2811,0.3422,0.5147,0.4372,0.2470,0.1708,0.1343,0.0838,0.0755,0.0304,0.0074,0.0069,0.0025,0.0103,0.0074,0.0123,0.0069,0.0076,0.0073,0.0030,0.0138,1
-0.0353,0.0713,0.0326,0.0272,0.0370,0.0792,0.1083,0.0687,0.0298,0.0880,0.1078,0.0979,0.2250,0.2819,0.2099,0.1240,0.1699,0.0939,0.1091,0.1410,0.1268,0.3151,0.1430,0.2264,0.5756,0.7876,0.7158,0.5998,0.5583,0.6295,0.7659,0.8940,0.8436,0.6807,0.8380,1.0000,0.9497,0.7866,0.5647,0.3480,0.2585,0.2304,0.2948,0.3363,0.3017,0.2193,0.1316,0.1078,0.0559,0.0035,0.0098,0.0163,0.0242,0.0043,0.0202,0.0108,0.0037,0.0096,0.0093,0.0053,1
-0.0131,0.0068,0.0308,0.0311,0.0085,0.0767,0.0771,0.0640,0.0726,0.0901,0.0750,0.0844,0.1226,0.1619,0.2317,0.2934,0.3526,0.3657,0.3221,0.3093,0.4084,0.4285,0.4663,0.5956,0.6948,0.8386,0.8875,0.6404,0.3308,0.3425,0.4920,0.4592,0.3034,0.4366,0.5175,0.5122,0.4746,0.4902,0.4603,0.4460,0.4196,0.2873,0.2296,0.0949,0.0095,0.0527,0.0383,0.0107,0.0108,0.0077,0.0109,0.0062,0.0028,0.0040,0.0075,0.0039,0.0053,0.0013,0.0052,0.0023,1
-0.0087,0.0046,0.0081,0.0230,0.0586,0.0682,0.0993,0.0717,0.0576,0.0818,0.1315,0.1862,0.2789,0.2579,0.2240,0.2568,0.2933,0.2991,0.3924,0.4691,0.5665,0.6464,0.6774,0.7577,0.8856,0.9419,1.0000,0.8564,0.6790,0.5587,0.4147,0.2946,0.2025,0.0688,0.1171,0.2157,0.2216,0.2776,0.2309,0.1444,0.1513,0.1745,0.1756,0.1424,0.0908,0.0138,0.0469,0.0480,0.0159,0.0045,0.0015,0.0052,0.0038,0.0079,0.0114,0.0050,0.0030,0.0064,0.0058,0.0030,1
-0.0293,0.0378,0.0257,0.0062,0.0130,0.0612,0.0895,0.1107,0.0973,0.0751,0.0528,0.1209,0.1763,0.2039,0.2727,0.2321,0.2676,0.2934,0.3295,0.4910,0.5402,0.6257,0.6826,0.7527,0.8504,0.8938,0.9928,0.9134,0.7080,0.6318,0.6126,0.4638,0.2797,0.1721,0.1665,0.2561,0.2735,0.3209,0.2724,0.1880,0.1552,0.2522,0.2121,0.1801,0.1473,0.0681,0.1091,0.0919,0.0397,0.0093,0.0076,0.0065,0.0072,0.0108,0.0051,0.0102,0.0041,0.0055,0.0050,0.0087,1
-0.0132,0.0080,0.0188,0.0141,0.0436,0.0668,0.0609,0.0131,0.0899,0.0922,0.1445,0.1475,0.2087,0.2558,0.2603,0.1985,0.2394,0.3134,0.4077,0.4529,0.4893,0.5666,0.6234,0.6741,0.8282,0.8823,0.9196,0.8965,0.7549,0.6736,0.6463,0.5007,0.3663,0.2298,0.1362,0.2123,0.2395,0.2673,0.2865,0.2060,0.1659,0.2633,0.2552,0.1696,0.1467,0.1286,0.0926,0.0716,0.0325,0.0258,0.0136,0.0044,0.0028,0.0021,0.0022,0.0048,0.0138,0.0140,0.0028,0.0064,1
-0.0201,0.0116,0.0123,0.0245,0.0547,0.0208,0.0891,0.0836,0.1335,0.1199,0.1742,0.1387,0.2042,0.2580,0.2616,0.2097,0.2532,0.3213,0.4327,0.4760,0.5328,0.6057,0.6696,0.7476,0.8930,0.9405,1.0000,0.9785,0.8473,0.7639,0.6701,0.4989,0.3718,0.2196,0.1416,0.2680,0.2630,0.3104,0.3392,0.2123,0.1170,0.2655,0.2203,0.1541,0.1464,0.1044,0.1225,0.0745,0.0490,0.0224,0.0032,0.0076,0.0045,0.0056,0.0075,0.0037,0.0045,0.0029,0.0008,0.0018,1
-0.0152,0.0102,0.0113,0.0263,0.0097,0.0391,0.0857,0.0915,0.0949,0.1504,0.1911,0.2115,0.2249,0.2573,0.1701,0.2023,0.2538,0.3417,0.4026,0.4553,0.5525,0.5991,0.5854,0.7114,0.9500,0.9858,1.0000,0.9578,0.8642,0.7128,0.5893,0.4323,0.2897,0.1744,0.0770,0.2297,0.2459,0.3101,0.3312,0.2220,0.0871,0.2064,0.1808,0.1624,0.1120,0.0815,0.1117,0.0950,0.0412,0.0120,0.0048,0.0049,0.0041,0.0036,0.0013,0.0046,0.0037,0.0011,0.0034,0.0033,1
-0.0216,0.0124,0.0174,0.0152,0.0608,0.1026,0.1139,0.0877,0.1160,0.0866,0.1564,0.0780,0.0997,0.0915,0.0662,0.1134,0.1740,0.2573,0.3294,0.3910,0.5438,0.6115,0.7022,0.7610,0.7973,0.9105,0.8807,0.7949,0.7990,0.7180,0.6407,0.6312,0.5929,0.6168,0.6498,0.6764,0.6253,0.5117,0.3890,0.3273,0.2509,0.1530,0.1323,0.1657,0.1215,0.0978,0.0452,0.0273,0.0179,0.0092,0.0018,0.0052,0.0049,0.0096,0.0134,0.0122,0.0047,0.0018,0.0006,0.0023,1
-0.0225,0.0019,0.0075,0.0097,0.0445,0.0906,0.0889,0.0655,0.1624,0.1452,0.1442,0.0948,0.0618,0.1641,0.0708,0.0844,0.2590,0.2679,0.3094,0.4678,0.5958,0.7245,0.8773,0.9214,0.9282,0.9942,1.0000,0.9071,0.8545,0.7293,0.6499,0.6071,0.5588,0.5967,0.6275,0.5459,0.4786,0.3965,0.2087,0.1651,0.1836,0.0652,0.0758,0.0486,0.0353,0.0297,0.0241,0.0379,0.0119,0.0073,0.0051,0.0034,0.0129,0.0100,0.0044,0.0057,0.0030,0.0035,0.0021,0.0027,1
-0.0125,0.0152,0.0218,0.0175,0.0362,0.0696,0.0873,0.0616,0.1252,0.1302,0.0888,0.0500,0.0628,0.1274,0.0801,0.0742,0.2048,0.2950,0.3193,0.4567,0.5959,0.7101,0.8225,0.8425,0.9065,0.9802,1.0000,0.8752,0.7583,0.6616,0.5786,0.5128,0.4776,0.4994,0.5197,0.5071,0.4577,0.3505,0.1845,0.1890,0.1967,0.1041,0.0550,0.0492,0.0622,0.0505,0.0247,0.0219,0.0102,0.0047,0.0019,0.0041,0.0074,0.0030,0.0050,0.0048,0.0017,0.0041,0.0086,0.0058,1
-0.0130,0.0006,0.0088,0.0456,0.0525,0.0778,0.0931,0.0941,0.1711,0.1483,0.1532,0.1100,0.0890,0.1236,0.1197,0.1145,0.2137,0.2838,0.3640,0.5430,0.6673,0.7979,0.9273,0.9027,0.9192,1.0000,0.9821,0.9092,0.8184,0.6962,0.5900,0.5447,0.5142,0.5389,0.5531,0.5318,0.4826,0.3790,0.1831,0.1750,0.1679,0.0674,0.0609,0.0375,0.0533,0.0278,0.0179,0.0114,0.0073,0.0116,0.0092,0.0078,0.0041,0.0013,0.0011,0.0045,0.0039,0.0022,0.0023,0.0016,1
-0.0135,0.0045,0.0051,0.0289,0.0561,0.0929,0.1031,0.0883,0.1596,0.1908,0.1576,0.1112,0.1197,0.1174,0.1415,0.2215,0.2658,0.2713,0.3862,0.5717,0.6797,0.8747,1.0000,0.8948,0.8420,0.9174,0.9307,0.9050,0.8228,0.6986,0.5831,0.4924,0.4563,0.5159,0.5670,0.5284,0.5144,0.3742,0.2282,0.1193,0.1088,0.0431,0.1070,0.0583,0.0046,0.0473,0.0408,0.0290,0.0192,0.0094,0.0025,0.0037,0.0084,0.0102,0.0096,0.0024,0.0037,0.0028,0.0030,0.0030,1
-0.0086,0.0215,0.0242,0.0445,0.0667,0.0771,0.0499,0.0906,0.1229,0.1185,0.0775,0.1101,0.1042,0.0853,0.0456,0.1304,0.2690,0.2947,0.3669,0.4948,0.6275,0.8162,0.9237,0.8710,0.8052,0.8756,1.0000,0.9858,0.9427,0.8114,0.6987,0.6810,0.6591,0.6954,0.7290,0.6680,0.5917,0.4899,0.3439,0.2366,0.1716,0.1013,0.0766,0.0845,0.0260,0.0333,0.0205,0.0309,0.0101,0.0095,0.0047,0.0072,0.0054,0.0022,0.0016,0.0029,0.0058,0.0050,0.0024,0.0030,1
-0.0067,0.0096,0.0024,0.0058,0.0197,0.0618,0.0432,0.0951,0.0836,0.1180,0.0978,0.0909,0.0656,0.0593,0.0832,0.1297,0.2038,0.3811,0.4451,0.5224,0.5911,0.6566,0.6308,0.5998,0.4958,0.5647,0.6906,0.8513,1.0000,0.9166,0.7676,0.6177,0.5468,0.5516,0.5463,0.5515,0.4561,0.3466,0.3384,0.2853,0.2502,0.1641,0.1605,0.1491,0.1326,0.0687,0.0602,0.0561,0.0306,0.0154,0.0029,0.0048,0.0023,0.0020,0.0040,0.0019,0.0034,0.0034,0.0051,0.0031,1
-0.0071,0.0103,0.0135,0.0494,0.0253,0.0806,0.0701,0.0738,0.0117,0.0898,0.0289,0.1554,0.1437,0.1035,0.1424,0.1227,0.0892,0.2047,0.0827,0.1524,0.3031,0.1608,0.0667,0.1426,0.0395,0.1653,0.3399,0.4855,0.5206,0.5508,0.6102,0.5989,0.6764,0.8897,1.0000,0.9517,0.8459,0.7073,0.6697,0.6326,0.5102,0.4161,0.2816,0.1705,0.1421,0.0971,0.0879,0.0863,0.0355,0.0233,0.0252,0.0043,0.0048,0.0076,0.0124,0.0105,0.0054,0.0032,0.0073,0.0063,1
-0.0176,0.0172,0.0501,0.0285,0.0262,0.0351,0.0362,0.0535,0.0258,0.0474,0.0526,0.1854,0.1040,0.0948,0.0912,0.1688,0.1568,0.0375,0.1316,0.2086,0.1976,0.0946,0.1965,0.1242,0.0616,0.2141,0.4642,0.6471,0.6340,0.6107,0.7046,0.5376,0.5934,0.8443,0.9481,0.9705,0.7766,0.6313,0.5760,0.6148,0.5450,0.4813,0.3406,0.1916,0.1134,0.0640,0.0911,0.0980,0.0563,0.0187,0.0088,0.0042,0.0175,0.0171,0.0079,0.0050,0.0112,0.0179,0.0294,0.0063,1
-0.0265,0.0440,0.0137,0.0084,0.0305,0.0438,0.0341,0.0780,0.0844,0.0779,0.0327,0.2060,0.1908,0.1065,0.1457,0.2232,0.2070,0.1105,0.1078,0.1165,0.2224,0.0689,0.2060,0.2384,0.0904,0.2278,0.5872,0.8457,0.8467,0.7679,0.8055,0.6260,0.6545,0.8747,0.9885,0.9348,0.6960,0.5733,0.5872,0.6663,0.5651,0.5247,0.3684,0.1997,0.1512,0.0508,0.0931,0.0982,0.0524,0.0188,0.0100,0.0038,0.0187,0.0156,0.0068,0.0097,0.0073,0.0081,0.0086,0.0095,1
-0.0368,0.0403,0.0317,0.0293,0.0820,0.1342,0.1161,0.0663,0.0155,0.0506,0.0906,0.2545,0.1464,0.1272,0.1223,0.1669,0.1424,0.1285,0.1857,0.1136,0.2069,0.0219,0.2400,0.2547,0.0240,0.1923,0.4753,0.7003,0.6825,0.6443,0.7063,0.5373,0.6601,0.8708,0.9518,0.9605,0.7712,0.6772,0.6431,0.6720,0.6035,0.5155,0.3802,0.2278,0.1522,0.0801,0.0804,0.0752,0.0566,0.0175,0.0058,0.0091,0.0160,0.0160,0.0081,0.0070,0.0135,0.0067,0.0078,0.0068,1
-0.0195,0.0142,0.0181,0.0406,0.0391,0.0249,0.0892,0.0973,0.0840,0.1191,0.1522,0.1322,0.1434,0.1244,0.0653,0.0890,0.1226,0.1846,0.3880,0.3658,0.2297,0.2610,0.4193,0.5848,0.5643,0.5448,0.4772,0.6897,0.9797,1.0000,0.9546,0.8835,0.7662,0.6547,0.5447,0.4593,0.4679,0.1987,0.0699,0.1493,0.1713,0.1654,0.2600,0.3846,0.3754,0.2414,0.1077,0.0224,0.0155,0.0187,0.0125,0.0028,0.0067,0.0120,0.0012,0.0022,0.0058,0.0042,0.0067,0.0012,1
-0.0216,0.0215,0.0273,0.0139,0.0357,0.0785,0.0906,0.0908,0.1151,0.0973,0.1203,0.1102,0.1192,0.1762,0.2390,0.2138,0.1929,0.1765,0.0746,0.1265,0.2005,0.1571,0.2605,0.5386,0.8440,1.0000,0.8684,0.6742,0.5537,0.4638,0.3609,0.2055,0.1620,0.2092,0.3100,0.2344,0.1058,0.0383,0.0528,0.1291,0.2241,0.1915,0.1587,0.0942,0.0840,0.0670,0.0342,0.0469,0.0357,0.0136,0.0082,0.0140,0.0044,0.0052,0.0073,0.0021,0.0047,0.0024,0.0009,0.0017,1
-0.0065,0.0122,0.0068,0.0108,0.0217,0.0284,0.0527,0.0575,0.1054,0.1109,0.0937,0.0827,0.0920,0.0911,0.1487,0.1666,0.1268,0.1374,0.1095,0.1286,0.2146,0.2889,0.4238,0.6168,0.8167,0.9622,0.8280,0.5816,0.4667,0.3539,0.2727,0.1410,0.1863,0.2176,0.2360,0.1725,0.0589,0.0621,0.1847,0.2452,0.2984,0.3041,0.2275,0.1480,0.1102,0.1178,0.0608,0.0333,0.0276,0.0100,0.0023,0.0069,0.0025,0.0027,0.0052,0.0036,0.0026,0.0036,0.0006,0.0035,1
-0.0036,0.0078,0.0092,0.0387,0.0530,0.1197,0.1243,0.1026,0.1239,0.0888,0.0937,0.1245,0.1599,0.1542,0.1846,0.1732,0.1477,0.1748,0.1455,0.1579,0.2257,0.1975,0.3368,0.5828,0.8505,1.0000,0.8457,0.6624,0.5564,0.3925,0.3233,0.2054,0.1920,0.2227,0.3147,0.2268,0.0795,0.0748,0.1166,0.1969,0.2619,0.2507,0.1983,0.0948,0.0931,0.0965,0.0381,0.0435,0.0336,0.0055,0.0079,0.0119,0.0055,0.0035,0.0036,0.0004,0.0018,0.0049,0.0024,0.0016,1
-0.0208,0.0186,0.0131,0.0211,0.0610,0.0613,0.0612,0.0506,0.0989,0.1093,0.1063,0.1179,0.1291,0.1591,0.1680,0.1918,0.1615,0.1647,0.1397,0.1426,0.2429,0.2816,0.4290,0.6443,0.9061,1.0000,0.8087,0.6119,0.5260,0.3677,0.2746,0.1020,0.1339,0.1582,0.1952,0.1787,0.0429,0.1096,0.1762,0.2481,0.3150,0.2920,0.1902,0.0696,0.0758,0.0910,0.0441,0.0244,0.0265,0.0095,0.0140,0.0074,0.0063,0.0081,0.0087,0.0044,0.0028,0.0019,0.0049,0.0023,1
-0.0139,0.0222,0.0089,0.0108,0.0215,0.0136,0.0659,0.0954,0.0786,0.1015,0.1261,0.0828,0.0493,0.0848,0.1514,0.1396,0.1066,0.1923,0.2991,0.3247,0.3797,0.5658,0.7483,0.8757,0.9048,0.7511,0.6858,0.7043,0.5864,0.3773,0.2206,0.2628,0.2672,0.2907,0.1982,0.2288,0.3186,0.2871,0.2921,0.2806,0.2682,0.2112,0.1513,0.1789,0.1850,0.1717,0.0898,0.0656,0.0445,0.0110,0.0024,0.0062,0.0072,0.0113,0.0012,0.0022,0.0025,0.0059,0.0039,0.0048,1
-0.0109,0.0093,0.0121,0.0378,0.0679,0.0863,0.1004,0.0664,0.0941,0.1036,0.0972,0.0501,0.1546,0.3404,0.4804,0.6570,0.7738,0.7827,0.8152,0.8129,0.8297,0.8535,0.8870,0.8894,0.8980,0.9667,1.0000,0.9134,0.6762,0.4659,0.2895,0.2959,0.1746,0.2112,0.2569,0.2276,0.2149,0.1601,0.0371,0.0117,0.0488,0.0288,0.0597,0.0431,0.0369,0.0025,0.0327,0.0257,0.0182,0.0108,0.0124,0.0077,0.0023,0.0117,0.0053,0.0077,0.0076,0.0056,0.0055,0.0039,1
-0.0202,0.0104,0.0325,0.0239,0.0807,0.1529,0.1154,0.0608,0.1317,0.1370,0.0843,0.0269,0.1254,0.3046,0.5584,0.7973,0.8341,0.8057,0.8616,0.8769,0.9413,0.9403,0.9409,1.0000,0.9725,0.9309,0.9351,0.7317,0.4421,0.3244,0.4161,0.4611,0.4031,0.3000,0.2459,0.1348,0.2541,0.2255,0.1598,0.1485,0.0845,0.0569,0.0855,0.1262,0.1153,0.0570,0.0426,0.0425,0.0235,0.0006,0.0188,0.0127,0.0081,0.0067,0.0043,0.0065,0.0049,0.0054,0.0073,0.0054,1
-0.0239,0.0189,0.0466,0.0440,0.0657,0.0742,0.1380,0.1099,0.1384,0.1376,0.0938,0.0259,0.1499,0.2851,0.5743,0.8278,0.8669,0.8131,0.9045,0.9046,1.0000,0.9976,0.9872,0.9761,0.9009,0.9724,0.9675,0.7633,0.4434,0.3822,0.4727,0.4007,0.3381,0.3172,0.2222,0.0733,0.2692,0.1888,0.0712,0.1062,0.0694,0.0300,0.0893,0.1459,0.1348,0.0391,0.0546,0.0469,0.0201,0.0095,0.0155,0.0091,0.0151,0.0080,0.0018,0.0078,0.0045,0.0026,0.0036,0.0024,1
-0.0336,0.0294,0.0476,0.0539,0.0794,0.0804,0.1136,0.1228,0.1235,0.0842,0.0357,0.0689,0.1705,0.3257,0.4602,0.6225,0.7327,0.7843,0.7988,0.8261,1.0000,0.9814,0.9620,0.9601,0.9118,0.9086,0.7931,0.5877,0.3474,0.4235,0.4633,0.3410,0.2849,0.2847,0.1742,0.0549,0.1192,0.1154,0.0855,0.1811,0.1264,0.0799,0.0378,0.1268,0.1125,0.0505,0.0949,0.0677,0.0259,0.0170,0.0033,0.0150,0.0111,0.0032,0.0035,0.0169,0.0137,0.0015,0.0069,0.0051,1
-0.0231,0.0351,0.0030,0.0304,0.0339,0.0860,0.1738,0.1351,0.1063,0.0347,0.0575,0.1382,0.2274,0.4038,0.5223,0.6847,0.7521,0.7760,0.7708,0.8627,1.0000,0.8873,0.8057,0.8760,0.9066,0.9430,0.8846,0.6500,0.2970,0.2423,0.2992,0.2285,0.2277,0.1529,0.1037,0.0352,0.1073,0.1373,0.1331,0.1454,0.1115,0.0440,0.0762,0.1381,0.0831,0.0654,0.0844,0.0595,0.0497,0.0313,0.0154,0.0106,0.0097,0.0022,0.0052,0.0072,0.0056,0.0038,0.0043,0.0030,1
-0.0108,0.0086,0.0058,0.0460,0.0752,0.0887,0.1015,0.0494,0.0472,0.0393,0.1106,0.1412,0.2202,0.2976,0.4116,0.4754,0.5390,0.6279,0.7060,0.7918,0.9493,1.0000,0.9645,0.9432,0.8658,0.7895,0.6501,0.4492,0.4739,0.6153,0.4929,0.3195,0.3735,0.3336,0.1052,0.0671,0.0379,0.0461,0.1694,0.2169,0.1677,0.0644,0.0159,0.0778,0.0653,0.0210,0.0509,0.0387,0.0262,0.0101,0.0161,0.0029,0.0078,0.0114,0.0083,0.0058,0.0003,0.0023,0.0026,0.0027,1
-0.0229,0.0369,0.0040,0.0375,0.0455,0.1452,0.2211,0.1188,0.0750,0.1631,0.2709,0.3358,0.4091,0.4400,0.5485,0.7213,0.8137,0.9185,1.0000,0.9418,0.9116,0.9349,0.7484,0.5146,0.4106,0.3443,0.6981,0.8713,0.9013,0.8014,0.4380,0.1319,0.1709,0.2484,0.3044,0.2312,0.1338,0.2056,0.2474,0.2790,0.1610,0.0056,0.0351,0.1148,0.1331,0.0276,0.0763,0.0631,0.0309,0.0240,0.0115,0.0064,0.0022,0.0122,0.0151,0.0056,0.0026,0.0029,0.0104,0.0163,1
-0.0100,0.0194,0.0155,0.0489,0.0839,0.1009,0.1627,0.2071,0.2696,0.2990,0.3242,0.3565,0.3951,0.5201,0.6953,0.8468,1.0000,0.9278,0.8510,0.8010,0.8142,0.8825,0.7302,0.6107,0.7159,0.8458,0.6319,0.4808,0.6291,0.7152,0.6005,0.4235,0.4106,0.3992,0.1730,0.1975,0.2370,0.1339,0.1583,0.3151,0.1968,0.2054,0.1272,0.1129,0.1946,0.2195,0.1930,0.1498,0.0773,0.0196,0.0122,0.0130,0.0073,0.0077,0.0075,0.0060,0.0080,0.0019,0.0053,0.0019,1
-0.0409,0.0421,0.0573,0.0130,0.0183,0.1019,0.1054,0.1070,0.2302,0.2259,0.2373,0.3323,0.3827,0.4840,0.6812,0.7555,0.9522,0.9826,0.8871,0.8268,0.7561,0.8217,0.6967,0.6444,0.6948,0.8014,0.6053,0.6084,0.8877,0.8557,0.5563,0.2897,0.3638,0.4786,0.2908,0.0899,0.2043,0.1707,0.0407,0.1286,0.1581,0.2191,0.1701,0.0971,0.2217,0.2732,0.1874,0.1062,0.0665,0.0405,0.0113,0.0028,0.0036,0.0105,0.0120,0.0087,0.0061,0.0061,0.0030,0.0078,1
-0.0217,0.0340,0.0392,0.0236,0.1081,0.1164,0.1398,0.1009,0.1147,0.1777,0.4079,0.4113,0.3973,0.5078,0.6509,0.8073,0.9819,1.0000,0.9407,0.8452,0.8106,0.8460,0.6212,0.5815,0.7745,0.8204,0.5601,0.2989,0.5009,0.6628,0.5753,0.4055,0.3746,0.3481,0.1580,0.1422,0.2130,0.1866,0.1003,0.2396,0.2241,0.2029,0.0710,0.1606,0.1669,0.1700,0.1829,0.1403,0.0506,0.0224,0.0095,0.0031,0.0103,0.0078,0.0077,0.0094,0.0031,0.0030,0.0013,0.0069,1
-0.0378,0.0318,0.0423,0.0350,0.1787,0.1635,0.0887,0.0817,0.1779,0.2053,0.3135,0.3118,0.3686,0.3885,0.5850,0.7868,0.9739,1.0000,0.9843,0.8610,0.8443,0.9061,0.5847,0.4033,0.5946,0.6793,0.6389,0.5002,0.5578,0.4831,0.4729,0.3318,0.3969,0.3894,0.2314,0.1036,0.1312,0.0864,0.2569,0.3179,0.2649,0.2714,0.1713,0.0584,0.1230,0.2200,0.2198,0.1074,0.0423,0.0162,0.0093,0.0046,0.0044,0.0078,0.0102,0.0065,0.0061,0.0062,0.0043,0.0053,1
-0.0365,0.1632,0.1636,0.1421,0.1130,0.1306,0.2112,0.2268,0.2992,0.3735,0.3042,0.0387,0.2679,0.5397,0.6204,0.7257,0.8350,0.6888,0.4450,0.3921,0.5605,0.7545,0.8311,1.0000,0.8762,0.7092,0.7009,0.5014,0.3942,0.4456,0.4072,0.0773,0.1423,0.0401,0.3597,0.6847,0.7076,0.3597,0.0612,0.3027,0.3966,0.3868,0.2380,0.2059,0.2288,0.1704,0.1587,0.1792,0.1022,0.0151,0.0223,0.0110,0.0071,0.0205,0.0164,0.0063,0.0078,0.0094,0.0110,0.0068,1
-0.0188,0.0370,0.0953,0.0824,0.0249,0.0488,0.1424,0.1972,0.1873,0.1806,0.2139,0.1523,0.1975,0.4844,0.7298,0.7807,0.7906,0.6122,0.4200,0.2807,0.5148,0.7569,0.8596,1.0000,0.8457,0.6797,0.6971,0.5843,0.4772,0.5201,0.4241,0.1592,0.1668,0.0588,0.3967,0.7147,0.7319,0.3509,0.0589,0.2690,0.4200,0.3874,0.2440,0.2000,0.2307,0.1886,0.1960,0.1701,0.1366,0.0398,0.0143,0.0093,0.0033,0.0113,0.0030,0.0057,0.0090,0.0057,0.0068,0.0024,1
-0.0856,0.0454,0.0382,0.0203,0.0385,0.0534,0.2140,0.3110,0.2837,0.2751,0.2707,0.0946,0.1020,0.4519,0.6737,0.6699,0.7066,0.5632,0.3785,0.2721,0.5297,0.7697,0.8643,0.9304,0.9372,0.6247,0.6024,0.6810,0.5047,0.5775,0.4754,0.2400,0.2779,0.1997,0.5305,0.7409,0.7775,0.4424,0.1416,0.3508,0.4482,0.4208,0.3054,0.2235,0.2611,0.2798,0.2392,0.2021,0.1326,0.0358,0.0128,0.0172,0.0138,0.0079,0.0037,0.0051,0.0258,0.0102,0.0037,0.0037,1
-0.0274,0.0242,0.0621,0.0560,0.1129,0.0973,0.1823,0.1745,0.1440,0.1808,0.2366,0.0906,0.1749,0.4012,0.5187,0.7312,0.9062,0.9260,0.7434,0.4463,0.5103,0.6952,0.7755,0.8364,0.7283,0.6399,0.5759,0.4146,0.3495,0.4437,0.2665,0.2024,0.1942,0.0765,0.3725,0.5843,0.4827,0.2347,0.0999,0.3244,0.3990,0.2975,0.1684,0.1761,0.1683,0.0729,0.1190,0.1297,0.0748,0.0067,0.0255,0.0113,0.0108,0.0085,0.0047,0.0074,0.0104,0.0161,0.0220,0.0173,1
-0.0235,0.0291,0.0749,0.0519,0.0227,0.0834,0.0677,0.2002,0.2876,0.3674,0.2974,0.0837,0.1912,0.5040,0.6352,0.6804,0.7505,0.6595,0.4509,0.2964,0.4019,0.6794,0.8297,1.0000,0.8240,0.7115,0.7726,0.6124,0.4936,0.5648,0.4906,0.1820,0.1811,0.1107,0.4603,0.6650,0.6423,0.2166,0.1951,0.4947,0.4925,0.4041,0.2402,0.1392,0.1779,0.1946,0.1723,0.1522,0.0929,0.0179,0.0242,0.0083,0.0037,0.0095,0.0105,0.0030,0.0132,0.0068,0.0108,0.0090,1
-0.0126,0.0519,0.0621,0.0518,0.1072,0.2587,0.2304,0.2067,0.3416,0.4284,0.3015,0.1207,0.3299,0.5707,0.6962,0.9751,1.0000,0.9293,0.6210,0.4586,0.5001,0.5032,0.7082,0.8420,0.8109,0.7690,0.8105,0.6203,0.2356,0.2595,0.6299,0.6762,0.2903,0.4393,0.8529,0.7180,0.4801,0.5856,0.4993,0.2866,0.0601,0.1167,0.2737,0.2812,0.2078,0.0660,0.0491,0.0345,0.0172,0.0287,0.0027,0.0208,0.0048,0.0199,0.0126,0.0022,0.0037,0.0034,0.0114,0.0077,1
-0.0253,0.0808,0.0507,0.0244,0.1724,0.3823,0.3729,0.3583,0.3429,0.2197,0.2653,0.3223,0.5582,0.6916,0.7943,0.7152,0.3512,0.2008,0.2676,0.4299,0.5280,0.3489,0.1430,0.5453,0.6338,0.7712,0.6838,0.8015,0.8073,0.8310,0.7792,0.5049,0.1413,0.2767,0.5084,0.4787,0.1356,0.2299,0.2789,0.3833,0.2933,0.1155,0.1705,0.1294,0.0909,0.0800,0.0567,0.0198,0.0114,0.0151,0.0085,0.0178,0.0073,0.0079,0.0038,0.0116,0.0033,0.0039,0.0081,0.0053,1
-0.0260,0.0192,0.0254,0.0061,0.0352,0.0701,0.1263,0.1080,0.1523,0.1630,0.1030,0.2187,0.1542,0.2630,0.2940,0.2978,0.0699,0.1401,0.2990,0.3915,0.3598,0.2403,0.4208,0.5675,0.6094,0.6323,0.6549,0.7673,1.0000,0.8463,0.5509,0.4444,0.5169,0.4268,0.1802,0.0791,0.0535,0.1906,0.2561,0.2153,0.2769,0.2841,0.1733,0.0815,0.0335,0.0933,0.1018,0.0309,0.0208,0.0318,0.0132,0.0118,0.0120,0.0051,0.0070,0.0015,0.0035,0.0008,0.0044,0.0077,1
-0.0459,0.0437,0.0347,0.0456,0.0067,0.0890,0.1798,0.1741,0.1598,0.1408,0.2693,0.3259,0.4545,0.5785,0.4471,0.2231,0.2164,0.3201,0.2915,0.4235,0.4460,0.2380,0.6415,0.8966,0.8918,0.7529,0.6838,0.8390,1.0000,0.8362,0.5427,0.4577,0.8067,0.6973,0.3915,0.1558,0.1598,0.2161,0.5178,0.4782,0.2344,0.3599,0.2785,0.1807,0.0352,0.0473,0.0322,0.0408,0.0163,0.0088,0.0121,0.0067,0.0032,0.0109,0.0164,0.0151,0.0070,0.0085,0.0117,0.0056,1
-0.0025,0.0309,0.0171,0.0228,0.0434,0.1224,0.1947,0.1661,0.1368,0.1430,0.0994,0.2250,0.2444,0.3239,0.3039,0.2410,0.0367,0.1672,0.3038,0.4069,0.3613,0.1994,0.4611,0.6849,0.7272,0.7152,0.7102,0.8516,1.0000,0.7690,0.4841,0.3717,0.6096,0.5110,0.2586,0.0916,0.0947,0.2287,0.3480,0.2095,0.1901,0.2941,0.2211,0.1524,0.0746,0.0606,0.0692,0.0446,0.0344,0.0082,0.0108,0.0149,0.0077,0.0036,0.0114,0.0085,0.0101,0.0016,0.0028,0.0014,1
-0.0291,0.0400,0.0771,0.0809,0.0521,0.1051,0.0145,0.0674,0.1294,0.1146,0.0942,0.0794,0.0252,0.1191,0.1045,0.2050,0.1556,0.2690,0.3784,0.4024,0.3470,0.1395,0.1208,0.2827,0.1500,0.2626,0.4468,0.7520,0.9036,0.7812,0.4766,0.2483,0.5372,0.6279,0.3647,0.4572,0.6359,0.6474,0.5520,0.3253,0.2292,0.0653,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0056,0.0237,0.0204,0.0050,0.0137,0.0164,0.0081,0.0139,0.0111,1
-0.0181,0.0146,0.0026,0.0141,0.0421,0.0473,0.0361,0.0741,0.1398,0.1045,0.0904,0.0671,0.0997,0.1056,0.0346,0.1231,0.1626,0.3652,0.3262,0.2995,0.2109,0.2104,0.2085,0.2282,0.0747,0.1969,0.4086,0.6385,0.7970,0.7508,0.5517,0.2214,0.4672,0.4479,0.2297,0.3235,0.4480,0.5581,0.6520,0.5354,0.2478,0.2268,0.1788,0.0898,0.0536,0.0374,0.0990,0.0956,0.0317,0.0142,0.0076,0.0223,0.0255,0.0145,0.0233,0.0041,0.0018,0.0048,0.0089,0.0085,1
-0.0491,0.0279,0.0592,0.1270,0.1772,0.1908,0.2217,0.0768,0.1246,0.2028,0.0947,0.2497,0.2209,0.3195,0.3340,0.3323,0.2780,0.2975,0.2948,0.1729,0.3264,0.3834,0.3523,0.5410,0.5228,0.4475,0.5340,0.5323,0.3907,0.3456,0.4091,0.4639,0.5580,0.5727,0.6355,0.7563,0.6903,0.6176,0.5379,0.5622,0.6508,0.4797,0.3736,0.2804,0.1982,0.2438,0.1789,0.1706,0.0762,0.0238,0.0268,0.0081,0.0129,0.0161,0.0063,0.0119,0.0194,0.0140,0.0332,0.0439,-1
-0.1313,0.2339,0.3059,0.4264,0.4010,0.1791,0.1853,0.0055,0.1929,0.2231,0.2907,0.2259,0.3136,0.3302,0.3660,0.3956,0.4386,0.4670,0.5255,0.3735,0.2243,0.1973,0.4337,0.6532,0.5070,0.2796,0.4163,0.5950,0.5242,0.4178,0.3714,0.2375,0.0863,0.1437,0.2896,0.4577,0.3725,0.3372,0.3803,0.4181,0.3603,0.2711,0.1653,0.1951,0.2811,0.2246,0.1921,0.1500,0.0665,0.0193,0.0156,0.0362,0.0210,0.0154,0.0180,0.0013,0.0106,0.0127,0.0178,0.0231,-1
-0.0201,0.0423,0.0554,0.0783,0.0620,0.0871,0.1201,0.2707,0.1206,0.0279,0.2251,0.2615,0.1770,0.3709,0.4533,0.5553,0.4616,0.3797,0.3450,0.2665,0.2395,0.1127,0.2556,0.5169,0.3779,0.4082,0.5353,0.5116,0.4544,0.4258,0.3869,0.3939,0.4661,0.3974,0.2194,0.1816,0.1023,0.2108,0.3253,0.3697,0.2912,0.3010,0.2563,0.1927,0.2062,0.1751,0.0841,0.1035,0.0641,0.0153,0.0081,0.0191,0.0182,0.0160,0.0290,0.0090,0.0242,0.0224,0.0190,0.0096,-1
-0.0629,0.1065,0.1526,0.1229,0.1437,0.1190,0.0884,0.0907,0.2107,0.3597,0.5466,0.5205,0.5127,0.5395,0.6558,0.8705,0.9786,0.9335,0.7917,0.7383,0.6908,0.3850,0.0671,0.0502,0.2717,0.2839,0.2234,0.1911,0.0408,0.2531,0.1979,0.1891,0.2433,0.1956,0.2667,0.1340,0.1073,0.2023,0.1794,0.0227,0.1313,0.1775,0.1549,0.1626,0.0708,0.0129,0.0795,0.0762,0.0117,0.0061,0.0257,0.0089,0.0262,0.0108,0.0138,0.0187,0.0230,0.0057,0.0113,0.0131,-1
-0.0335,0.0134,0.0696,0.1180,0.0348,0.1180,0.1948,0.1607,0.3036,0.4372,0.5533,0.5771,0.7022,0.7067,0.7367,0.7391,0.8622,0.9458,0.8782,0.7913,0.5760,0.3061,0.0563,0.0239,0.2554,0.4862,0.5027,0.4402,0.2847,0.1797,0.3560,0.3522,0.3321,0.3112,0.3638,0.0754,0.1834,0.1820,0.1815,0.1593,0.0576,0.0954,0.1086,0.0812,0.0784,0.0487,0.0439,0.0586,0.0370,0.0185,0.0302,0.0244,0.0232,0.0093,0.0159,0.0193,0.0032,0.0377,0.0126,0.0156,-1
-0.0587,0.1210,0.1268,0.1498,0.1436,0.0561,0.0832,0.0672,0.1372,0.2352,0.3208,0.4257,0.5201,0.4914,0.5950,0.7221,0.9039,0.9111,0.8723,0.7686,0.7326,0.5222,0.3097,0.3172,0.2270,0.1640,0.1746,0.1835,0.2048,0.1674,0.2767,0.3104,0.3399,0.4441,0.5046,0.2814,0.1681,0.2633,0.3198,0.1933,0.0934,0.0443,0.0780,0.0722,0.0405,0.0553,0.1081,0.1139,0.0767,0.0265,0.0215,0.0331,0.0111,0.0088,0.0158,0.0122,0.0038,0.0101,0.0228,0.0124,-1
-0.0162,0.0253,0.0262,0.0386,0.0645,0.0472,0.1056,0.1388,0.0598,0.1334,0.2969,0.4754,0.5677,0.5690,0.6421,0.7487,0.8999,1.0000,0.9690,0.9032,0.7685,0.6998,0.6644,0.5964,0.3711,0.0921,0.0481,0.0876,0.1040,0.1714,0.3264,0.4612,0.3939,0.5050,0.4833,0.3511,0.2319,0.4029,0.3676,0.1510,0.0745,0.1395,0.1552,0.0377,0.0636,0.0443,0.0264,0.0223,0.0187,0.0077,0.0137,0.0071,0.0082,0.0232,0.0198,0.0074,0.0035,0.0100,0.0048,0.0019,-1
-0.0307,0.0523,0.0653,0.0521,0.0611,0.0577,0.0665,0.0664,0.1460,0.2792,0.3877,0.4992,0.4981,0.4972,0.5607,0.7339,0.8230,0.9173,0.9975,0.9911,0.8240,0.6498,0.5980,0.4862,0.3150,0.1543,0.0989,0.0284,0.1008,0.2636,0.2694,0.2930,0.2925,0.3998,0.3660,0.3172,0.4609,0.4374,0.1820,0.3376,0.6202,0.4448,0.1863,0.1420,0.0589,0.0576,0.0672,0.0269,0.0245,0.0190,0.0063,0.0321,0.0189,0.0137,0.0277,0.0152,0.0052,0.0121,0.0124,0.0055,-1
-0.0116,0.0179,0.0449,0.1096,0.1913,0.0924,0.0761,0.1092,0.0757,0.1006,0.2500,0.3988,0.3809,0.4753,0.6165,0.6464,0.8024,0.9208,0.9832,0.9634,0.8646,0.8325,0.8276,0.8007,0.6102,0.4853,0.4355,0.4307,0.4399,0.3833,0.3032,0.3035,0.3197,0.2292,0.2131,0.2347,0.3201,0.4455,0.3655,0.2715,0.1747,0.1781,0.2199,0.1056,0.0573,0.0307,0.0237,0.0470,0.0102,0.0057,0.0031,0.0163,0.0099,0.0084,0.0270,0.0277,0.0097,0.0054,0.0148,0.0092,-1
-0.0331,0.0423,0.0474,0.0818,0.0835,0.0756,0.0374,0.0961,0.0548,0.0193,0.0897,0.1734,0.1936,0.2803,0.3313,0.5020,0.6360,0.7096,0.8333,0.8730,0.8073,0.7507,0.7526,0.7298,0.6177,0.4946,0.4531,0.4099,0.4540,0.4124,0.3139,0.3194,0.3692,0.3776,0.4469,0.4777,0.4716,0.4664,0.3893,0.4255,0.4064,0.3712,0.3863,0.2802,0.1283,0.1117,0.1303,0.0787,0.0436,0.0224,0.0133,0.0078,0.0174,0.0176,0.0038,0.0129,0.0066,0.0044,0.0134,0.0092,-1
-0.0428,0.0555,0.0708,0.0618,0.1215,0.1524,0.1543,0.0391,0.0610,0.0113,0.1255,0.2473,0.3011,0.3747,0.4520,0.5392,0.6588,0.7113,0.7602,0.8672,0.8416,0.7974,0.8385,0.9317,0.8555,0.6162,0.4139,0.3269,0.3108,0.2554,0.3367,0.4465,0.5000,0.5111,0.5194,0.4619,0.4234,0.4372,0.4277,0.4433,0.3700,0.3324,0.2564,0.2527,0.2137,0.1789,0.1010,0.0528,0.0453,0.0118,0.0009,0.0142,0.0179,0.0079,0.0060,0.0131,0.0089,0.0084,0.0113,0.0049,-1
-0.0599,0.0474,0.0498,0.0387,0.1026,0.0773,0.0853,0.0447,0.1094,0.0351,0.1582,0.2023,0.2268,0.2829,0.3819,0.4665,0.6687,0.8647,0.9361,0.9367,0.9144,0.9162,0.9311,0.8604,0.7327,0.5763,0.4162,0.4113,0.4146,0.3149,0.2936,0.3169,0.3149,0.4132,0.3994,0.4195,0.4532,0.4419,0.4737,0.3431,0.3194,0.3370,0.2493,0.2650,0.1748,0.0932,0.0530,0.0081,0.0342,0.0137,0.0028,0.0013,0.0005,0.0227,0.0209,0.0081,0.0117,0.0114,0.0112,0.0100,-1
-0.0264,0.0071,0.0342,0.0793,0.1043,0.0783,0.1417,0.1176,0.0453,0.0945,0.1132,0.0840,0.0717,0.1968,0.2633,0.4191,0.5050,0.6711,0.7922,0.8381,0.8759,0.9422,1.0000,0.9931,0.9575,0.8647,0.7215,0.5801,0.4964,0.4886,0.4079,0.2443,0.1768,0.2472,0.3518,0.3762,0.2909,0.2311,0.3168,0.3554,0.3741,0.4443,0.3261,0.1963,0.0864,0.1688,0.1991,0.1217,0.0628,0.0323,0.0253,0.0214,0.0262,0.0177,0.0037,0.0068,0.0121,0.0077,0.0078,0.0066,-1
-0.0210,0.0121,0.0203,0.1036,0.1675,0.0418,0.0723,0.0828,0.0494,0.0686,0.1125,0.1741,0.2710,0.3087,0.3575,0.4998,0.6011,0.6470,0.8067,0.9008,0.8906,0.9338,1.0000,0.9102,0.8496,0.7867,0.7688,0.7718,0.6268,0.4301,0.2077,0.1198,0.1660,0.2618,0.3862,0.3958,0.3248,0.2302,0.3250,0.4022,0.4344,0.4008,0.3370,0.2518,0.2101,0.1181,0.1150,0.0550,0.0293,0.0183,0.0104,0.0117,0.0101,0.0061,0.0031,0.0099,0.0080,0.0107,0.0161,0.0133,-1
-0.0530,0.0885,0.1997,0.2604,0.3225,0.2247,0.0617,0.2287,0.0950,0.0740,0.1610,0.2226,0.2703,0.3365,0.4266,0.4144,0.5655,0.6921,0.8547,0.9234,0.9171,1.0000,0.9532,0.9101,0.8337,0.7053,0.6534,0.4483,0.2460,0.2020,0.1446,0.0994,0.1510,0.2392,0.4434,0.5023,0.4441,0.4571,0.3927,0.2900,0.3408,0.4990,0.3632,0.1387,0.1800,0.1299,0.0523,0.0817,0.0469,0.0114,0.0299,0.0244,0.0199,0.0257,0.0082,0.0151,0.0171,0.0146,0.0134,0.0056,-1
-0.0454,0.0472,0.0697,0.1021,0.1397,0.1493,0.1487,0.0771,0.1171,0.1675,0.2799,0.3323,0.4012,0.4296,0.5350,0.5411,0.6870,0.8045,0.9194,0.9169,1.0000,0.9972,0.9093,0.7918,0.6705,0.5324,0.3572,0.2484,0.3161,0.3775,0.3138,0.1713,0.2937,0.5234,0.5926,0.5437,0.4516,0.3379,0.3215,0.2178,0.1674,0.2634,0.2980,0.2037,0.1155,0.0919,0.0882,0.0228,0.0380,0.0142,0.0137,0.0120,0.0042,0.0238,0.0129,0.0084,0.0218,0.0321,0.0154,0.0053,-1
-0.0283,0.0599,0.0656,0.0229,0.0839,0.1673,0.1154,0.1098,0.1370,0.1767,0.1995,0.2869,0.3275,0.3769,0.4169,0.5036,0.6180,0.8025,0.9333,0.9399,0.9275,0.9450,0.8328,0.7773,0.7007,0.6154,0.5810,0.4454,0.3707,0.2891,0.2185,0.1711,0.3578,0.3947,0.2867,0.2401,0.3619,0.3314,0.3763,0.4767,0.4059,0.3661,0.2320,0.1450,0.1017,0.1111,0.0655,0.0271,0.0244,0.0179,0.0109,0.0147,0.0170,0.0158,0.0046,0.0073,0.0054,0.0033,0.0045,0.0079,-1
-0.0114,0.0222,0.0269,0.0384,0.1217,0.2062,0.1489,0.0929,0.1350,0.1799,0.2486,0.2973,0.3672,0.4394,0.5258,0.6755,0.7402,0.8284,0.9033,0.9584,1.0000,0.9982,0.8899,0.7493,0.6367,0.6744,0.7207,0.6821,0.5512,0.4789,0.3924,0.2533,0.1089,0.1390,0.2551,0.3301,0.2818,0.2142,0.2266,0.2142,0.2354,0.2871,0.2596,0.1925,0.1256,0.1003,0.0951,0.1210,0.0728,0.0174,0.0213,0.0269,0.0152,0.0257,0.0097,0.0041,0.0050,0.0145,0.0103,0.0025,-1
-0.0414,0.0436,0.0447,0.0844,0.0419,0.1215,0.2002,0.1516,0.0818,0.1975,0.2309,0.3025,0.3938,0.5050,0.5872,0.6610,0.7417,0.8006,0.8456,0.7939,0.8804,0.8384,0.7852,0.8479,0.7434,0.6433,0.5514,0.3519,0.3168,0.3346,0.2056,0.1032,0.3168,0.4040,0.4282,0.4538,0.3704,0.3741,0.3839,0.3494,0.4380,0.4265,0.2854,0.2808,0.2395,0.0369,0.0805,0.0541,0.0177,0.0065,0.0222,0.0045,0.0136,0.0113,0.0053,0.0165,0.0141,0.0077,0.0246,0.0198,-1
-0.0094,0.0333,0.0306,0.0376,0.1296,0.1795,0.1909,0.1692,0.1870,0.1725,0.2228,0.3106,0.4144,0.5157,0.5369,0.5107,0.6441,0.7326,0.8164,0.8856,0.9891,1.0000,0.8750,0.8631,0.9074,0.8674,0.7750,0.6600,0.5615,0.4016,0.2331,0.1164,0.1095,0.0431,0.0619,0.1956,0.2120,0.3242,0.4102,0.2939,0.1911,0.1702,0.1010,0.1512,0.1427,0.1097,0.1173,0.0972,0.0703,0.0281,0.0216,0.0153,0.0112,0.0241,0.0164,0.0055,0.0078,0.0055,0.0091,0.0067,-1
-0.0228,0.0106,0.0130,0.0842,0.1117,0.1506,0.1776,0.0997,0.1428,0.2227,0.2621,0.3109,0.2859,0.3316,0.3755,0.4499,0.4765,0.6254,0.7304,0.8702,0.9349,0.9614,0.9126,0.9443,1.0000,0.9455,0.8815,0.7520,0.7068,0.5986,0.3857,0.2510,0.2162,0.0968,0.1323,0.1344,0.2250,0.3244,0.3939,0.3806,0.3258,0.3654,0.2983,0.1779,0.1535,0.1199,0.0959,0.0765,0.0649,0.0313,0.0185,0.0098,0.0178,0.0077,0.0074,0.0095,0.0055,0.0045,0.0063,0.0039,-1
-0.0363,0.0478,0.0298,0.0210,0.1409,0.1916,0.1349,0.1613,0.1703,0.1444,0.1989,0.2154,0.2863,0.3570,0.3980,0.4359,0.5334,0.6304,0.6995,0.7435,0.8379,0.8641,0.9014,0.9432,0.9536,1.0000,0.9547,0.9745,0.8962,0.7196,0.5462,0.3156,0.2525,0.1969,0.2189,0.1533,0.0711,0.1498,0.1755,0.2276,0.1322,0.1056,0.1973,0.1692,0.1881,0.1177,0.0779,0.0495,0.0492,0.0194,0.0250,0.0115,0.0190,0.0055,0.0096,0.0050,0.0066,0.0114,0.0073,0.0033,-1
-0.0261,0.0266,0.0223,0.0749,0.1364,0.1513,0.1316,0.1654,0.1864,0.2013,0.2890,0.3650,0.3510,0.3495,0.4325,0.5398,0.6237,0.6876,0.7329,0.8107,0.8396,0.8632,0.8747,0.9607,0.9716,0.9121,0.8576,0.8798,0.7720,0.5711,0.4264,0.2860,0.3114,0.2066,0.1165,0.0185,0.1302,0.2480,0.1637,0.1103,0.2144,0.2033,0.1887,0.1370,0.1376,0.0307,0.0373,0.0606,0.0399,0.0169,0.0135,0.0222,0.0175,0.0127,0.0022,0.0124,0.0054,0.0021,0.0028,0.0023,-1
-0.0346,0.0509,0.0079,0.0243,0.0432,0.0735,0.0938,0.1134,0.1228,0.1508,0.1809,0.2390,0.2947,0.2866,0.4010,0.5325,0.5486,0.5823,0.6041,0.6749,0.7084,0.7890,0.9284,0.9781,0.9738,1.0000,0.9702,0.9956,0.8235,0.6020,0.5342,0.4867,0.3526,0.1566,0.0946,0.1613,0.2824,0.3390,0.3019,0.2945,0.2978,0.2676,0.2055,0.2069,0.1625,0.1216,0.1013,0.0744,0.0386,0.0050,0.0146,0.0040,0.0122,0.0107,0.0112,0.0102,0.0052,0.0024,0.0079,0.0031,-1
-0.0162,0.0041,0.0239,0.0441,0.0630,0.0921,0.1368,0.1078,0.1552,0.1779,0.2164,0.2568,0.3089,0.3829,0.4393,0.5335,0.5996,0.6728,0.7309,0.8092,0.8941,0.9668,1.0000,0.9893,0.9376,0.8991,0.9184,0.9128,0.7811,0.6018,0.3765,0.3300,0.2280,0.0212,0.1117,0.1788,0.2373,0.2843,0.2241,0.2715,0.3363,0.2546,0.1867,0.2160,0.1278,0.0768,0.1070,0.0946,0.0636,0.0227,0.0128,0.0173,0.0135,0.0114,0.0062,0.0157,0.0088,0.0036,0.0053,0.0030,-1
-0.0249,0.0119,0.0277,0.0760,0.1218,0.1538,0.1192,0.1229,0.2119,0.2531,0.2855,0.2961,0.3341,0.4287,0.5205,0.6087,0.7236,0.7577,0.7726,0.8098,0.8995,0.9247,0.9365,0.9853,0.9776,1.0000,0.9896,0.9076,0.7306,0.5758,0.4469,0.3719,0.2079,0.0955,0.0488,0.1406,0.2554,0.2054,0.1614,0.2232,0.1773,0.2293,0.2521,0.1464,0.0673,0.0965,0.1492,0.1128,0.0463,0.0193,0.0140,0.0027,0.0068,0.0150,0.0012,0.0133,0.0048,0.0244,0.0077,0.0074,-1
-0.0270,0.0163,0.0341,0.0247,0.0822,0.1256,0.1323,0.1584,0.2017,0.2122,0.2210,0.2399,0.2964,0.4061,0.5095,0.5512,0.6613,0.6804,0.6520,0.6788,0.7811,0.8369,0.8969,0.9856,1.0000,0.9395,0.8917,0.8105,0.6828,0.5572,0.4301,0.3339,0.2035,0.0798,0.0809,0.1525,0.2626,0.2456,0.1980,0.2412,0.2409,0.1901,0.2077,0.1767,0.1119,0.0779,0.1344,0.0960,0.0598,0.0330,0.0197,0.0189,0.0204,0.0085,0.0043,0.0092,0.0138,0.0094,0.0105,0.0093,-1
-0.0388,0.0324,0.0688,0.0898,0.1267,0.1515,0.2134,0.2613,0.2832,0.2718,0.3645,0.3934,0.3843,0.4677,0.5364,0.4823,0.4835,0.5862,0.7579,0.6997,0.6918,0.8633,0.9107,0.9346,0.7884,0.8585,0.9261,0.7080,0.5779,0.5215,0.4505,0.3129,0.1448,0.1046,0.1820,0.1519,0.1017,0.1438,0.1986,0.2039,0.2778,0.2879,0.1331,0.1140,0.1310,0.1433,0.0624,0.0100,0.0098,0.0131,0.0152,0.0255,0.0071,0.0263,0.0079,0.0111,0.0107,0.0068,0.0097,0.0067,-1
-0.0228,0.0853,0.1000,0.0428,0.1117,0.1651,0.1597,0.2116,0.3295,0.3517,0.3330,0.3643,0.4020,0.4731,0.5196,0.6573,0.8426,0.8476,0.8344,0.8453,0.7999,0.8537,0.9642,1.0000,0.9357,0.9409,0.9070,0.7104,0.6320,0.5667,0.3501,0.2447,0.1698,0.3290,0.3674,0.2331,0.2413,0.2556,0.1892,0.1940,0.3074,0.2785,0.0308,0.1238,0.1854,0.1753,0.1079,0.0728,0.0242,0.0191,0.0159,0.0172,0.0191,0.0260,0.0140,0.0125,0.0116,0.0093,0.0012,0.0036,-1
-0.0715,0.0849,0.0587,0.0218,0.0862,0.1801,0.1916,0.1896,0.2960,0.4186,0.4867,0.5249,0.5959,0.6855,0.8573,0.9718,0.8693,0.8711,0.8954,0.9922,0.8980,0.8158,0.8373,0.7541,0.5893,0.5488,0.5643,0.5406,0.4783,0.4439,0.3698,0.2574,0.1478,0.1743,0.1229,0.1588,0.1803,0.1436,0.1667,0.2630,0.2234,0.1239,0.0869,0.2092,0.1499,0.0676,0.0899,0.0927,0.0658,0.0086,0.0216,0.0153,0.0121,0.0096,0.0196,0.0042,0.0066,0.0099,0.0083,0.0124,-1
-0.0209,0.0261,0.0120,0.0768,0.1064,0.1680,0.3016,0.3460,0.3314,0.4125,0.3943,0.1334,0.4622,0.9970,0.9137,0.8292,0.6994,0.7825,0.8789,0.8501,0.8920,0.9473,1.0000,0.8975,0.7806,0.8321,0.6502,0.4548,0.4732,0.3391,0.2747,0.0978,0.0477,0.1403,0.1834,0.2148,0.1271,0.1912,0.3391,0.3444,0.2369,0.1195,0.2665,0.2587,0.1393,0.1083,0.1383,0.1321,0.1069,0.0325,0.0316,0.0057,0.0159,0.0085,0.0372,0.0101,0.0127,0.0288,0.0129,0.0023,-1
-0.0374,0.0586,0.0628,0.0534,0.0255,0.1422,0.2072,0.2734,0.3070,0.2597,0.3483,0.3999,0.4574,0.5950,0.7924,0.8272,0.8087,0.8977,0.9828,0.8982,0.8890,0.9367,0.9122,0.7936,0.6718,0.6318,0.4865,0.3388,0.4832,0.3822,0.3075,0.1267,0.0743,0.1510,0.1906,0.1817,0.1709,0.0946,0.2829,0.3006,0.1602,0.1483,0.2875,0.2047,0.1064,0.1395,0.1065,0.0527,0.0395,0.0183,0.0353,0.0118,0.0063,0.0237,0.0032,0.0087,0.0124,0.0113,0.0098,0.0126,-1
-0.1371,0.1226,0.1385,0.1484,0.1776,0.1428,0.1773,0.2161,0.1630,0.2067,0.4257,0.5484,0.7131,0.7003,0.6777,0.7939,0.9382,0.8925,0.9146,0.7832,0.7960,0.7983,0.7716,0.6615,0.4860,0.5572,0.4697,0.5640,0.4517,0.3369,0.2684,0.2339,0.3052,0.3016,0.2753,0.1041,0.1757,0.3156,0.3603,0.2736,0.1301,0.2458,0.3404,0.1753,0.0679,0.1062,0.0643,0.0532,0.0531,0.0272,0.0171,0.0118,0.0129,0.0344,0.0065,0.0067,0.0022,0.0079,0.0146,0.0051,-1
-0.0443,0.0446,0.0235,0.1008,0.2252,0.2611,0.2061,0.1668,0.1801,0.3083,0.3794,0.5364,0.6173,0.7842,0.8392,0.9016,1.0000,0.8911,0.8753,0.7886,0.7156,0.7581,0.6372,0.3210,0.2076,0.2279,0.3309,0.2847,0.1949,0.1671,0.1025,0.1362,0.2212,0.1124,0.1677,0.1039,0.2562,0.2624,0.2236,0.1180,0.1103,0.2831,0.2385,0.0255,0.1967,0.1483,0.0434,0.0627,0.0513,0.0473,0.0248,0.0274,0.0205,0.0141,0.0185,0.0055,0.0045,0.0115,0.0152,0.0100,-1
-0.1150,0.1163,0.0866,0.0358,0.0232,0.1267,0.2417,0.2661,0.4346,0.5378,0.3816,0.0991,0.0616,0.1795,0.3907,0.3602,0.3041,0.2428,0.4060,0.8395,0.9777,0.4680,0.0610,0.2143,0.1348,0.2854,0.1617,0.2649,0.4565,0.6502,0.2848,0.3296,0.5370,0.6627,0.8626,0.8547,0.7848,0.9016,0.8827,0.6086,0.2810,0.0906,0.1177,0.2694,0.5214,0.4232,0.2340,0.1928,0.1092,0.0507,0.0228,0.0099,0.0065,0.0085,0.0166,0.0110,0.0190,0.0141,0.0068,0.0086,-1
-0.0968,0.0821,0.0629,0.0608,0.0617,0.1207,0.0944,0.4223,0.5744,0.5025,0.3488,0.1700,0.2076,0.3087,0.4224,0.5312,0.2436,0.1884,0.1908,0.8321,1.0000,0.4076,0.0960,0.1928,0.2419,0.3790,0.2893,0.3451,0.3777,0.5213,0.2316,0.3335,0.4781,0.6116,0.6705,0.7375,0.7356,0.7792,0.6788,0.5259,0.2762,0.1545,0.2019,0.2231,0.4221,0.3067,0.1329,0.1349,0.1057,0.0499,0.0206,0.0073,0.0081,0.0303,0.0190,0.0212,0.0126,0.0201,0.0210,0.0041,-1
-0.0790,0.0707,0.0352,0.1660,0.1330,0.0226,0.0771,0.2678,0.5664,0.6609,0.5002,0.2583,0.1650,0.4347,0.4515,0.4579,0.3366,0.4000,0.5325,0.9010,0.9939,0.3689,0.1012,0.0248,0.2318,0.3981,0.2259,0.5247,0.6898,0.8316,0.4326,0.3741,0.5756,0.8043,0.7963,0.7174,0.7056,0.8148,0.7601,0.6034,0.4554,0.4729,0.4478,0.3722,0.4693,0.3839,0.0768,0.1467,0.0777,0.0469,0.0193,0.0298,0.0390,0.0294,0.0175,0.0249,0.0141,0.0073,0.0025,0.0101,-1
-0.1083,0.1070,0.0257,0.0837,0.0748,0.1125,0.3322,0.4590,0.5526,0.5966,0.5304,0.2251,0.2402,0.2689,0.6646,0.6632,0.1674,0.0837,0.4331,0.8718,0.7992,0.3712,0.1703,0.1611,0.2086,0.2847,0.2211,0.6134,0.5807,0.6925,0.3825,0.4303,0.7791,0.8703,1.0000,0.9212,0.9386,0.9303,0.7314,0.4791,0.2087,0.2016,0.1669,0.2872,0.4374,0.3097,0.1578,0.0553,0.0334,0.0209,0.0172,0.0180,0.0110,0.0234,0.0276,0.0032,0.0084,0.0122,0.0082,0.0143,-1
-0.0094,0.0611,0.1136,0.1203,0.0403,0.1227,0.2495,0.4566,0.6587,0.5079,0.3350,0.0834,0.3004,0.3957,0.3769,0.3828,0.1247,0.1363,0.2678,0.9188,0.9779,0.3236,0.1944,0.1874,0.0885,0.3443,0.2953,0.5908,0.4564,0.7334,0.1969,0.2790,0.6212,0.8681,0.8621,0.9380,0.8327,0.9480,0.6721,0.4436,0.5163,0.3809,0.1557,0.1449,0.2662,0.1806,0.1699,0.2559,0.1129,0.0201,0.0480,0.0234,0.0175,0.0352,0.0158,0.0326,0.0201,0.0168,0.0245,0.0154,-1
-0.1088,0.1278,0.0926,0.1234,0.1276,0.1731,0.1948,0.4262,0.6828,0.5761,0.4733,0.2362,0.1023,0.2904,0.4713,0.4659,0.1415,0.0849,0.3257,0.9007,0.9312,0.4856,0.1346,0.1604,0.2737,0.5609,0.3654,0.6139,0.5470,0.8474,0.5638,0.5443,0.5086,0.6253,0.8497,0.8406,0.8420,0.9136,0.7713,0.4882,0.3724,0.4469,0.4586,0.4491,0.5616,0.4305,0.0945,0.0794,0.0274,0.0154,0.0140,0.0455,0.0213,0.0082,0.0124,0.0167,0.0103,0.0205,0.0178,0.0187,-1
-0.0430,0.0902,0.0833,0.0813,0.0165,0.0277,0.0569,0.2057,0.3887,0.7106,0.7342,0.5033,0.3000,0.1951,0.2767,0.3737,0.2507,0.2507,0.3292,0.4871,0.6527,0.8454,0.9739,1.0000,0.6665,0.5323,0.4024,0.3444,0.4239,0.4182,0.4393,0.1162,0.4336,0.6553,0.6172,0.4373,0.4118,0.3641,0.4572,0.4367,0.2964,0.4312,0.4155,0.1824,0.1487,0.0138,0.1164,0.2052,0.1069,0.0199,0.0208,0.0176,0.0197,0.0210,0.0141,0.0049,0.0027,0.0162,0.0059,0.0021,-1
-0.0731,0.1249,0.1665,0.1496,0.1443,0.2770,0.2555,0.1712,0.0466,0.1114,0.1739,0.3160,0.3249,0.2164,0.2031,0.2580,0.1796,0.2422,0.3609,0.1810,0.2604,0.6572,0.9734,0.9757,0.8079,0.6521,0.4915,0.5363,0.7649,0.5250,0.5101,0.4219,0.4160,0.1906,0.0223,0.4219,0.5496,0.2483,0.2034,0.2729,0.2837,0.4463,0.3178,0.0807,0.1192,0.2134,0.3241,0.2945,0.1474,0.0211,0.0361,0.0444,0.0230,0.0290,0.0141,0.0161,0.0177,0.0194,0.0207,0.0057,-1
-0.0164,0.0627,0.0738,0.0608,0.0233,0.1048,0.1338,0.0644,0.1522,0.0780,0.1791,0.2681,0.1788,0.1039,0.1980,0.3234,0.3748,0.2586,0.3680,0.3508,0.5606,0.5231,0.5469,0.6954,0.6352,0.6757,0.8499,0.8025,0.6563,0.8591,0.6655,0.5369,0.3118,0.3763,0.2801,0.0875,0.3319,0.4237,0.1801,0.3743,0.4627,0.1614,0.2494,0.3202,0.2265,0.1146,0.0476,0.0943,0.0824,0.0171,0.0244,0.0258,0.0143,0.0226,0.0187,0.0185,0.0110,0.0094,0.0078,0.0112,-1
-0.0412,0.1135,0.0518,0.0232,0.0646,0.1124,0.1787,0.2407,0.2682,0.2058,0.1546,0.2671,0.3141,0.2904,0.3531,0.5079,0.4639,0.1859,0.4474,0.4079,0.5400,0.4786,0.4332,0.6113,0.5091,0.4606,0.7243,0.8987,0.8826,0.9201,0.8005,0.6033,0.2120,0.2866,0.4033,0.2803,0.3087,0.3550,0.2545,0.1432,0.5869,0.6431,0.5826,0.4286,0.4894,0.5777,0.4315,0.2640,0.1794,0.0772,0.0798,0.0376,0.0143,0.0272,0.0127,0.0166,0.0095,0.0225,0.0098,0.0085,-1
-0.0707,0.1252,0.1447,0.1644,0.1693,0.0844,0.0715,0.0947,0.1583,0.1247,0.2340,0.1764,0.2284,0.3115,0.4725,0.5543,0.5386,0.3746,0.4583,0.5961,0.7464,0.7644,0.5711,0.6257,0.6695,0.7131,0.7567,0.8077,0.8477,0.9289,0.9513,0.7995,0.4362,0.4048,0.4952,0.1712,0.3652,0.3763,0.2841,0.0427,0.5331,0.6952,0.4288,0.3063,0.5835,0.5692,0.2630,0.1196,0.0983,0.0374,0.0291,0.0156,0.0197,0.0135,0.0127,0.0138,0.0133,0.0131,0.0154,0.0218,-1
-0.0526,0.0563,0.1219,0.1206,0.0246,0.1022,0.0539,0.0439,0.2291,0.1632,0.2544,0.2807,0.3011,0.3361,0.3024,0.2285,0.2910,0.1316,0.1151,0.3404,0.5562,0.6379,0.6553,0.7384,0.6534,0.5423,0.6877,0.7325,0.7726,0.8229,0.8787,0.9108,0.6705,0.6092,0.7505,0.4775,0.1666,0.3749,0.3776,0.2106,0.5886,0.5628,0.2577,0.5245,0.6149,0.5123,0.3385,0.1499,0.0546,0.0270,0.0380,0.0339,0.0149,0.0335,0.0376,0.0174,0.0132,0.0103,0.0364,0.0208,-1
-0.0516,0.0944,0.0622,0.0415,0.0995,0.2431,0.1777,0.2018,0.2611,0.1294,0.2646,0.2778,0.4432,0.3672,0.2035,0.2764,0.3252,0.1536,0.2784,0.3508,0.5187,0.7052,0.7143,0.6814,0.5100,0.5308,0.6131,0.8388,0.9031,0.8607,0.9656,0.9168,0.7132,0.6898,0.7310,0.4134,0.1580,0.1819,0.1381,0.2960,0.6935,0.8246,0.5351,0.4403,0.6448,0.6214,0.3016,0.1379,0.0364,0.0355,0.0456,0.0432,0.0274,0.0152,0.0120,0.0129,0.0020,0.0109,0.0074,0.0078,-1
-0.0299,0.0688,0.0992,0.1021,0.0800,0.0629,0.0130,0.0813,0.1761,0.0998,0.0523,0.0904,0.2655,0.3099,0.3520,0.3892,0.3962,0.2449,0.2355,0.3045,0.3112,0.4698,0.5534,0.4532,0.4464,0.4670,0.4621,0.6988,0.7626,0.7025,0.7382,0.7446,0.7927,0.5227,0.3967,0.3042,0.1309,0.2408,0.1780,0.1598,0.5657,0.6443,0.4241,0.4567,0.5760,0.5293,0.3287,0.1283,0.0698,0.0334,0.0342,0.0459,0.0277,0.0172,0.0087,0.0046,0.0203,0.0130,0.0115,0.0015,-1
-0.0721,0.1574,0.1112,0.1085,0.0666,0.1800,0.1108,0.2794,0.1408,0.0795,0.2534,0.3920,0.3375,0.1610,0.1889,0.3308,0.2282,0.2177,0.1853,0.5167,0.5342,0.6298,0.8437,0.6756,0.5825,0.6141,0.8809,0.8375,0.3869,0.5051,0.5455,0.4241,0.1534,0.4950,0.6983,0.7109,0.5647,0.4870,0.5515,0.4433,0.5250,0.6075,0.5251,0.1359,0.4268,0.4442,0.2193,0.0900,0.1200,0.0628,0.0234,0.0309,0.0127,0.0082,0.0281,0.0117,0.0092,0.0147,0.0157,0.0129,-1
-0.1021,0.0830,0.0577,0.0627,0.0635,0.1328,0.0988,0.1787,0.1199,0.1369,0.2509,0.2631,0.2796,0.2977,0.3823,0.3129,0.3956,0.2093,0.3218,0.3345,0.3184,0.2887,0.3610,0.2566,0.4106,0.4591,0.4722,0.7278,0.7591,0.6579,0.7514,0.6666,0.4903,0.5962,0.6552,0.4014,0.1188,0.3245,0.3107,0.1354,0.5109,0.7988,0.7517,0.5508,0.5858,0.7292,0.5522,0.3339,0.1608,0.0475,0.1004,0.0709,0.0317,0.0309,0.0252,0.0087,0.0177,0.0214,0.0227,0.0106,-1
-0.0654,0.0649,0.0737,0.1132,0.2482,0.1257,0.1797,0.0989,0.2460,0.3422,0.2128,0.1377,0.4032,0.5684,0.2398,0.4331,0.5954,0.5772,0.8176,0.8835,0.5248,0.6373,0.8375,0.6699,0.7756,0.8750,0.8300,0.6896,0.3372,0.6405,0.7138,0.8202,0.6657,0.5254,0.2960,0.0704,0.0970,0.3941,0.6028,0.3521,0.3924,0.4808,0.4602,0.4164,0.5438,0.5649,0.3195,0.2484,0.1299,0.0825,0.0243,0.0210,0.0361,0.0239,0.0447,0.0394,0.0355,0.0440,0.0243,0.0098,-1
-0.0712,0.0901,0.1276,0.1497,0.1284,0.1165,0.1285,0.1684,0.1830,0.2127,0.2891,0.3985,0.4576,0.5821,0.5027,0.1930,0.2579,0.3177,0.2745,0.6186,0.8958,0.7442,0.5188,0.2811,0.1773,0.6607,0.7576,0.5122,0.4701,0.5479,0.4347,0.1276,0.0846,0.0927,0.0313,0.0998,0.1781,0.1586,0.3001,0.2208,0.1455,0.2895,0.3203,0.1414,0.0629,0.0734,0.0805,0.0608,0.0565,0.0286,0.0154,0.0154,0.0156,0.0054,0.0030,0.0048,0.0087,0.0101,0.0095,0.0068,-1
-0.0207,0.0535,0.0334,0.0818,0.0740,0.0324,0.0918,0.1070,0.1553,0.1234,0.1796,0.1787,0.1247,0.2577,0.3370,0.3990,0.1647,0.2266,0.3219,0.5356,0.8159,1.0000,0.8701,0.6889,0.6299,0.5738,0.5707,0.5976,0.4301,0.2058,0.1000,0.2247,0.2308,0.3977,0.3317,0.1726,0.1429,0.2168,0.1967,0.2140,0.3674,0.2023,0.0778,0.0925,0.2388,0.3400,0.2594,0.1102,0.0911,0.0462,0.0171,0.0033,0.0050,0.0190,0.0103,0.0121,0.0042,0.0090,0.0070,0.0099,-1
-0.0209,0.0278,0.0115,0.0445,0.0427,0.0766,0.1458,0.1430,0.1894,0.1853,0.1748,0.1556,0.1476,0.1378,0.2584,0.3827,0.4784,0.5360,0.6192,0.7912,0.9264,1.0000,0.9080,0.7435,0.5557,0.3172,0.1295,0.0598,0.2722,0.3616,0.3293,0.4855,0.3936,0.1845,0.0342,0.2489,0.3837,0.3514,0.2654,0.1760,0.1599,0.0866,0.0590,0.0813,0.0492,0.0417,0.0495,0.0367,0.0115,0.0118,0.0133,0.0096,0.0014,0.0049,0.0039,0.0029,0.0078,0.0047,0.0021,0.0011,-1
-0.0231,0.0315,0.0170,0.0226,0.0410,0.0116,0.0223,0.0805,0.2365,0.2461,0.2245,0.1520,0.1732,0.3099,0.4380,0.5595,0.6820,0.6164,0.6803,0.8435,0.9921,1.0000,0.7983,0.5426,0.3952,0.5179,0.5650,0.3042,0.1881,0.3960,0.2286,0.3544,0.4187,0.2398,0.1847,0.3760,0.4331,0.3626,0.2519,0.1870,0.1046,0.2339,0.1991,0.1100,0.0684,0.0303,0.0674,0.0785,0.0455,0.0246,0.0151,0.0125,0.0036,0.0123,0.0043,0.0114,0.0052,0.0091,0.0008,0.0092,-1
-0.0131,0.0201,0.0045,0.0217,0.0230,0.0481,0.0742,0.0333,0.1369,0.2079,0.2295,0.1990,0.1184,0.1891,0.2949,0.5343,0.6850,0.7923,0.8220,0.7290,0.7352,0.7918,0.8057,0.4898,0.1934,0.2924,0.6255,0.8546,0.8966,0.7821,0.5168,0.4840,0.4038,0.3411,0.2849,0.2353,0.2699,0.4442,0.4323,0.3314,0.1195,0.1669,0.3702,0.3072,0.0945,0.1545,0.1394,0.0772,0.0615,0.0230,0.0111,0.0168,0.0086,0.0045,0.0062,0.0065,0.0030,0.0066,0.0029,0.0053,-1
-0.0233,0.0394,0.0416,0.0547,0.0993,0.1515,0.1674,0.1513,0.1723,0.2078,0.1239,0.0236,0.1771,0.3115,0.4990,0.6707,0.7655,0.8485,0.9805,1.0000,1.0000,0.9992,0.9067,0.6803,0.5103,0.4716,0.4980,0.6196,0.7171,0.6316,0.3554,0.2897,0.4316,0.3791,0.2421,0.0944,0.0351,0.0844,0.0436,0.1130,0.2045,0.1937,0.0834,0.1502,0.1675,0.1058,0.1111,0.0849,0.0596,0.0201,0.0071,0.0104,0.0062,0.0026,0.0025,0.0061,0.0038,0.0101,0.0078,0.0006,-1
-0.0117,0.0069,0.0279,0.0583,0.0915,0.1267,0.1577,0.1927,0.2361,0.2169,0.1180,0.0754,0.2782,0.3758,0.5093,0.6592,0.7071,0.7532,0.8357,0.8593,0.9615,0.9838,0.8705,0.6403,0.5067,0.5395,0.6934,0.8487,0.8213,0.5962,0.2950,0.2758,0.2885,0.1893,0.1446,0.0955,0.0888,0.0836,0.0894,0.1547,0.2318,0.2225,0.1035,0.1721,0.2017,0.1787,0.1112,0.0398,0.0305,0.0084,0.0039,0.0053,0.0029,0.0020,0.0013,0.0029,0.0020,0.0062,0.0026,0.0052,-1
-0.0211,0.0128,0.0015,0.0450,0.0711,0.1563,0.1518,0.1206,0.1666,0.1345,0.0785,0.0367,0.1227,0.2614,0.4280,0.6122,0.7435,0.8130,0.9006,0.9603,0.9162,0.9140,0.7851,0.5134,0.3439,0.3290,0.2571,0.3685,0.5765,0.6190,0.4613,0.3615,0.4434,0.3864,0.3093,0.2138,0.1112,0.1386,0.1523,0.0996,0.1644,0.1902,0.1313,0.1776,0.2000,0.0765,0.0727,0.0749,0.0449,0.0134,0.0174,0.0117,0.0023,0.0047,0.0049,0.0031,0.0024,0.0039,0.0051,0.0015,-1
-0.0047,0.0059,0.0080,0.0554,0.0883,0.1278,0.1674,0.1373,0.2922,0.3469,0.3265,0.3263,0.2301,0.1253,0.2102,0.2401,0.1928,0.1673,0.1228,0.0902,0.1557,0.3291,0.5268,0.6740,0.7906,0.8938,0.9395,0.9493,0.9040,0.9151,0.8828,0.8086,0.7180,0.6720,0.6447,0.6879,0.6241,0.4936,0.4144,0.4240,0.4546,0.4392,0.4323,0.4921,0.4710,0.3196,0.2241,0.1806,0.0990,0.0251,0.0129,0.0095,0.0126,0.0069,0.0039,0.0068,0.0060,0.0045,0.0002,0.0029,-1
-0.0201,0.0178,0.0274,0.0232,0.0724,0.0833,0.1232,0.1298,0.2085,0.2720,0.2188,0.3037,0.2959,0.2059,0.0906,0.1610,0.1800,0.2180,0.2026,0.1506,0.0521,0.2143,0.4333,0.5943,0.6926,0.7576,0.8787,0.9060,0.8528,0.9087,0.9657,0.9306,0.7774,0.6643,0.6604,0.6884,0.6938,0.5932,0.5774,0.6223,0.5841,0.4527,0.4911,0.5762,0.5013,0.4042,0.3123,0.2232,0.1085,0.0414,0.0253,0.0131,0.0049,0.0104,0.0102,0.0092,0.0083,0.0020,0.0048,0.0036,-1
-0.0107,0.0453,0.0289,0.0713,0.1075,0.1019,0.1606,0.2119,0.3061,0.2936,0.3104,0.3431,0.2456,0.1887,0.1184,0.2080,0.2736,0.3274,0.2344,0.1260,0.0576,0.1241,0.3239,0.4357,0.5734,0.7825,0.9252,0.9349,0.9348,1.0000,0.9308,0.8478,0.7605,0.7040,0.7539,0.7990,0.7673,0.5955,0.4731,0.4840,0.4340,0.3954,0.4837,0.5379,0.4485,0.2674,0.1541,0.1359,0.0941,0.0261,0.0079,0.0164,0.0120,0.0113,0.0021,0.0097,0.0072,0.0060,0.0017,0.0036,-1
-0.0235,0.0220,0.0167,0.0516,0.0746,0.1121,0.1258,0.1717,0.3074,0.3199,0.2946,0.2484,0.2510,0.1806,0.1413,0.3019,0.3635,0.3887,0.2980,0.2219,0.1624,0.1343,0.2046,0.3791,0.5771,0.7545,0.8406,0.8547,0.9036,1.0000,0.9646,0.7912,0.6412,0.5986,0.6835,0.7771,0.8084,0.7426,0.6295,0.5708,0.4433,0.3361,0.3795,0.4950,0.4373,0.2404,0.1128,0.1654,0.0933,0.0225,0.0214,0.0221,0.0152,0.0083,0.0058,0.0023,0.0057,0.0052,0.0027,0.0021,-1
-0.0258,0.0433,0.0547,0.0681,0.0784,0.1250,0.1296,0.1729,0.2794,0.2954,0.2506,0.2601,0.2249,0.2115,0.1270,0.1193,0.1794,0.2185,0.1646,0.0740,0.0625,0.2381,0.4824,0.6372,0.7531,0.8959,0.9941,0.9957,0.9328,0.9344,0.8854,0.7690,0.6865,0.6390,0.6378,0.6629,0.5983,0.4565,0.3129,0.4158,0.4325,0.4031,0.4201,0.4557,0.3955,0.2966,0.2095,0.1558,0.0884,0.0265,0.0121,0.0091,0.0062,0.0019,0.0045,0.0079,0.0031,0.0063,0.0048,0.0050,-1
-0.0305,0.0363,0.0214,0.0227,0.0456,0.0665,0.0939,0.0972,0.2535,0.3127,0.2192,0.2621,0.2419,0.2179,0.1159,0.1237,0.0886,0.1755,0.1758,0.1540,0.0512,0.1805,0.4039,0.5697,0.6577,0.7474,0.8543,0.9085,0.8668,0.8892,0.9065,0.8522,0.7204,0.6200,0.6253,0.6848,0.7337,0.6281,0.5725,0.6119,0.5597,0.4965,0.5027,0.5772,0.5907,0.4803,0.3877,0.2779,0.1427,0.0424,0.0271,0.0200,0.0070,0.0070,0.0086,0.0089,0.0074,0.0042,0.0055,0.0021,-1
-0.0217,0.0152,0.0346,0.0346,0.0484,0.0526,0.0773,0.0862,0.1451,0.2110,0.2343,0.2087,0.1645,0.1689,0.1650,0.1967,0.2934,0.3709,0.4309,0.4161,0.5116,0.6501,0.7717,0.8491,0.9104,0.8912,0.8189,0.6779,0.5368,0.5207,0.5651,0.5749,0.5250,0.4255,0.3330,0.2331,0.1451,0.1648,0.2694,0.3730,0.4467,0.4133,0.3743,0.3021,0.2069,0.1790,0.1689,0.1341,0.0769,0.0222,0.0205,0.0123,0.0067,0.0011,0.0026,0.0049,0.0029,0.0022,0.0022,0.0032,-1
-0.0072,0.0027,0.0089,0.0061,0.0420,0.0865,0.1182,0.0999,0.1976,0.2318,0.2472,0.2880,0.2126,0.0708,0.1194,0.2808,0.4221,0.5279,0.5857,0.6153,0.6753,0.7873,0.8974,0.9828,1.0000,0.8460,0.6055,0.3036,0.0144,0.2526,0.4335,0.4918,0.5409,0.5961,0.5248,0.3777,0.2369,0.1720,0.1878,0.3250,0.2575,0.2423,0.2706,0.2323,0.1724,0.1457,0.1175,0.0868,0.0392,0.0131,0.0092,0.0078,0.0071,0.0081,0.0034,0.0064,0.0037,0.0036,0.0012,0.0037,-1
-0.0163,0.0198,0.0202,0.0386,0.0752,0.1444,0.1487,0.1484,0.2442,0.2822,0.3691,0.3750,0.3927,0.3308,0.1085,0.1139,0.3446,0.5441,0.6470,0.7276,0.7894,0.8264,0.8697,0.7836,0.7140,0.5698,0.2908,0.4636,0.6409,0.7405,0.8069,0.8420,1.0000,0.9536,0.6755,0.3905,0.1249,0.3629,0.6356,0.8116,0.7664,0.5417,0.2614,0.1723,0.2814,0.2764,0.1985,0.1502,0.1219,0.0493,0.0027,0.0077,0.0026,0.0031,0.0083,0.0020,0.0084,0.0108,0.0083,0.0033,-1
-0.0221,0.0065,0.0164,0.0487,0.0519,0.0849,0.0812,0.1833,0.2228,0.1810,0.2549,0.2984,0.2624,0.1893,0.0668,0.2666,0.4274,0.6291,0.7782,0.7686,0.8099,0.8493,0.9440,0.9450,0.9655,0.8045,0.4969,0.3960,0.3856,0.5574,0.7309,0.8549,0.9425,0.8726,0.6673,0.4694,0.1546,0.1748,0.3607,0.5208,0.5177,0.3702,0.2240,0.0816,0.0395,0.0785,0.1052,0.1034,0.0764,0.0216,0.0167,0.0089,0.0051,0.0015,0.0075,0.0058,0.0016,0.0070,0.0074,0.0038,-1
-0.0411,0.0277,0.0604,0.0525,0.0489,0.0385,0.0611,0.1117,0.1237,0.2300,0.1370,0.1335,0.2137,0.1526,0.0775,0.1196,0.0903,0.0689,0.2071,0.2975,0.2836,0.3353,0.3622,0.3202,0.3452,0.3562,0.3892,0.6622,0.9254,1.0000,0.8528,0.6297,0.5250,0.4012,0.2901,0.2007,0.3356,0.4799,0.6147,0.6246,0.4973,0.3492,0.2662,0.3137,0.4282,0.4262,0.3511,0.2458,0.1259,0.0327,0.0181,0.0217,0.0038,0.0019,0.0065,0.0132,0.0108,0.0050,0.0085,0.0044,-1
-0.0137,0.0297,0.0116,0.0082,0.0241,0.0253,0.0279,0.0130,0.0489,0.0874,0.1100,0.1084,0.1094,0.1023,0.0601,0.0906,0.1313,0.2758,0.3660,0.5269,0.5810,0.6181,0.5875,0.4639,0.5424,0.7367,0.9089,1.0000,0.8247,0.5441,0.3349,0.0877,0.1600,0.4169,0.6576,0.7390,0.7963,0.7493,0.6795,0.4713,0.2355,0.1704,0.2728,0.4016,0.4125,0.3470,0.2739,0.1790,0.0922,0.0276,0.0169,0.0081,0.0040,0.0025,0.0036,0.0058,0.0067,0.0035,0.0043,0.0033,-1
-0.0015,0.0186,0.0289,0.0195,0.0515,0.0817,0.1005,0.0124,0.1168,0.1476,0.2118,0.2575,0.2354,0.1334,0.0092,0.1951,0.3685,0.4646,0.5418,0.6260,0.7420,0.8257,0.8609,0.8400,0.8949,0.9945,1.0000,0.9649,0.8747,0.6257,0.2184,0.2945,0.3645,0.5012,0.7843,0.9361,0.8195,0.6207,0.4513,0.3004,0.2674,0.2241,0.3141,0.3693,0.2986,0.2226,0.0849,0.0359,0.0289,0.0122,0.0045,0.0108,0.0075,0.0089,0.0036,0.0029,0.0013,0.0010,0.0032,0.0047,-1
-0.0130,0.0120,0.0436,0.0624,0.0428,0.0349,0.0384,0.0446,0.1318,0.1375,0.2026,0.2389,0.2112,0.1444,0.0742,0.1533,0.3052,0.4116,0.5466,0.5933,0.6663,0.7333,0.7136,0.7014,0.7758,0.9137,0.9964,1.0000,0.8881,0.6585,0.2707,0.1746,0.2709,0.4853,0.7184,0.8209,0.7536,0.6496,0.4708,0.3482,0.3508,0.3181,0.3524,0.3659,0.2846,0.1714,0.0694,0.0303,0.0292,0.0116,0.0024,0.0084,0.0100,0.0018,0.0035,0.0058,0.0011,0.0009,0.0033,0.0026,-1
-0.0134,0.0172,0.0178,0.0363,0.0444,0.0744,0.0800,0.0456,0.0368,0.1250,0.2405,0.2325,0.2523,0.1472,0.0669,0.1100,0.2353,0.3282,0.4416,0.5167,0.6508,0.7793,0.7978,0.7786,0.8587,0.9321,0.9454,0.8645,0.7220,0.4850,0.1357,0.2951,0.4715,0.6036,0.8083,0.9870,0.8800,0.6411,0.4276,0.2702,0.2642,0.3342,0.4335,0.4542,0.3960,0.2525,0.1084,0.0372,0.0286,0.0099,0.0046,0.0094,0.0048,0.0047,0.0016,0.0008,0.0042,0.0024,0.0027,0.0041,-1
-0.0179,0.0136,0.0408,0.0633,0.0596,0.0808,0.2090,0.3465,0.5276,0.5965,0.6254,0.4507,0.3693,0.2864,0.1635,0.0422,0.1785,0.4394,0.6950,0.8097,0.8550,0.8717,0.8601,0.9201,0.8729,0.8084,0.8694,0.8411,0.5793,0.3754,0.3485,0.4639,0.6495,0.6901,0.5666,0.5188,0.5060,0.3885,0.3762,0.3738,0.2605,0.1591,0.1875,0.2267,0.1577,0.1211,0.0883,0.0850,0.0355,0.0219,0.0086,0.0123,0.0060,0.0187,0.0111,0.0126,0.0081,0.0155,0.0160,0.0085,-1
-0.0180,0.0444,0.0476,0.0698,0.1615,0.0887,0.0596,0.1071,0.3175,0.2918,0.3273,0.3035,0.3033,0.2587,0.1682,0.1308,0.2803,0.4519,0.6641,0.7683,0.6960,0.4393,0.2432,0.2886,0.4974,0.8172,1.0000,0.9238,0.8519,0.7722,0.5772,0.5190,0.6824,0.6220,0.5054,0.3578,0.3809,0.3813,0.3359,0.2771,0.3648,0.3834,0.3453,0.2096,0.1031,0.0798,0.0701,0.0526,0.0241,0.0117,0.0122,0.0122,0.0114,0.0098,0.0027,0.0025,0.0026,0.0050,0.0073,0.0022,-1
-0.0329,0.0216,0.0386,0.0627,0.1158,0.1482,0.2054,0.1605,0.2532,0.2672,0.3056,0.3161,0.2314,0.2067,0.1804,0.2808,0.4423,0.5947,0.6601,0.5844,0.4539,0.4789,0.5646,0.5281,0.7115,1.0000,0.9564,0.6090,0.5112,0.4000,0.0482,0.1852,0.2186,0.1436,0.1757,0.1428,0.1644,0.3089,0.3648,0.4441,0.3859,0.2813,0.1238,0.0953,0.1201,0.0825,0.0618,0.0141,0.0108,0.0124,0.0104,0.0095,0.0151,0.0059,0.0015,0.0053,0.0016,0.0042,0.0053,0.0074,-1
-0.0191,0.0173,0.0291,0.0301,0.0463,0.0690,0.0576,0.1103,0.2423,0.3134,0.4786,0.5239,0.4393,0.3440,0.2869,0.3889,0.4420,0.3892,0.4088,0.5006,0.7271,0.9385,1.0000,0.9831,0.9932,0.9161,0.8237,0.6957,0.4536,0.3281,0.2522,0.3964,0.4154,0.3308,0.1445,0.1923,0.3208,0.3367,0.5683,0.5505,0.3231,0.0448,0.3131,0.3387,0.4130,0.3639,0.2069,0.0859,0.0600,0.0267,0.0125,0.0040,0.0136,0.0137,0.0172,0.0132,0.0110,0.0122,0.0114,0.0068,-1
-0.0294,0.0123,0.0117,0.0113,0.0497,0.0998,0.1326,0.1117,0.2984,0.3473,0.4231,0.5044,0.5237,0.4398,0.3236,0.2956,0.3286,0.3231,0.4528,0.6339,0.7044,0.8314,0.8449,0.8512,0.9138,0.9985,1.0000,0.7544,0.4661,0.3924,0.3849,0.4674,0.4245,0.3095,0.0752,0.2885,0.4072,0.3170,0.2863,0.2634,0.0541,0.1874,0.3459,0.4646,0.4366,0.2581,0.1319,0.0505,0.0112,0.0059,0.0041,0.0056,0.0104,0.0079,0.0014,0.0054,0.0015,0.0006,0.0081,0.0043,-1
-0.0635,0.0709,0.0453,0.0333,0.0185,0.1260,0.1015,0.1918,0.3362,0.3900,0.4674,0.5632,0.5506,0.4343,0.3052,0.3492,0.3975,0.3875,0.5280,0.7198,0.7702,0.8562,0.8688,0.9236,1.0000,0.9662,0.9822,0.7360,0.4158,0.2918,0.3280,0.3690,0.3450,0.2863,0.0864,0.3724,0.4649,0.3488,0.1817,0.1142,0.1220,0.2621,0.4461,0.4726,0.3263,0.1423,0.0390,0.0406,0.0311,0.0086,0.0154,0.0048,0.0025,0.0087,0.0072,0.0095,0.0086,0.0085,0.0040,0.0051,-1
-0.0201,0.0165,0.0344,0.0330,0.0397,0.0443,0.0684,0.0903,0.1739,0.2571,0.2931,0.3108,0.3603,0.3002,0.2718,0.2007,0.1801,0.2234,0.3568,0.5492,0.7209,0.8318,0.8864,0.9520,0.9637,1.0000,0.9673,0.8664,0.7896,0.6345,0.5351,0.4056,0.2563,0.2894,0.3588,0.4296,0.4773,0.4516,0.3765,0.3051,0.1921,0.1184,0.1984,0.1570,0.0660,0.1294,0.0797,0.0052,0.0233,0.0152,0.0125,0.0054,0.0057,0.0137,0.0109,0.0035,0.0056,0.0105,0.0082,0.0036,-1
-0.0197,0.0394,0.0384,0.0076,0.0251,0.0629,0.0747,0.0578,0.1357,0.1695,0.1734,0.2470,0.3141,0.3297,0.2759,0.2056,0.1162,0.1884,0.3390,0.3926,0.4282,0.5418,0.6448,0.7223,0.7853,0.7984,0.8847,0.9582,0.8990,0.6831,0.6108,0.5480,0.5058,0.4476,0.2401,0.1405,0.1772,0.1742,0.3326,0.4021,0.3009,0.2075,0.1206,0.0255,0.0298,0.0691,0.0781,0.0777,0.0369,0.0057,0.0091,0.0134,0.0097,0.0042,0.0058,0.0072,0.0041,0.0045,0.0047,0.0054,-1
-0.0394,0.0420,0.0446,0.0551,0.0597,0.1416,0.0956,0.0802,0.1618,0.2558,0.3078,0.3404,0.3400,0.3951,0.3352,0.2252,0.2086,0.2248,0.3382,0.4578,0.6474,0.6708,0.7007,0.7619,0.7745,0.6767,0.7373,0.7834,0.9619,1.0000,0.8086,0.5558,0.5409,0.4988,0.3108,0.2897,0.2244,0.0960,0.2287,0.3228,0.3454,0.3882,0.3240,0.0926,0.1173,0.0566,0.0766,0.0969,0.0588,0.0050,0.0118,0.0146,0.0040,0.0114,0.0032,0.0062,0.0101,0.0068,0.0053,0.0087,-1
-0.0310,0.0221,0.0433,0.0191,0.0964,0.1827,0.1106,0.1702,0.2804,0.4432,0.5222,0.5611,0.5379,0.4048,0.2245,0.1784,0.2297,0.2720,0.5209,0.6898,0.8202,0.8780,0.7600,0.7616,0.7152,0.7288,0.8686,0.9509,0.8348,0.5730,0.4363,0.4289,0.4240,0.3156,0.1287,0.1477,0.2062,0.2400,0.5173,0.5168,0.1491,0.2407,0.3415,0.4494,0.4624,0.2001,0.0775,0.1232,0.0783,0.0089,0.0249,0.0204,0.0059,0.0053,0.0079,0.0037,0.0015,0.0056,0.0067,0.0054,-1
-0.0423,0.0321,0.0709,0.0108,0.1070,0.0973,0.0961,0.1323,0.2462,0.2696,0.3412,0.4292,0.3682,0.3940,0.2965,0.3172,0.2825,0.3050,0.2408,0.5420,0.6802,0.6320,0.5824,0.6805,0.5984,0.8412,0.9911,0.9187,0.8005,0.6713,0.5632,0.7332,0.6038,0.2575,0.0349,0.1799,0.3039,0.4760,0.5756,0.4254,0.5046,0.7179,0.6163,0.5663,0.5749,0.3593,0.2526,0.2299,0.1271,0.0356,0.0367,0.0176,0.0035,0.0093,0.0121,0.0075,0.0056,0.0021,0.0043,0.0017,-1
-0.0095,0.0308,0.0539,0.0411,0.0613,0.1039,0.1016,0.1394,0.2592,0.3745,0.4229,0.4499,0.5404,0.4303,0.3333,0.3496,0.3426,0.2851,0.4062,0.6833,0.7650,0.6670,0.5703,0.5995,0.6484,0.8614,0.9819,0.9380,0.8435,0.6074,0.5403,0.6890,0.5977,0.3244,0.0516,0.3157,0.3590,0.3881,0.5716,0.4314,0.3051,0.4393,0.4302,0.4831,0.5084,0.1952,0.1539,0.2037,0.1054,0.0251,0.0357,0.0181,0.0019,0.0102,0.0133,0.0040,0.0042,0.0030,0.0031,0.0033,-1
-0.0096,0.0404,0.0682,0.0688,0.0887,0.0932,0.0955,0.2140,0.2546,0.2952,0.4025,0.5148,0.4901,0.4127,0.3575,0.3447,0.3068,0.2945,0.4351,0.7264,0.8147,0.8103,0.6665,0.6958,0.7748,0.8688,1.0000,0.9941,0.8793,0.6482,0.5876,0.6408,0.4972,0.2755,0.0300,0.3356,0.3167,0.4133,0.6281,0.4977,0.2613,0.4697,0.4806,0.4921,0.5294,0.2216,0.1401,0.1888,0.0947,0.0134,0.0310,0.0237,0.0078,0.0144,0.0170,0.0012,0.0109,0.0036,0.0043,0.0018,-1
-0.0269,0.0383,0.0505,0.0707,0.1313,0.2103,0.2263,0.2524,0.3595,0.5915,0.6675,0.5679,0.5175,0.3334,0.2002,0.2856,0.2937,0.3424,0.5949,0.7526,0.8959,0.8147,0.7109,0.7378,0.7201,0.8254,0.8917,0.9820,0.8179,0.4848,0.3203,0.2775,0.2382,0.2911,0.1675,0.3156,0.1869,0.3391,0.5993,0.4124,0.1181,0.3651,0.4655,0.4777,0.3517,0.0920,0.1227,0.1785,0.1085,0.0300,0.0346,0.0167,0.0199,0.0145,0.0081,0.0045,0.0043,0.0027,0.0055,0.0057,-1
-0.0340,0.0625,0.0381,0.0257,0.0441,0.1027,0.1287,0.1850,0.2647,0.4117,0.5245,0.5341,0.5554,0.3915,0.2950,0.3075,0.3021,0.2719,0.5443,0.7932,0.8751,0.8667,0.7107,0.6911,0.7287,0.8792,1.0000,0.9816,0.8984,0.6048,0.4934,0.5371,0.4586,0.2908,0.0774,0.2249,0.1602,0.3958,0.6117,0.5196,0.2321,0.4370,0.3797,0.4322,0.4892,0.1901,0.0940,0.1364,0.0906,0.0144,0.0329,0.0141,0.0019,0.0067,0.0099,0.0042,0.0057,0.0051,0.0033,0.0058,-1
-0.0209,0.0191,0.0411,0.0321,0.0698,0.1579,0.1438,0.1402,0.3048,0.3914,0.3504,0.3669,0.3943,0.3311,0.3331,0.3002,0.2324,0.1381,0.3450,0.4428,0.4890,0.3677,0.4379,0.4864,0.6207,0.7256,0.6624,0.7689,0.7981,0.8577,0.9273,0.7009,0.4851,0.3409,0.1406,0.1147,0.1433,0.1820,0.3605,0.5529,0.5988,0.5077,0.5512,0.5027,0.7034,0.5904,0.4069,0.2761,0.1584,0.0510,0.0054,0.0078,0.0201,0.0104,0.0039,0.0031,0.0062,0.0087,0.0070,0.0042,-1
-0.0368,0.0279,0.0103,0.0566,0.0759,0.0679,0.0970,0.1473,0.2164,0.2544,0.2936,0.2935,0.2657,0.3187,0.2794,0.2534,0.1980,0.1929,0.2826,0.3245,0.3504,0.3324,0.4217,0.4774,0.4808,0.6325,0.8334,0.9458,1.0000,0.8425,0.5524,0.4795,0.5200,0.3968,0.1940,0.1519,0.2010,0.1736,0.1029,0.2244,0.3717,0.4449,0.3939,0.2030,0.2010,0.2187,0.1840,0.1477,0.0971,0.0224,0.0151,0.0105,0.0024,0.0018,0.0057,0.0092,0.0009,0.0086,0.0110,0.0052,-1
-0.0089,0.0274,0.0248,0.0237,0.0224,0.0845,0.1488,0.1224,0.1569,0.2119,0.3003,0.3094,0.2743,0.2547,0.1870,0.1452,0.1457,0.2429,0.3259,0.3679,0.3355,0.3100,0.3914,0.5280,0.6409,0.7707,0.8754,1.0000,0.9806,0.6969,0.4973,0.5020,0.5359,0.3842,0.1848,0.1149,0.1570,0.1311,0.1583,0.2631,0.3103,0.4512,0.3785,0.1269,0.1459,0.1092,0.1485,0.1385,0.0716,0.0176,0.0199,0.0096,0.0103,0.0093,0.0025,0.0044,0.0021,0.0069,0.0060,0.0018,-1
-0.0158,0.0239,0.0150,0.0494,0.0988,0.1425,0.1463,0.1219,0.1697,0.1923,0.2361,0.2719,0.3049,0.2986,0.2226,0.1745,0.2459,0.3100,0.3572,0.4283,0.4268,0.3735,0.4585,0.6094,0.7221,0.7595,0.8706,1.0000,0.9815,0.7187,0.5848,0.4192,0.3756,0.3263,0.1944,0.1394,0.1670,0.1275,0.1666,0.2574,0.2258,0.2777,0.1613,0.1335,0.1976,0.1234,0.1554,0.1057,0.0490,0.0097,0.0223,0.0121,0.0108,0.0057,0.0028,0.0079,0.0034,0.0046,0.0022,0.0021,-1
-0.0156,0.0210,0.0282,0.0596,0.0462,0.0779,0.1365,0.0780,0.1038,0.1567,0.2476,0.2783,0.2896,0.2956,0.3189,0.1892,0.1730,0.2226,0.2427,0.3149,0.4102,0.3808,0.4896,0.6292,0.7519,0.7985,0.8830,0.9915,0.9223,0.6981,0.6167,0.5069,0.3921,0.3524,0.2183,0.1245,0.1592,0.1626,0.2356,0.2483,0.2437,0.2715,0.1184,0.1157,0.1449,0.1883,0.1954,0.1492,0.0511,0.0155,0.0189,0.0150,0.0060,0.0082,0.0091,0.0038,0.0056,0.0056,0.0048,0.0024,-1
-0.0315,0.0252,0.0167,0.0479,0.0902,0.1057,0.1024,0.1209,0.1241,0.1533,0.2128,0.2536,0.2686,0.2803,0.1886,0.1485,0.2160,0.2417,0.2989,0.3341,0.3786,0.3956,0.5232,0.6913,0.7868,0.8337,0.9199,1.0000,0.8990,0.6456,0.5967,0.4355,0.2997,0.2294,0.1866,0.0922,0.1829,0.1743,0.2452,0.2407,0.2518,0.3184,0.1685,0.0675,0.1186,0.1833,0.1878,0.1114,0.0310,0.0143,0.0138,0.0108,0.0062,0.0044,0.0072,0.0007,0.0054,0.0035,0.0001,0.0055,-1
-0.0056,0.0267,0.0221,0.0561,0.0936,0.1146,0.0706,0.0996,0.1673,0.1859,0.2481,0.2712,0.2934,0.2637,0.1880,0.1405,0.2028,0.2613,0.2778,0.3346,0.3830,0.4003,0.5114,0.6860,0.7490,0.7843,0.9021,1.0000,0.8888,0.6511,0.6083,0.4463,0.2948,0.1729,0.1488,0.0801,0.1770,0.1382,0.2404,0.2046,0.1970,0.2778,0.1377,0.0685,0.0664,0.1665,0.1807,0.1245,0.0516,0.0044,0.0185,0.0072,0.0055,0.0074,0.0068,0.0084,0.0037,0.0024,0.0034,0.0007,-1
-0.0203,0.0121,0.0380,0.0128,0.0537,0.0874,0.1021,0.0852,0.1136,0.1747,0.2198,0.2721,0.2105,0.1727,0.2040,0.1786,0.1318,0.2260,0.2358,0.3107,0.3906,0.3631,0.4809,0.6531,0.7812,0.8395,0.9180,0.9769,0.8937,0.7022,0.6500,0.5069,0.3903,0.3009,0.1565,0.0985,0.2200,0.2243,0.2736,0.2152,0.2438,0.3154,0.2112,0.0991,0.0594,0.1940,0.1937,0.1082,0.0336,0.0177,0.0209,0.0134,0.0094,0.0047,0.0045,0.0042,0.0028,0.0036,0.0013,0.0016,-1
-0.0392,0.0108,0.0267,0.0257,0.0410,0.0491,0.1053,0.1690,0.2105,0.2471,0.2680,0.3049,0.2863,0.2294,0.1165,0.2127,0.2062,0.2222,0.3241,0.4330,0.5071,0.5944,0.7078,0.7641,0.8878,0.9711,0.9880,0.9812,0.9464,0.8542,0.6457,0.3397,0.3828,0.3204,0.1331,0.0440,0.1234,0.2030,0.1652,0.1043,0.1066,0.2110,0.2417,0.1631,0.0769,0.0723,0.0912,0.0812,0.0496,0.0101,0.0089,0.0083,0.0080,0.0026,0.0079,0.0042,0.0071,0.0044,0.0022,0.0014,-1
-0.0129,0.0141,0.0309,0.0375,0.0767,0.0787,0.0662,0.1108,0.1777,0.2245,0.2431,0.3134,0.3206,0.2917,0.2249,0.2347,0.2143,0.2939,0.4898,0.6127,0.7531,0.7718,0.7432,0.8673,0.9308,0.9836,1.0000,0.9595,0.8722,0.6862,0.4901,0.3280,0.3115,0.1969,0.1019,0.0317,0.0756,0.0907,0.1066,0.1380,0.0665,0.1475,0.2470,0.2788,0.2709,0.2283,0.1818,0.1185,0.0546,0.0219,0.0204,0.0124,0.0093,0.0072,0.0019,0.0027,0.0054,0.0017,0.0024,0.0029,-1
-0.0050,0.0017,0.0270,0.0450,0.0958,0.0830,0.0879,0.1220,0.1977,0.2282,0.2521,0.3484,0.3309,0.2614,0.1782,0.2055,0.2298,0.3545,0.6218,0.7265,0.8346,0.8268,0.8366,0.9408,0.9510,0.9801,0.9974,1.0000,0.9036,0.6409,0.3857,0.2908,0.2040,0.1653,0.1769,0.1140,0.0740,0.0941,0.0621,0.0426,0.0572,0.1068,0.1909,0.2229,0.2203,0.2265,0.1766,0.1097,0.0558,0.0142,0.0281,0.0165,0.0056,0.0010,0.0027,0.0062,0.0024,0.0063,0.0017,0.0028,-1
-0.0366,0.0421,0.0504,0.0250,0.0596,0.0252,0.0958,0.0991,0.1419,0.1847,0.2222,0.2648,0.2508,0.2291,0.1555,0.1863,0.2387,0.3345,0.5233,0.6684,0.7766,0.7928,0.7940,0.9129,0.9498,0.9835,1.0000,0.9471,0.8237,0.6252,0.4181,0.3209,0.2658,0.2196,0.1588,0.0561,0.0948,0.1700,0.1215,0.1282,0.0386,0.1329,0.2331,0.2468,0.1960,0.1985,0.1570,0.0921,0.0549,0.0194,0.0166,0.0132,0.0027,0.0022,0.0059,0.0016,0.0025,0.0017,0.0027,0.0027,-1
-0.0238,0.0318,0.0422,0.0399,0.0788,0.0766,0.0881,0.1143,0.1594,0.2048,0.2652,0.3100,0.2381,0.1918,0.1430,0.1735,0.1781,0.2852,0.5036,0.6166,0.7616,0.8125,0.7793,0.8788,0.8813,0.9470,1.0000,0.9739,0.8446,0.6151,0.4302,0.3165,0.2869,0.2017,0.1206,0.0271,0.0580,0.1262,0.1072,0.1082,0.0360,0.1197,0.2061,0.2054,0.1878,0.2047,0.1716,0.1069,0.0477,0.0170,0.0186,0.0096,0.0071,0.0084,0.0038,0.0026,0.0028,0.0013,0.0035,0.0060,-1
-0.0116,0.0744,0.0367,0.0225,0.0076,0.0545,0.1110,0.1069,0.1708,0.2271,0.3171,0.2882,0.2657,0.2307,0.1889,0.1791,0.2298,0.3715,0.6223,0.7260,0.7934,0.8045,0.8067,0.9173,0.9327,0.9562,1.0000,0.9818,0.8684,0.6381,0.3997,0.3242,0.2835,0.2413,0.2321,0.1260,0.0693,0.0701,0.1439,0.1475,0.0438,0.0469,0.1476,0.1742,0.1555,0.1651,0.1181,0.0720,0.0321,0.0056,0.0202,0.0141,0.0103,0.0100,0.0034,0.0026,0.0037,0.0044,0.0057,0.0035,-1
-0.0131,0.0387,0.0329,0.0078,0.0721,0.1341,0.1626,0.1902,0.2610,0.3193,0.3468,0.3738,0.3055,0.1926,0.1385,0.2122,0.2758,0.4576,0.6487,0.7154,0.8010,0.7924,0.8793,1.0000,0.9865,0.9474,0.9474,0.9315,0.8326,0.6213,0.3772,0.2822,0.2042,0.2190,0.2223,0.1327,0.0521,0.0618,0.1416,0.1460,0.0846,0.1055,0.1639,0.1916,0.2085,0.2335,0.1964,0.1300,0.0633,0.0183,0.0137,0.0150,0.0076,0.0032,0.0037,0.0071,0.0040,0.0009,0.0015,0.0085,-1
-0.0335,0.0258,0.0398,0.0570,0.0529,0.1091,0.1709,0.1684,0.1865,0.2660,0.3188,0.3553,0.3116,0.1965,0.1780,0.2794,0.2870,0.3969,0.5599,0.6936,0.7969,0.7452,0.8203,0.9261,0.8810,0.8814,0.9301,0.9955,0.8576,0.6069,0.3934,0.2464,0.1645,0.1140,0.0956,0.0080,0.0702,0.0936,0.0894,0.1127,0.0873,0.1020,0.1964,0.2256,0.1814,0.2012,0.1688,0.1037,0.0501,0.0136,0.0130,0.0120,0.0039,0.0053,0.0062,0.0046,0.0045,0.0022,0.0005,0.0031,-1
-0.0272,0.0378,0.0488,0.0848,0.1127,0.1103,0.1349,0.2337,0.3113,0.3997,0.3941,0.3309,0.2926,0.1760,0.1739,0.2043,0.2088,0.2678,0.2434,0.1839,0.2802,0.6172,0.8015,0.8313,0.8440,0.8494,0.9168,1.0000,0.7896,0.5371,0.6472,0.6505,0.4959,0.2175,0.0990,0.0434,0.1708,0.1979,0.1880,0.1108,0.1702,0.0585,0.0638,0.1391,0.0638,0.0581,0.0641,0.1044,0.0732,0.0275,0.0146,0.0091,0.0045,0.0043,0.0043,0.0098,0.0054,0.0051,0.0065,0.0103,-1
-0.0187,0.0346,0.0168,0.0177,0.0393,0.1630,0.2028,0.1694,0.2328,0.2684,0.3108,0.2933,0.2275,0.0994,0.1801,0.2200,0.2732,0.2862,0.2034,0.1740,0.4130,0.6879,0.8120,0.8453,0.8919,0.9300,0.9987,1.0000,0.8104,0.6199,0.6041,0.5547,0.4160,0.1472,0.0849,0.0608,0.0969,0.1411,0.1676,0.1200,0.1201,0.1036,0.1977,0.1339,0.0902,0.1085,0.1521,0.1363,0.0858,0.0290,0.0203,0.0116,0.0098,0.0199,0.0033,0.0101,0.0065,0.0115,0.0193,0.0157,-1
-0.0323,0.0101,0.0298,0.0564,0.0760,0.0958,0.0990,0.1018,0.1030,0.2154,0.3085,0.3425,0.2990,0.1402,0.1235,0.1534,0.1901,0.2429,0.2120,0.2395,0.3272,0.5949,0.8302,0.9045,0.9888,0.9912,0.9448,1.0000,0.9092,0.7412,0.7691,0.7117,0.5304,0.2131,0.0928,0.1297,0.1159,0.1226,0.1768,0.0345,0.1562,0.0824,0.1149,0.1694,0.0954,0.0080,0.0790,0.1255,0.0647,0.0179,0.0051,0.0061,0.0093,0.0135,0.0063,0.0063,0.0034,0.0032,0.0062,0.0067,-1
-0.0522,0.0437,0.0180,0.0292,0.0351,0.1171,0.1257,0.1178,0.1258,0.2529,0.2716,0.2374,0.1878,0.0983,0.0683,0.1503,0.1723,0.2339,0.1962,0.1395,0.3164,0.5888,0.7631,0.8473,0.9424,0.9986,0.9699,1.0000,0.8630,0.6979,0.7717,0.7305,0.5197,0.1786,0.1098,0.1446,0.1066,0.1440,0.1929,0.0325,0.1490,0.0328,0.0537,0.1309,0.0910,0.0757,0.1059,0.1005,0.0535,0.0235,0.0155,0.0160,0.0029,0.0051,0.0062,0.0089,0.0140,0.0138,0.0077,0.0031,-1
-0.0303,0.0353,0.0490,0.0608,0.0167,0.1354,0.1465,0.1123,0.1945,0.2354,0.2898,0.2812,0.1578,0.0273,0.0673,0.1444,0.2070,0.2645,0.2828,0.4293,0.5685,0.6990,0.7246,0.7622,0.9242,1.0000,0.9979,0.8297,0.7032,0.7141,0.6893,0.4961,0.2584,0.0969,0.0776,0.0364,0.1572,0.1823,0.1349,0.0849,0.0492,0.1367,0.1552,0.1548,0.1319,0.0985,0.1258,0.0954,0.0489,0.0241,0.0042,0.0086,0.0046,0.0126,0.0036,0.0035,0.0034,0.0079,0.0036,0.0048,-1
-0.0260,0.0363,0.0136,0.0272,0.0214,0.0338,0.0655,0.1400,0.1843,0.2354,0.2720,0.2442,0.1665,0.0336,0.1302,0.1708,0.2177,0.3175,0.3714,0.4552,0.5700,0.7397,0.8062,0.8837,0.9432,1.0000,0.9375,0.7603,0.7123,0.8358,0.7622,0.4567,0.1715,0.1549,0.1641,0.1869,0.2655,0.1713,0.0959,0.0768,0.0847,0.2076,0.2505,0.1862,0.1439,0.1470,0.0991,0.0041,0.0154,0.0116,0.0181,0.0146,0.0129,0.0047,0.0039,0.0061,0.0040,0.0036,0.0061,0.0115,-1
-

GPy/util/datasets/thyroid_gland.dat

@@ -1,216 +0,0 @@
-1,107,10.1,2.2,0.9,2.7
-1,113,9.9,3.1,2.0,5.9
-1,127,12.9,2.4,1.4,0.6
-1,109,5.3,1.6,1.4,1.5
-1,105,7.3,1.5,1.5,-0.1
-1,105,6.1,2.1,1.4,7.0
-1,110,10.4,1.6,1.6,2.7
-1,114,9.9,2.4,1.5,5.7
-1,106,9.4,2.2,1.5,0.0
-1,107,13.0,1.1,0.9,3.1
-1,106,4.2,1.2,1.6,1.4
-1,110,11.3,2.3,0.9,3.3
-1,116,9.2,2.7,1.0,4.2
-1,112,8.1,1.9,3.7,2.0
-1,122,9.7,1.6,0.9,2.2
-1,109,8.4,2.1,1.1,3.6
-1,111,8.4,1.5,0.8,1.2
-1,114,6.7,1.5,1.0,3.5
-1,119,10.6,2.1,1.3,1.1
-1,115,7.1,1.3,1.3,2.0
-1,101,7.8,1.2,1.0,1.7
-1,103,10.1,1.3,0.7,0.1
-1,109,10.4,1.9,0.4,-0.1
-1,102,7.6,1.8,2.0,2.5
-1,121,10.1,1.7,1.3,0.1
-1,100,6.1,2.4,1.8,3.8
-1,106,9.6,2.4,1.0,1.3
-1,116,10.1,2.2,1.6,0.8
-1,105,11.1,2.0,1.0,1.0
-1,110,10.4,1.8,1.0,2.3
-1,120,8.4,1.1,1.4,1.4
-1,116,11.1,2.0,1.2,2.3
-1,110,7.8,1.9,2.1,6.4
-1,90,8.1,1.6,1.4,1.1
-1,117,12.2,1.9,1.2,3.9
-1,117,11.0,1.4,1.5,2.1
-1,113,9.0,2.0,1.8,1.6
-1,106,9.4,1.5,0.8,0.5
-1,130,9.5,1.7,0.4,3.2
-1,100,10.5,2.4,0.9,1.9
-1,121,10.1,2.4,0.8,3.0
-1,110,9.2,1.6,1.5,0.3
-1,129,11.9,2.7,1.2,3.5
-1,121,13.5,1.5,1.6,0.5
-1,123,8.1,2.3,1.0,5.1
-1,107,8.4,1.8,1.5,0.8
-1,109,10.0,1.3,1.8,4.3
-1,120,6.8,1.9,1.3,1.9
-1,100,9.5,2.5,1.3,-0.2
-1,118,8.1,1.9,1.5,13.7
-1,100,11.3,2.5,0.7,-0.3
-1,103,12.2,1.2,1.3,2.7
-1,115,8.1,1.7,0.6,2.2
-1,119,8.0,2.0,0.6,3.2
-1,106,9.4,1.7,0.9,3.1
-1,114,10.9,2.1,0.3,1.4
-1,93,8.9,1.5,0.8,2.7
-1,120,10.4,2.1,1.1,1.8
-1,106,11.3,1.8,0.9,1.0
-1,110,8.7,1.9,1.6,4.4
-1,103,8.1,1.4,0.5,3.8
-1,101,7.1,2.2,0.8,2.2
-1,115,10.4,1.8,1.6,2.0
-1,116,10.0,1.7,1.5,4.3
-1,117,9.2,1.9,1.5,6.8
-1,106,6.7,1.5,1.2,3.9
-1,118,10.5,2.1,0.7,3.5
-1,97,7.8,1.3,1.2,0.9
-1,113,11.1,1.7,0.8,2.3
-1,104,6.3,2.0,1.2,4.0
-1,96,9.4,1.5,1.0,3.1
-1,120,12.4,2.4,0.8,1.9
-1,133,9.7,2.9,0.8,1.9
-1,126,9.4,2.3,1.0,4.0
-1,113,8.5,1.8,0.8,0.5
-1,109,9.7,1.4,1.1,2.1
-1,119,12.9,1.5,1.3,3.6
-1,101,7.1,1.6,1.5,1.6
-1,108,10.4,2.1,1.3,2.4
-1,117,6.7,2.2,1.8,6.7
-1,115,15.3,2.3,2.0,2.0
-1,91,8.0,1.7,2.1,4.6
-1,103,8.5,1.8,1.9,1.1
-1,98,9.1,1.4,1.9,-0.3
-1,111,7.8,2.0,1.8,4.1
-1,107,13.0,1.5,2.8,1.7
-1,119,11.4,2.3,2.2,1.6
-1,122,11.8,2.7,1.7,2.3
-1,105,8.1,2.0,1.9,-0.5
-1,109,7.6,1.3,2.2,1.9
-1,105,9.5,1.8,1.6,3.6
-1,112,5.9,1.7,2.0,1.3
-1,112,9.5,2.0,1.2,0.7
-1,98,8.6,1.6,1.6,6.0
-1,109,12.4,2.3,1.7,0.8
-1,114,9.1,2.6,1.5,1.5
-1,114,11.1,2.4,2.0,-0.3
-1,110,8.4,1.4,1.0,1.9
-1,120,7.1,1.2,1.5,4.3
-1,108,10.9,1.2,1.9,1.0
-1,108,8.7,1.2,2.2,2.5
-1,116,11.9,1.8,1.9,1.5
-1,113,11.5,1.5,1.9,2.9
-1,105,7.0,1.5,2.7,4.3
-1,114,8.4,1.6,1.6,-0.2
-1,114,8.1,1.6,1.6,0.5
-1,105,11.1,1.1,0.8,1.2
-1,107,13.8,1.5,1.0,1.9
-1,116,11.5,1.8,1.4,5.4
-1,102,9.5,1.4,1.1,1.6
-1,116,16.1,0.9,1.3,1.5
-1,118,10.6,1.8,1.4,3.0
-1,109,8.9,1.7,1.0,0.9
-1,110,7.0,1.0,1.6,4.3
-1,104,9.6,1.1,1.3,0.8
-1,105,8.7,1.5,1.1,1.5
-1,102,8.5,1.2,1.3,1.4
-1,112,6.8,1.7,1.4,3.3
-1,111,8.5,1.6,1.1,3.9
-1,111,8.5,1.6,1.2,7.7
-1,103,7.3,1.0,0.7,0.5
-1,98,10.4,1.6,2.3,-0.7
-1,117,7.8,2.0,1.0,3.9
-1,111,9.1,1.7,1.2,4.1
-1,101,6.3,1.5,0.9,2.9
-1,106,8.9,0.7,1.0,2.3
-1,102,8.4,1.5,0.8,2.4
-1,115,10.6,0.8,2.1,4.6
-1,130,10.0,1.6,0.9,4.6
-1,101,6.7,1.3,1.0,5.7
-1,110,6.3,1.0,0.8,1.0
-1,103,9.5,2.9,1.4,-0.1
-1,113,7.8,2.0,1.1,3.0
-1,112,10.6,1.6,0.9,-0.1
-1,118,6.5,1.2,1.2,1.7
-1,109,9.2,1.8,1.1,4.4
-1,116,7.8,1.4,1.1,3.7
-1,127,7.7,1.8,1.9,6.4
-1,108,6.5,1.0,0.9,1.5
-1,108,7.1,1.3,1.6,2.2
-1,105,5.7,1.0,0.9,0.9
-1,98,5.7,0.4,1.3,2.8
-1,112,6.5,1.2,1.2,2.0
-1,118,12.2,1.5,1.0,2.3
-1,94,7.5,1.2,1.3,4.4
-1,126,10.4,1.7,1.2,3.5
-1,114,7.5,1.1,1.6,4.4
-1,111,11.9,2.3,0.9,3.8
-1,104,6.1,1.8,0.5,0.8
-1,102,6.6,1.2,1.4,1.3
-2,139,16.4,3.8,1.1,-0.2
-2,111,16.0,2.1,0.9,-0.1
-2,113,17.2,1.8,1.0,0.0
-2,65,25.3,5.8,1.3,0.2
-2,88,24.1,5.5,0.8,0.1
-2,65,18.2,10.0,1.3,0.1
-2,134,16.4,4.8,0.6,0.1
-2,110,20.3,3.7,0.6,0.2
-2,67,23.3,7.4,1.8,-0.6
-2,95,11.1,2.7,1.6,-0.3
-2,89,14.3,4.1,0.5,0.2
-2,89,23.8,5.4,0.5,0.1
-2,88,12.9,2.7,0.1,0.2
-2,105,17.4,1.6,0.3,0.4
-2,89,20.1,7.3,1.1,-0.2
-2,99,13.0,3.6,0.7,-0.1
-2,80,23.0,10.0,0.9,-0.1
-2,89,21.8,7.1,0.7,-0.1
-2,99,13.0,3.1,0.5,-0.1
-2,68,14.7,7.8,0.6,-0.2
-2,97,14.2,3.6,1.5,0.3
-2,84,21.5,2.7,1.1,-0.6
-2,84,18.5,4.4,1.1,-0.3
-2,98,16.7,4.3,1.7,0.2
-2,94,20.5,1.8,1.4,-0.5
-2,99,17.5,1.9,1.4,0.3
-2,76,25.3,4.5,1.2,-0.1
-2,110,15.2,1.9,0.7,-0.2
-2,144,22.3,3.3,1.3,0.6
-2,105,12.0,3.3,1.1,0.0
-2,88,16.5,4.9,0.8,0.1
-2,97,15.1,1.8,1.2,-0.2
-2,106,13.4,3.0,1.1,0.0
-2,79,19.0,5.5,0.9,0.3
-2,92,11.1,2.0,0.7,-0.2
-3,125,2.3,0.9,16.5,9.5
-3,120,6.8,2.1,10.4,38.6
-3,108,3.5,0.6,1.7,1.4
-3,120,3.0,2.5,1.2,4.5
-3,119,3.8,1.1,23.0,5.7
-3,141,5.6,1.8,9.2,14.4
-3,129,1.5,0.6,12.5,2.9
-3,118,3.6,1.5,11.6,48.8
-3,120,1.9,0.7,18.5,24.0
-3,119,0.8,0.7,56.4,21.6
-3,123,5.6,1.1,13.7,56.3
-3,115,6.3,1.2,4.7,14.4
-3,126,0.5,0.2,12.2,8.8
-3,121,4.7,1.8,11.2,53.0
-3,131,2.7,0.8,9.9,4.7
-3,134,2.0,0.5,12.2,2.2
-3,141,2.5,1.3,8.5,7.5
-3,113,5.1,0.7,5.8,19.6
-3,136,1.4,0.3,32.6,8.4
-3,120,3.4,1.8,7.5,21.5
-3,125,3.7,1.1,8.5,25.9
-3,123,1.9,0.3,22.8,22.2
-3,112,2.6,0.7,41.0,19.0
-3,134,1.9,0.6,18.4,8.2
-3,119,5.1,1.1,7.0,40.8
-3,118,6.5,1.3,1.7,11.5
-3,139,4.2,0.7,4.3,6.3
-3,103,5.1,1.4,1.2,5.0
-3,97,4.7,1.1,2.1,12.6
-3,102,5.3,1.4,1.3,6.7
-

GPy/util/debug.py

@@ -0,0 +1,36 @@
+# Copyright (c) 2012, GPy authors (see AUTHORS.txt).
+# Licensed under the BSD 3-clause license (see LICENSE.txt)
+
+"""
+The module for some general debug tools
+"""
+
+import numpy as np
+
+def checkFinite(arr, name=None):
+    if name is None:
+        name = 'Array with ID['+str(id(arr))+']'
+        
+    if np.any(np.logical_not(np.isfinite(arr))):
+        idx = np.where(np.logical_not(np.isfinite(arr)))[0]
+        print name+' at indices '+str(idx)+' have not finite values: '+str(arr[idx])+'!'
+        return False
+    return True
+
+def checkFullRank(m, tol=1e-10, name=None, force_check=False):
+    if name is None:
+        name = 'Matrix with ID['+str(id(m))+']'
+    assert len(m.shape)==2 and m.shape[0]==m.shape[1], 'The input of checkFullRank has to be a square matrix!'
+    
+    if not force_check and m.shape[0]>=10000:
+        print 'The size of '+name+'is too big to check (>=10000)!'
+        return True
+    
+    s = np.linalg.eigvals(m)
+    
+    if s.min()/s.max()<tol:
+        print name+' is close to singlar!'
+        print 'The eigen values of '+name+' is '+str(s)
+        return False
+    return True
+    

GPy/util/gpu_init.py

@@ -5,12 +5,44 @@ Global variables: initSuccess
 providing CUBLAS handle: cublas_handle
 """
 
+gpu_initialized = False
+gpu_device = None
+gpu_context = None
+MPI_enabled = False
+
 try:
+    from mpi4py import MPI
+    MPI_enabled = True
+except:
+    pass
+
+try:
+    if MPI_enabled and MPI.COMM_WORLD.size>1:
+        from .parallel import get_id_within_node
+        gpuid = get_id_within_node()
+        import pycuda.driver
+        pycuda.driver.init()
+        if gpuid>=pycuda.driver.Device.count():
+            print '['+MPI.Get_processor_name()+'] more processes than the GPU numbers!'
+            #MPI.COMM_WORLD.Abort()
+            raise
+        gpu_device = pycuda.driver.Device(gpuid)
+        gpu_context = gpu_device.make_context()
+        gpu_initialized = True
+    else:
         import pycuda.autoinit
+        gpu_initialized = True
+except:
+    pass
+
+try:
     from scikits.cuda import cublas
     import scikits.cuda.linalg as culinalg
     culinalg.init()
     cublas_handle = cublas.cublasCreate()
-    initSuccess = True
 except:
-    initSuccess = False
+    pass
+
+def closeGPU():
+    if gpu_context is not None:
+        gpu_context.detach()

GPy/util/initialization.py

@@ -8,12 +8,16 @@ import numpy as np
 from GPy.util.pca import pca
 
 def initialize_latent(init, input_dim, Y):
-    Xr = np.random.randn(Y.shape[0], input_dim)
+    Xr = np.asfortranarray(np.random.randn(Y.shape[0], input_dim))
     if init == 'PCA':
         p = pca(Y)
         PC = p.project(Y, min(input_dim, Y.shape[1]))
         Xr[:PC.shape[0], :PC.shape[1]] = PC
+        var = p.fracs[:input_dim]
     else:
         var = Xr.var(0)
+
+    Xr -= Xr.mean(0)
+    Xr /= Xr.std(0)
+
     return Xr, var/var.max()
-    return Xr, p.fracs[:input_dim]