Merge branch 'devel' for extending prod kernel

GPy/core/mapping.py

@@ -13,11 +13,7 @@ class Mapping(Parameterized):
     def __init__(self, input_dim, output_dim, name='mapping'):
         self.input_dim = input_dim
         self.output_dim = output_dim
-
         super(Mapping, self).__init__(name=name)
-        # Model.__init__(self)
-        # All leaf nodes should call self._set_params(self._get_params()) at
-        # the end
 
     def f(self, X):
         raise NotImplementedError
@@ -56,7 +52,8 @@ class Mapping(Parameterized):
         Can plot only part of the data and part of the posterior functions
         using which_data and which_functions
 
-        This is a convenience function: arguments are passed to GPy.plotting.matplot_dep.models_plots.plot_mapping
+        This is a convenience function: arguments are passed to
+        GPy.plotting.matplot_dep.models_plots.plot_mapping
         """
 
         if "matplotlib" in sys.modules:
@@ -66,7 +63,10 @@ class Mapping(Parameterized):
             raise NameError, "matplotlib package has not been imported."
 
 class Bijective_mapping(Mapping):
-    """This is a mapping that is bijective, i.e. you can go from X to f and also back from f to X. The inverse mapping is called g()."""
+    """
+    This is a mapping that is bijective, i.e. you can go from X to f and
+    also back from f to X. The inverse mapping is called g().
+    """
     def __init__(self, input_dim, output_dim, name='bijective_mapping'):
         super(Bijective_apping, self).__init__(name=name)
 
@@ -77,7 +77,11 @@ class Bijective_mapping(Mapping):
 from model import Model
 
 class Mapping_check_model(Model):
-    """This is a dummy model class used as a base class for checking that the gradients of a given mapping are implemented correctly. It enables checkgradient() to be called independently on each mapping."""
+    """
+    This is a dummy model class used as a base class for checking that the
+    gradients of a given mapping are implemented correctly. It enables
+    checkgradient() to be called independently on each mapping.
+    """
     def __init__(self, mapping=None, dL_df=None, X=None):
         num_samples = 20
         if mapping==None:

GPy/core/svigp.py

@@ -42,10 +42,10 @@ class SVIGP(GP):
     """
 
 
-    def __init__(self, X, likelihood, kernel, Z, q_u=None, batchsize=10, X_variance=None):
+    def __init__(self, X, Y, kernel, Z, q_u=None, batchsize=10):
-        GP.__init__(self, X, likelihood, kernel, normalize_X=False)
+        raise NotImplementedError, "This is a work in progress, see github issue "
+        GP.__init__(self, X, Y, kernel)
         self.batchsize=batchsize
-        self.Y = self.likelihood.Y.copy()
         self.Z = Z
         self.num_inducing = Z.shape[0]
 
@@ -57,15 +57,9 @@ class SVIGP(GP):
         self.param_steplength = 1e-5
         self.momentum = 0.9
 
-        if X_variance is None:
-            self.has_uncertain_inputs = False
-        else:
-            self.has_uncertain_inputs = True
-            self.X_variance = X_variance
-
-
         if q_u is None:
              q_u = np.hstack((np.random.randn(self.num_inducing*self.output_dim),-.5*np.eye(self.num_inducing).flatten()))
+
         self.set_vb_param(q_u)
 
         self._permutation = np.random.permutation(self.num_data)
@@ -76,82 +70,25 @@ class SVIGP(GP):
         self._grad_trace = []
 
         #set the adaptive steplength parameters
-        self.hbar_t = 0.0
+        #self.hbar_t = 0.0
-        self.tau_t = 100.0
+        #self.tau_t = 100.0
-        self.gbar_t = 0.0
+        #self.gbar_t = 0.0
-        self.gbar_t1 = 0.0
+        #self.gbar_t1 = 0.0
-        self.gbar_t2 = 0.0
+        #self.gbar_t2 = 0.0
-        self.hbar_tp = 0.0
+        #self.hbar_tp = 0.0
-        self.tau_tp = 10000.0
+        #self.tau_tp = 10000.0
-        self.gbar_tp = 0.0
+        #self.gbar_tp = 0.0
-        self.adapt_param_steplength = True
+        #self.adapt_param_steplength = True
-        self.adapt_vb_steplength = True
+        #self.adapt_vb_steplength = True
-        self._param_steplength_trace = []
+        #self._param_steplength_trace = []
-        self._vb_steplength_trace = []
+        #self._vb_steplength_trace = []
-
-#     def _getstate(self):
-#         steplength_params = [self.hbar_t, self.tau_t, self.gbar_t, self.gbar_t1, self.gbar_t2, self.hbar_tp, self.tau_tp, self.gbar_tp, self.adapt_param_steplength, self.adapt_vb_steplength, self.vb_steplength, self.param_steplength]
-#         return GP._getstate(self) + \
-#             [self.get_vb_param(),
-#              self.Z,
-#              self.num_inducing,
-#              self.has_uncertain_inputs,
-#              self.X_variance,
-#              self.X_batch,
-#              self.X_variance_batch,
-#              steplength_params,
-#              self.batchcounter,
-#              self.batchsize,
-#              self.epochs,
-#              self.momentum,
-#              self.data_prop,
-#              self._param_trace,
-#              self._param_steplength_trace,
-#              self._vb_steplength_trace,
-#              self._ll_trace,
-#              self._grad_trace,
-#              self.Y,
-#              self._permutation,
-#              self.iterations
-#             ]
-# 
-#     def _setstate(self, state):
-#         self.iterations = state.pop()
-#         self._permutation = state.pop()
-#         self.Y = state.pop()
-#         self._grad_trace = state.pop()
-#         self._ll_trace = state.pop()
-#         self._vb_steplength_trace = state.pop()
-#         self._param_steplength_trace = state.pop()
-#         self._param_trace = state.pop()
-#         self.data_prop = state.pop()
-#         self.momentum = state.pop()
-#         self.epochs = state.pop()
-#         self.batchsize = state.pop()
-#         self.batchcounter = state.pop()
-#         steplength_params = state.pop()
-#         (self.hbar_t, self.tau_t, self.gbar_t, self.gbar_t1, self.gbar_t2, self.hbar_tp, self.tau_tp, self.gbar_tp, self.adapt_param_steplength, self.adapt_vb_steplength, self.vb_steplength, self.param_steplength) = steplength_params
-#         self.X_variance_batch = state.pop()
-#         self.X_batch = state.pop()
-#         self.X_variance = state.pop()
-#         self.has_uncertain_inputs = state.pop()
-#         self.num_inducing = state.pop()
-#         self.Z = state.pop()
-#         vb_param = state.pop()
-#         GP._setstate(self, state)
-#         self.set_vb_param(vb_param)
 
     def _compute_kernel_matrices(self):
         # kernel computations, using BGPLVM notation
         self.Kmm = self.kern.K(self.Z)
-        if self.has_uncertain_inputs:
+        self.psi0 = self.kern.Kdiag(self.X_batch)
-            self.psi0 = self.kern.psi0(self.Z, self.X_batch, self.X_variance_batch)
+        self.psi1 = self.kern.K(self.X_batch, self.Z)
-            self.psi1 = self.kern.psi1(self.Z, self.X_batch, self.X_variance_batch)
+        self.psi2 = None
-            self.psi2 = self.kern.psi2(self.Z, self.X_batch, self.X_variance_batch)
-        else:
-            self.psi0 = self.kern.Kdiag(self.X_batch)
-            self.psi1 = self.kern.K(self.X_batch, self.Z)
-            self.psi2 = None
 
     def dL_dtheta(self):
         dL_dtheta = self.kern._param_grad_helper(self.dL_dKmm, self.Z)
@@ -179,7 +116,7 @@ class SVIGP(GP):
 
     def load_batch(self):
         """
-        load a batch of data (set self.X_batch and self.likelihood.Y from self.X, self.Y)
+        load a batch of data (set self.X_batch and self.Y_batch from self.X, self.Y)
         """
 
         #if we've seen all the data, start again with them in a new random order
@@ -191,9 +128,7 @@ class SVIGP(GP):
         this_perm = self._permutation[self.batchcounter:self.batchcounter+self.batchsize]
 
         self.X_batch = self.X[this_perm]
-        self.likelihood.set_data(self.Y[this_perm])
+        self.Y_batch = self.Y[this_perm]
-        if self.has_uncertain_inputs:
-            self.X_variance_batch = self.X_variance[this_perm]
 
         self.batchcounter += self.batchsize
 
@@ -288,7 +223,9 @@ class SVIGP(GP):
         Compute the gradients of the lower bound wrt the canonical and
         Expectation parameters of u.
 
-        Note that the natural gradient in either is given by the gradient in the other (See Hensman et al 2012 Fast Variational inference in the conjugate exponential Family)
+        Note that the natural gradient in either is given by the gradient in
+        the other (See Hensman et al 2012 Fast Variational inference in the
+        conjugate exponential Family)
         """
 
         # Gradient for eta

GPy/defaults.cfg

@@ -19,4 +19,9 @@ dir=$HOME/tmp/GPy-datasets/
 # if you have an anaconda python installation please specify it here.
 installed = False
 location = None
-MKL = False # set this to true if you have the MKL optimizations installed
+ # set this to true if you have the MKL optimizations installed:
+MKL = False
+
+[weave]
+#if true, try to use weave, and fall back to numpy. if false, just use numpy.
+working = True

GPy/likelihoods/exponential.py

@@ -5,7 +5,6 @@
 import numpy as np
 from scipy import stats,special
 import scipy as sp
-from GPy.util.univariate_Gaussian import std_norm_pdf,std_norm_cdf
 import link_functions
 from likelihood import Likelihood
 

GPy/likelihoods/gamma.py

@@ -5,7 +5,6 @@
 import numpy as np
 from scipy import stats,special
 import scipy as sp
-from ..util.univariate_Gaussian import std_norm_pdf,std_norm_cdf
 from ..core.parameterization import Param
 import link_functions
 from likelihood import Likelihood

GPy/likelihoods/gaussian.py

@@ -13,7 +13,6 @@ James 11/12/13
 
 import numpy as np
 from scipy import stats, special
-from GPy.util.univariate_Gaussian import std_norm_pdf, std_norm_cdf
 import link_functions
 from likelihood import Likelihood
 from ..core.parameterization import Param

GPy/likelihoods/likelihood.py

@@ -4,7 +4,6 @@
 import numpy as np
 from scipy import stats,special
 import scipy as sp
-from ..util.univariate_Gaussian import std_norm_pdf,std_norm_cdf
 import link_functions
 from ..util.misc import chain_1, chain_2, chain_3
 from scipy.integrate import quad

GPy/likelihoods/mixed_noise.py

@@ -1,6 +1,5 @@
 import numpy as np
 from scipy import stats, special
-from GPy.util.univariate_Gaussian import std_norm_pdf, std_norm_cdf
 import link_functions
 from likelihood import Likelihood
 from gaussian import Gaussian

GPy/likelihoods/negative_binomial.py

@@ -11,7 +11,6 @@ except ImportError:
     sympy_available=False
 
 import numpy as np
-from ..util.univariate_Gaussian import std_norm_pdf, std_norm_cdf
 import link_functions
 from symbolic import Symbolic
 from scipy import stats

GPy/likelihoods/ordinal.py

@@ -5,7 +5,6 @@
 import sympy as sym
 from GPy.util.symbolic import gammaln, normcdfln, normcdf, IndMatrix, create_matrix
 import numpy as np
-from ..util.univariate_Gaussian import std_norm_pdf, std_norm_cdf
 import link_functions
 from symbolic import Symbolic
 from scipy import stats

GPy/likelihoods/skew_normal.py

@@ -11,7 +11,6 @@ except ImportError:
     sympy_available=False
 
 import numpy as np
-from ..util.univariate_Gaussian import std_norm_pdf, std_norm_cdf
 from GPy.util.functions import clip_exp
 import link_functions
 from symbolic import Symbolic

GPy/util/linalg.py

@@ -10,11 +10,10 @@ from scipy import linalg, weave
 import types
 import ctypes
 from ctypes import byref, c_char, c_int, c_double # TODO
-# import scipy.lib.lapack
 import scipy
 import warnings
 import os
-from config import *
+from config import config
 import logging
 
 _scipyversion = np.float64((scipy.__version__).split('.')[:2])
@@ -521,6 +520,27 @@ def symmetrify(A, upper=False):
     Take the square matrix A and make it symmetrical by copting elements from the lower half to the upper
 
     works IN PLACE.
+
+    note: tries to use weave, falls back to a slower numpy version
+    """
+    if config.getboolean('weave', 'working'):
+        try:
+            symmetrify_weave(A, upper)
+        except:
+            print "\n Weave compilation failed. Falling back to (slower) numpy implementation\n"
+            config.set('weave', 'working', False)
+            symmetrify_numpy(A, upper)
+    else:
+        symmetrify_numpy(A, upper)
+
+
+def symmetrify_weave(A, upper=False):
+    """
+    Take the square matrix A and make it symmetrical by copting elements from the lower half to the upper
+
+    works IN PLACE.
+
+
     """
     N, M = A.shape
     assert N == M
@@ -563,10 +583,15 @@ def symmetrify(A, upper=False):
         A += np.tril(tmp, -1).T
 
 
-def symmetrify_murray(A):
+def symmetrify_numpy(A, upper=False):
-    A += A.T
+    """
-    nn = A.shape[0]
+    Force a matrix to be symmetric
-    A[[range(nn), range(nn)]] /= 2.0
+    """
+    triu = np.triu_indices_from(A,k=1)
+    if upper:
+        A.T[triu] = A[triu]
+    else:
+        A[triu] = A.T[triu]
 
 def cholupdate(L, x):
     """

GPy/util/misc.py

@@ -2,7 +2,6 @@
 # Licensed under the BSD 3-clause license (see LICENSE.txt)
 
 import numpy as np
-from scipy import weave
 from config import *
 
 def chain_1(df_dg, dg_dx):
@@ -84,96 +83,6 @@ def kmm_init(X, m = 10):
     inducing = np.array(inducing)
     return X[inducing]
 
-def fast_array_equal(A, B):
-
-
-    if config.getboolean('parallel', 'openmp'):
-        pragma_string = '#pragma omp parallel for private(i, j)'
-    else:
-        pragma_string = ''
-
-    code2="""
-    int i, j;
-    return_val = 1;
-
-    %s
-    for(i=0;i<N;i++){
-       for(j=0;j<D;j++){
-          if(A(i, j) != B(i, j)){
-              return_val = 0;
-              break;
-          }
-       }
-    }
-    """ % pragma_string
-
-    if config.getboolean('parallel', 'openmp'):
-        pragma_string = '#pragma omp parallel for private(i, j, z)'
-    else:
-        pragma_string = ''
-
-    code3="""
-    int i, j, z;
-    return_val = 1;
-
-    %s
-    for(i=0;i<N;i++){
-       for(j=0;j<D;j++){
-         for(z=0;z<Q;z++){
-            if(A(i, j, z) != B(i, j, z)){
-               return_val = 0;
-               break;
-            }
-          }
-       }
-    }
-    """ % pragma_string
-
-    if config.getboolean('parallel', 'openmp'):
-        header_string = '#include <omp.h>'
-    else:
-        header_string = ''
-
-    support_code = """
-    %s
-    #include <math.h>
-    """ % header_string
-
-
-    weave_options_openmp = {'headers'           : ['<omp.h>'],
-                            'extra_compile_args': ['-fopenmp -O3'],
-                            'extra_link_args'   : ['-lgomp'],
-                            'libraries': ['gomp']}
-    weave_options_noopenmp = {'extra_compile_args': ['-O3']}
-
-    if config.getboolean('parallel', 'openmp'):
-        weave_options = weave_options_openmp
-    else:
-        weave_options = weave_options_noopenmp
-
-    value = False
-
-
-    if (A == None) and (B == None):
-        return True
-    elif ((A == None) and (B != None)) or ((A != None) and (B == None)):
-        return False
-    elif A.shape == B.shape:
-        if A.ndim == 2:
-            N, D = [int(i) for i in A.shape]
-            value = weave.inline(code2, support_code=support_code,
-                                 arg_names=['A', 'B', 'N', 'D'],
-                                 type_converters=weave.converters.blitz, **weave_options)
-        elif A.ndim == 3:
-            N, D, Q = [int(i) for i in A.shape]
-            value = weave.inline(code3, support_code=support_code,
-                                 arg_names=['A', 'B', 'N', 'D', 'Q'],
-                                 type_converters=weave.converters.blitz, **weave_options)
-        else:
-            value = np.array_equal(A,B)
-
-    return value
-
 ### make a parameter to its corresponding array:
 def param_to_array(*param):
     """

GPy/util/univariate_Gaussian.py

@@ -12,6 +12,39 @@ def std_norm_cdf(x):
     """
     Cumulative standard Gaussian distribution
     Based on Abramowitz, M. and Stegun, I. (1970)
+    """
+    x_shape = np.asarray(x).shape
+
+    if len(x_shape) == 0 or x_shape[0] == 1:
+        sign = np.sign(x)
+        x *= sign
+        x /= np.sqrt(2.)
+        t = 1.0/(1.0 +  0.3275911*x)
+        erf = 1. - np.exp(-x**2)*t*(0.254829592 + t*(-0.284496736 + t*(1.421413741 + t*(-1.453152027 + t*(1.061405429)))))
+        cdf_x = 0.5*(1.0 + sign*erf)
+        return cdf_x
+    else:
+        x = np.atleast_1d(x).copy()
+        cdf_x = np.zeros_like(x)
+        sign = np.ones_like(x)
+        neg_x_ind = x<0
+        sign[neg_x_ind] = -1.0
+        x[neg_x_ind] = -x[neg_x_ind]
+        x /= np.sqrt(2.)
+        t = 1.0/(1.0 +  0.3275911*x)
+        erf = 1. - np.exp(-x**2)*t*(0.254829592 + t*(-0.284496736 + t*(1.421413741 + t*(-1.453152027 + t*(1.061405429)))))
+        cdf_x = 0.5*(1.0 + sign*erf)
+        cdf_x = cdf_x.reshape(x_shape)
+    return cdf_x
+
+def std_norm_cdf_weave(x):
+    """
+    Cumulative standard Gaussian distribution
+    Based on Abramowitz, M. and Stegun, I. (1970)
+
+    A weave implementation of std_norm_cdf, which is faster. this is unused,
+    because of the difficulties of a weave dependency. (see github issue #94)
+
     """
     #Generalize for many x
     x = np.asarray(x).copy()
@@ -40,37 +73,6 @@ def std_norm_cdf(x):
     weave.inline(code, arg_names=['x', 'cdf_x', 'N'], support_code=support_code)
     return cdf_x
 
-def std_norm_cdf_np(x):
-    """
-    Cumulative standard Gaussian distribution
-    Based on Abramowitz, M. and Stegun, I. (1970)
-    Around 3 times slower when x is a scalar otherwise quite a lot slower
-    """
-    x_shape = np.asarray(x).shape
-
-    if len(x_shape) == 0 or x_shape[0] == 1:
-        sign = np.sign(x)
-        x *= sign
-        x /= np.sqrt(2.)
-        t = 1.0/(1.0 +  0.3275911*x)
-        erf = 1. - np.exp(-x**2)*t*(0.254829592 + t*(-0.284496736 + t*(1.421413741 + t*(-1.453152027 + t*(1.061405429)))))
-        cdf_x = 0.5*(1.0 + sign*erf)
-        return cdf_x
-    else:
-        x = np.atleast_1d(x).copy()
-        cdf_x = np.zeros_like(x)
-        sign = np.ones_like(x)
-        neg_x_ind = x<0
-        sign[neg_x_ind] = -1.0
-        x[neg_x_ind] = -x[neg_x_ind]
-        x /= np.sqrt(2.)
-        t = 1.0/(1.0 +  0.3275911*x)
-        erf = 1. - np.exp(-x**2)*t*(0.254829592 + t*(-0.284496736 + t*(1.421413741 + t*(-1.453152027 + t*(1.061405429)))))
-        cdf_x = 0.5*(1.0 + sign*erf)
-        cdf_x = cdf_x.reshape(x_shape)
-    return cdf_x
-
-
 def inv_std_norm_cdf(x):
     """
     Inverse cumulative standard Gaussian distribution