import csv
import os
import copy
import numpy as np
import GPy
import scipy.io
import cPickle as pickle
import zipfile
import tarfile
import datetime
import json
import re

from config import *

ipython_available=True
try:
    import IPython
except ImportError:
    ipython_available=False

import sys, urllib2

def reporthook(a,b,c):
    # ',' at the end of the line is important!
    #print "% 3.1f%% of %d bytes\r" % (min(100, float(a * b) / c * 100), c),
    #you can also use sys.stdout.write
    sys.stdout.write("\r% 3.1f%% of %d bytes" % (min(100, float(a * b) / c * 100), c))
    sys.stdout.flush()

# Global variables
data_path = os.path.expandvars(config.get('datasets', 'dir'))
#data_path = os.path.join(os.path.dirname(__file__), 'datasets')
default_seed = 10000
overide_manual_authorize=False
neil_url = 'http://staffwww.dcs.shef.ac.uk/people/N.Lawrence/dataset_mirror/'

# Read data resources from json file.
# Don't do this when ReadTheDocs is scanning as it breaks things
on_rtd = os.environ.get('READTHEDOCS', None) == 'True' #Checks if RTD is scanning

if not (on_rtd):
    path = os.path.join(os.path.dirname(__file__), 'data_resources.json')
    json_data=open(path).read()
    data_resources = json.loads(json_data)

    path = os.path.join(os.path.dirname(__file__), 'football_teams.json')
    football_dict = json.loads(json_data)

def prompt_user(prompt):
    """Ask user for agreeing to data set licenses."""
    # raw_input returns the empty string for "enter"
    yes = set(['yes', 'y'])
    no = set(['no','n'])

    try:
        print(prompt)
        choice = raw_input().lower()
        # would like to test for exception here, but not sure if we can do that without importing IPython
    except:
        print('Stdin is not implemented.')
        print('You need to set')
        print('overide_manual_authorize=True')
        print('to proceed with the download. Please set that variable and continue.')
        raise

    if choice in yes:
        return True
    elif choice in no:
        return False
    else:
        print("Your response was a " + choice)
        print("Please respond with 'yes', 'y' or 'no', 'n'")
        #return prompt_user()

def data_available(dataset_name=None):
    """Check if the data set is available on the local machine already."""
    from itertools import izip_longest
    dr = data_resources[dataset_name]
    zip_urls = (dr['files'], )
    if dr.has_key('save_names'): zip_urls += (dr['save_names'], )
    else: zip_urls += ([],)

    for file_list, save_list in izip_longest(*zip_urls, fillvalue=[]):
        for f, s in izip_longest(file_list, save_list, fillvalue=None):
            if s is not None: f=s # If there is a save_name given, use that one
            if not os.path.exists(os.path.join(data_path, dataset_name, f)):
                return False
    return True

def download_url(url, store_directory, save_name=None, messages=True, suffix=''):
    """Download a file from a url and save it to disk."""
    i = url.rfind('/')
    file = url[i+1:]
    print file
    dir_name = os.path.join(data_path, store_directory)

    if save_name is None: save_name = os.path.join(dir_name, file)
    else: save_name = os.path.join(dir_name, save_name)

    if suffix is None: suffix=''

    print "Downloading ", url, "->", save_name
    if not os.path.exists(dir_name):
        os.makedirs(dir_name)
    try:
        response = urllib2.urlopen(url+suffix)
    except urllib2.URLError, e:
        if not hasattr(e, "code"):
            raise
        response = e
        if response.code > 399 and response.code<500:
            raise ValueError('Tried url ' + url + suffix + ' and received client error ' + str(response.code))
        elif response.code > 499:
            raise ValueError('Tried url ' + url + suffix + ' and received server error ' + str(response.code))
    with open(save_name, 'wb') as f:
        meta = response.info()
        content_length_str = meta.getheaders("Content-Length")
        if content_length_str:
            file_size = int(content_length_str[0])
        else:
            file_size = None
        status = ""
        file_size_dl = 0
        block_sz = 8192
        line_length=30
        while True:
            buff = response.read(block_sz)
            if not buff:
                break
            file_size_dl += len(buff)
            f.write(buff)
            sys.stdout.write(" "*(len(status)) + "\r")
            if file_size:
                status = r"[{perc: <{ll}}] {dl:7.3f}/{full:.3f}MB".format(dl=file_size_dl/(1048576.),
                                                                       full=file_size/(1048576.), ll=line_length,
                                                                       perc="="*int(line_length*float(file_size_dl)/file_size))
            else:
                status = r"[{perc: <{ll}}] {dl:7.3f}MB".format(dl=file_size_dl/(1048576.),
                                                                       ll=line_length,
                                                                       perc="."*int(line_length*float(file_size_dl/(10*1048576.))))

            sys.stdout.write(status)
            sys.stdout.flush()
        sys.stdout.write(" "*(len(status)) + "\r")
        print status
    # if we wanted to get more sophisticated maybe we should check the response code here again even for successes.
    #with open(save_name, 'wb') as f:
    #    f.write(response.read())

    #urllib.urlretrieve(url+suffix, save_name, reporthook)

def authorize_download(dataset_name=None):
    """Check with the user that the are happy with terms and conditions for the data set."""
    print('Acquiring resource: ' + dataset_name)
    # TODO, check resource is in dictionary!
    print('')
    print('Details of data: ')
    print(dr['details'])
    print('')
    if dr['citation']:
        print('Please cite:')
        print(dr['citation'])
        print('')
    if dr['size']:
        print('After downloading the data will take up ' + str(dr['size']) + ' bytes of space.')
        print('')
    print('Data will be stored in ' + os.path.join(data_path, dataset_name) + '.')
    print('')
    if overide_manual_authorize:
        if dr['license']:
            print('You have agreed to the following license:')
            print(dr['license'])
            print('')
    else:
            print('You must also agree to the following license:')
            print('')
        return prompt_user('Do you wish to proceed with the download? [yes/no]')

def download_data(dataset_name=None):
    """Check with the user that the are happy with terms and conditions for the data set, then download it."""
    import itertools

    if not authorize_download(dataset_name):
        raise Exception("Permission to download data set denied.")

    zip_urls = (dr['urls'], dr['files'])

    if dr.has_key('suffices'): zip_urls += (dr['suffices'], )

    for url, files, save_names, suffices in itertools.izip_longest(*zip_urls, fillvalue=[]):
        for f, save_name, suffix in itertools.izip_longest(files, save_names, suffices, fillvalue=None):
            download_url(os.path.join(url,f), dataset_name, save_name, suffix=suffix)

def data_details_return(data, data_set):
    """Update the data component of the data dictionary with details drawn from the data_resources."""
    data.update(data_resources[data_set])
    return data

def cmu_urls_files(subj_motions, messages = True):
    '''
    Find which resources are missing on the local disk for the requested CMU motion capture motions.
    '''
    dr = data_resources['cmu_mocap_full']
    cmu_url = dr['urls'][0]

    subjects_num = subj_motions[0]
    motions_num = subj_motions[1]

    resource = {'urls' : [], 'files' : []}
    # Convert numbers to strings
    subjects = []
    motions = [list() for _ in range(len(subjects_num))]
    for i in range(len(subjects_num)):
        curSubj = str(int(subjects_num[i]))
        if int(subjects_num[i]) < 10:
            curSubj = '0' + curSubj
        subjects.append(curSubj)
        for j in range(len(motions_num[i])):
            curMot = str(int(motions_num[i][j]))
            if int(motions_num[i][j]) < 10:
                curMot = '0' + curMot
            motions[i].append(curMot)

    all_skels = []

    assert len(subjects) == len(motions)

    all_motions = []

    for i in range(len(subjects)):
        skel_dir = os.path.join(data_path, 'cmu_mocap')
        cur_skel_file = os.path.join(skel_dir, subjects[i] + '.asf')

        url_required = False
        file_download = []
        if not os.path.exists(cur_skel_file):
            # Current skel file doesn't exist.
            if not os.path.isdir(skel_dir):
                os.makedirs(skel_dir)
            # Add skel file to list.
            url_required = True
            file_download.append(subjects[i] + '.asf')
        for j in range(len(motions[i])):
            file_name = subjects[i] + '_' + motions[i][j] + '.amc'
            cur_motion_file = os.path.join(skel_dir, file_name)
            if not os.path.exists(cur_motion_file):
                url_required = True
                file_download.append(subjects[i] + '_' + motions[i][j] + '.amc')
        if url_required:
            resource['urls'].append(cmu_url + '/' + subjects[i] + '/')
            resource['files'].append(file_download)
    return resource

try:
    import gpxpy
    import gpxpy.gpx
    gpxpy_available = True

    gpxpy_available = False

if gpxpy_available:
    def epomeo_gpx(data_set='epomeo_gpx', sample_every=4):
        if not data_available(data_set):
            download_data(data_set)
        files = ['endomondo_1', 'endomondo_2', 'garmin_watch_via_endomondo','viewranger_phone', 'viewranger_tablet']

        X = []
        for file in files:
            gpx_file = open(os.path.join(data_path, 'epomeo_gpx', file + '.gpx'), 'r')

            gpx = gpxpy.parse(gpx_file)
            segment = gpx.tracks[0].segments[0]
            points = [point for track in gpx.tracks for segment in track.segments for point in segment.points]
            data = [[(point.time-datetime.datetime(2013,8,21)).total_seconds(), point.latitude, point.longitude, point.elevation] for point in points]
            X.append(np.asarray(data)[::sample_every, :])
            gpx_file.close()
        return data_details_return({'X' : X, 'info' : 'Data is an array containing time in seconds, latitude, longitude and elevation in that order.'}, data_set)

#del gpxpy_available

# Some general utilities.
def sample_class(f):
    p = 1. / (1. + np.exp(-f))
    c = np.random.binomial(1, p)
    c = np.where(c, 1, -1)
    return c

def boston_housing(data_set='boston_housing'):
    if not data_available(data_set):
        download_data(data_set)
    all_data = np.genfromtxt(os.path.join(data_path, data_set, 'housing.data'))
    X = all_data[:, 0:13]
    Y = all_data[:, 13:14]
    return data_details_return({'X' : X, 'Y': Y}, data_set)

def brendan_faces(data_set='brendan_faces'):
    mat_data = scipy.io.loadmat(os.path.join(data_path, data_set, 'frey_rawface.mat'))
    Y = mat_data['ff'].T
    return data_details_return({'Y': Y}, data_set)

def della_gatta_TRP63_gene_expression(data_set='della_gatta', gene_number=None):
    mat_data = scipy.io.loadmat(os.path.join(data_path, data_set, 'DellaGattadata.mat'))
    X = np.double(mat_data['timepoints'])
    if gene_number == None:
        Y = mat_data['exprs_tp53_RMA']
    else:
        Y = mat_data['exprs_tp53_RMA'][:, gene_number]
        if len(Y.shape) == 1:
            Y = Y[:, None]
    return data_details_return({'X': X, 'Y': Y, 'gene_number' : gene_number}, data_set)

def football_data(season='1314', data_set='football_data'):
    """Football data from English games since 1993. This downloads data from football-data.co.uk for the given season. """
    def league2num(string):
        league_dict = {'E0':0, 'E1':1, 'E2': 2, 'E3': 3, 'EC':4}
        return league_dict[string]

    def football2num(string):
        if football_dict.has_key(string):
            return football_dict[string]
        else:
            football_dict[string] = len(football_dict)+1
            return len(football_dict)+1

    data_set_season = data_set + '_' + season
    data_resources[data_set_season] = copy.deepcopy(data_resources[data_set])
    data_resources[data_set_season]['urls'][0]+=season + '/'
    start_year = int(season[0:2])
    end_year = int(season[2:4])
    files = ['E0.csv', 'E1.csv', 'E2.csv', 'E3.csv']
    if start_year>4 and start_year < 93:
        files += ['EC.csv']
    data_resources[data_set_season]['files'] = [files]
    if not data_available(data_set_season):
        download_data(data_set_season)
    import pylab as pb
    for file in reversed(files):
        filename = os.path.join(data_path, data_set_season, file)
        # rewrite files removing blank rows.
        writename = os.path.join(data_path, data_set_season, 'temp.csv')
        input = open(filename, 'rb')
        output = open(writename, 'wb')
        writer = csv.writer(output)
        for row in csv.reader(input):
            if any(field.strip() for field in row):
                writer.writerow(row)
        input.close()
        output.close()
        table = np.loadtxt(writename,skiprows=1, usecols=(0, 1, 2, 3, 4, 5), converters = {0: league2num, 1: pb.datestr2num, 2:football2num, 3:football2num}, delimiter=',')
        X = table[:, :4]
        Y = table[:, 4:]
    return data_details_return({'X': X, 'Y': Y}, data_set)

def sod1_mouse(data_set='sod1_mouse'):
    from pandas import read_csv
    dir_path = os.path.join(data_path, data_set)
    filename = os.path.join(dir_path, 'sod1_C57_129_exprs.csv')
    Y = read_csv(filename, header=0, index_col=0)
    num_repeats=4
    num_time=4
    num_cond=4
    X = 1

def spellman_yeast(data_set='spellman_yeast'):
    filename = os.path.join(dir_path, 'combined.txt')
    Y = read_csv(filename, header=0, index_col=0, sep='\t')

def spellman_yeast_cdc15(data_set='spellman_yeast'):
    t = np.asarray([10, 30, 50, 70, 80, 90, 100, 110, 120, 130, 140, 150, 170, 180, 190, 200, 210, 220, 230, 240, 250, 270, 290])
    times = ['cdc15_'+str(time) for time in t]
    Y = Y[times].T
    t = t[:, None]
    return data_details_return({'Y' : Y, 't': t, 'info': 'Time series of synchronized yeast cells from the CDC-15 experiment of Spellman et al (1998).'}, data_set)

def lee_yeast_ChIP(data_set='lee_yeast_ChIP'):
    import zipfile
    filename = os.path.join(dir_path, 'binding_by_gene.tsv')
    S = read_csv(filename, header=1, index_col=0, sep='\t')
    transcription_factors = [col for col in S.columns if col[:7] != 'Unnamed']
    annotations = S[['Unnamed: 1', 'Unnamed: 2', 'Unnamed: 3']]
    S = S[transcription_factors]
    return data_details_return({'annotations' : annotations, 'Y' : S, 'transcription_factors': transcription_factors}, data_set)

def fruitfly_tomancak(data_set='fruitfly_tomancak', gene_number=None):
    filename = os.path.join(dir_path, 'tomancak_exprs.csv')
    Y = read_csv(filename, header=0, index_col=0).T
    num_repeats = 3
    num_time = 12
    xt = np.linspace(0, num_time-1, num_time)
    xr = np.linspace(0, num_repeats-1, num_repeats)
    xtime, xrepeat = np.meshgrid(xt, xr)
    X = np.vstack((xtime.flatten(), xrepeat.flatten())).T

def drosophila_protein(data_set='drosophila_protein'):
    filename = os.path.join(dir_path, 'becker_et_al.csv')
    Y = read_csv(filename, header=0)

def drosophila_knirps(data_set='drosophila_protein'):
    # in the csv file we have facts_kni and ext_kni. We treat facts_kni as protein and ext_kni as mRNA
    df = read_csv(filename, header=0)
    t = df['t'][:,None]
    x = df['x'][:,None]

    g = df['expression1'][:,None]
    p = df['expression2'][:,None]

    leng = x.shape[0]

    T = np.vstack([t,t])
    S = np.vstack([x,x])
    inx = np.zeros(leng*2)[:,None]

    inx[leng*2/2:leng*2]=1
    X = np.hstack([T,S,inx])
    Y = np.vstack([g,p])
    return data_details_return({'Y': Y, 'X': X}, data_set)

# This will be for downloading google trends data.
def google_trends(query_terms=['big data', 'machine learning', 'data science'], data_set='google_trends', refresh_data=False):
    """Data downloaded from Google trends for given query terms. Warning, if you use this function multiple times in a row you get blocked due to terms of service violations. The function will cache the result of your query, if you wish to refresh an old query set refresh_data to True. The function is inspired by this notebook: http://nbviewer.ipython.org/github/sahuguet/notebooks/blob/master/GoogleTrends%20meet%20Notebook.ipynb"""
    query_terms.sort()
    import pandas

    # Create directory name for data
    dir_path = os.path.join(data_path,'google_trends')
    if not os.path.isdir(dir_path):
        os.makedirs(dir_path)
    dir_name = '-'.join(query_terms)
    dir_name = dir_name.replace(' ', '_')
    dir_path = os.path.join(dir_path,dir_name)
    file = 'data.csv'
    file_name = os.path.join(dir_path,file)
    if not os.path.exists(file_name) or refresh_data:
        print "Accessing Google trends to acquire the data. Note that repeated accesses will result in a block due to a google terms of service violation. Failure at this point may be due to such blocks."
        # quote the query terms.
        quoted_terms = []
        for term in query_terms:
            quoted_terms.append(urllib2.quote(term))
        print "Query terms: ", ', '.join(query_terms)

        print "Fetching query:"
        query = 'http://www.google.com/trends/fetchComponent?q=%s&cid=TIMESERIES_GRAPH_0&export=3' % ",".join(quoted_terms)

        data = urllib2.urlopen(query).read()
        print "Done."
        # In the notebook they did some data cleaning: remove Javascript header+footer, and translate new Date(....,..,..) into YYYY-MM-DD.
        header = """// Data table response\ngoogle.visualization.Query.setResponse("""
        data = data[len(header):-2]
        data = re.sub('new Date\((\d+),(\d+),(\d+)\)', (lambda m: '"%s-%02d-%02d"' % (m.group(1).strip(), 1+int(m.group(2)), int(m.group(3)))), data)
        timeseries = json.loads(data)
        columns = [k['label'] for k in timeseries['table']['cols']]
        rows = map(lambda x: [k['v'] for k in x['c']], timeseries['table']['rows'])
        df = pandas.DataFrame(rows, columns=columns)
        if not os.path.isdir(dir_path):
            os.makedirs(dir_path)

        df.to_csv(file_name)
    else:
        print "Reading cached data for google trends. To refresh the cache set 'refresh_data=True' when calling this function."

        df = pandas.read_csv(file_name, parse_dates=[0])

    columns = df.columns
    terms = len(query_terms)
    import datetime
    X = np.asarray([(row, i) for i in range(terms) for row in df.index])
    Y = np.asarray([[df.ix[row][query_terms[i]]] for i in range(terms) for row in df.index ])
    output_info = columns[1:]

    return data_details_return({'data frame' : df, 'X': X, 'Y': Y, 'query_terms': output_info, 'info': "Data downloaded from google trends with query terms: " + ', '.join(output_info) + '.'}, data_set)

# The data sets
def oil(data_set='three_phase_oil_flow'):
    """The three phase oil data from Bishop and James (1993)."""
    oil_train_file = os.path.join(data_path, data_set, 'DataTrn.txt')
    oil_trainlbls_file = os.path.join(data_path, data_set, 'DataTrnLbls.txt')
    oil_test_file = os.path.join(data_path, data_set, 'DataTst.txt')
    oil_testlbls_file = os.path.join(data_path, data_set, 'DataTstLbls.txt')
    oil_valid_file = os.path.join(data_path, data_set, 'DataVdn.txt')
    oil_validlbls_file = os.path.join(data_path, data_set, 'DataVdnLbls.txt')
    fid = open(oil_train_file)
    X = np.fromfile(fid, sep='\t').reshape((-1, 12))
    fid.close()
    fid = open(oil_test_file)
    Xtest = np.fromfile(fid, sep='\t').reshape((-1, 12))
    fid = open(oil_valid_file)
    Xvalid = np.fromfile(fid, sep='\t').reshape((-1, 12))
    fid = open(oil_trainlbls_file)
    Y = np.fromfile(fid, sep='\t').reshape((-1, 3)) * 2. - 1.
    fid = open(oil_testlbls_file)
    Ytest = np.fromfile(fid, sep='\t').reshape((-1, 3)) * 2. - 1.
    fid = open(oil_validlbls_file)
    Yvalid = np.fromfile(fid, sep='\t').reshape((-1, 3)) * 2. - 1.
    return data_details_return({'X': X, 'Y': Y, 'Xtest': Xtest, 'Ytest': Ytest, 'Xtest' : Xtest, 'Xvalid': Xvalid, 'Yvalid': Yvalid}, data_set)
    #else:
    # throw an error

def oil_100(seed=default_seed, data_set = 'three_phase_oil_flow'):
    np.random.seed(seed=seed)
    data = oil()
    indices = np.random.permutation(1000)
    indices = indices[0:100]
    X = data['X'][indices, :]
    Y = data['Y'][indices, :]
    return data_details_return({'X': X, 'Y': Y, 'info': "Subsample of the full oil data extracting 100 values randomly without replacement, here seed was " + str(seed)}, data_set)

def pumadyn(seed=default_seed, data_set='pumadyn-32nm'):
        path = os.path.join(data_path, data_set)
        tar = tarfile.open(os.path.join(path, 'pumadyn-32nm.tar.gz'))
        print('Extracting file.')
        tar.extractall(path=path)
        tar.close()
    # Data is variance 1, no need to normalize.
    data = np.loadtxt(os.path.join(data_path, data_set, 'pumadyn-32nm', 'Dataset.data.gz'))
    indices = np.random.permutation(data.shape[0])
    indicesTrain = indices[0:7168]
    indicesTest = indices[7168:-1]
    indicesTrain.sort(axis=0)
    indicesTest.sort(axis=0)
    X = data[indicesTrain, 0:-2]
    Y = data[indicesTrain, -1][:, None]
    Xtest = data[indicesTest, 0:-2]
    Ytest = data[indicesTest, -1][:, None]
    return data_details_return({'X': X, 'Y': Y, 'Xtest': Xtest, 'Ytest': Ytest, 'seed': seed}, data_set)

def robot_wireless(data_set='robot_wireless'):
    # WiFi access point strengths on a tour around UW Paul Allen building.
    file_name = os.path.join(data_path, data_set, 'uw-floor.txt')
    all_time = np.genfromtxt(file_name, usecols=(0))
    macaddress = np.genfromtxt(file_name, usecols=(1), dtype='string')
    x = np.genfromtxt(file_name, usecols=(2))
    y = np.genfromtxt(file_name, usecols=(3))
    strength = np.genfromtxt(file_name, usecols=(4))
    addresses = np.unique(macaddress)
    times = np.unique(all_time)
    addresses.sort()
    times.sort()
    allY = np.zeros((len(times), len(addresses)))
    allX = np.zeros((len(times), 2))
    allY[:]=-92.
    strengths={}
    for address, j in zip(addresses, range(len(addresses))):
        ind = np.nonzero(address==macaddress)
        temp_strengths=strength[ind]
        temp_x=x[ind]
        temp_y=y[ind]
        temp_times = all_time[ind]
        for time in temp_times:
            vals = time==temp_times
            if any(vals):
                ind2 = np.nonzero(vals)
                i = np.nonzero(time==times)
                allY[i, j] = temp_strengths[ind2]
                allX[i, 0] = temp_x[ind2]
                allX[i, 1] = temp_y[ind2]
    allY = (allY + 85.)/15.

    X = allX[0:215, :]
    Y = allY[0:215, :]

    Xtest = allX[215:, :]
    Ytest = allY[215:, :]
    return data_details_return({'X': X, 'Y': Y, 'Xtest': Xtest, 'Ytest': Ytest, 'addresses' : addresses, 'times' : times}, data_set)

def silhouette(data_set='ankur_pose_data'):
    # Ankur Agarwal and Bill Trigg's silhoutte data.
    mat_data = scipy.io.loadmat(os.path.join(data_path, data_set, 'ankurDataPoseSilhouette.mat'))
    inMean = np.mean(mat_data['Y'])
    inScales = np.sqrt(np.var(mat_data['Y']))
    X = mat_data['Y'] - inMean
    X = X / inScales
    Xtest = mat_data['Y_test'] - inMean
    Xtest = Xtest / inScales
    Y = mat_data['Z']
    Ytest = mat_data['Z_test']
    return data_details_return({'X': X, 'Y': Y, 'Xtest': Xtest, 'Ytest': Ytest}, data_set)

def decampos_digits(data_set='decampos_characters', which_digits=[0,1,2,3,4,5,6,7,8,9]):
    path = os.path.join(data_path, data_set)
    digits = np.load(os.path.join(path, 'digits.npy'))
    digits = digits[which_digits,:,:,:]
    num_classes, num_samples, height, width = digits.shape
    Y = digits.reshape((digits.shape[0]*digits.shape[1],digits.shape[2]*digits.shape[3]))
    lbls = np.array([[l]*num_samples for l in which_digits]).reshape(Y.shape[0], 1)
    str_lbls = np.array([[str(l)]*num_samples for l in which_digits])
    return data_details_return({'Y': Y, 'lbls': lbls, 'str_lbls' : str_lbls, 'info': 'Digits data set from the de Campos characters data'}, data_set)

def ripley_synth(data_set='ripley_prnn_data'):
    train = np.genfromtxt(os.path.join(data_path, data_set, 'synth.tr'), skip_header=1)
    X = train[:, 0:2]
    y = train[:, 2:3]
    test = np.genfromtxt(os.path.join(data_path, data_set, 'synth.te'), skip_header=1)
    Xtest = test[:, 0:2]
    ytest = test[:, 2:3]
    return data_details_return({'X': X, 'Y': y, 'Xtest': Xtest, 'Ytest': ytest, 'info': 'Synthetic data generated by Ripley for a two class classification problem.'}, data_set)

def global_average_temperature(data_set='global_temperature', num_train=1000, refresh_data=False):
    if data_available(data_set) and not refresh_data:
        print 'Using cached version of the data set, to use latest version set refresh_data to True'
    else:
    data = np.loadtxt(os.path.join(data_path, data_set, 'GLBTS.long.data'))
    print 'Most recent data observation from month ', data[-1, 1], ' in year ', data[-1, 0]
    allX = data[data[:, 3]!=-99.99, 2:3]
    allY = data[data[:, 3]!=-99.99, 3:4]
    X = allX[:num_train, 0:1]
    Xtest = allX[num_train:, 0:1]
    Y = allY[:num_train, 0:1]
    Ytest = allY[num_train:, 0:1]
    return data_details_return({'X': X, 'Y': Y, 'Xtest': Xtest, 'Ytest': Ytest, 'info': "Mauna Loa data with " + str(num_train) + " values used as training points."}, data_set)

def mauna_loa(data_set='mauna_loa', num_train=545, refresh_data=False):
    else:
    data = np.loadtxt(os.path.join(data_path, data_set, 'co2_mm_mlo.txt'))

def boxjenkins_airline(data_set='boxjenkins_airline', num_train=96):
    data = np.loadtxt(os.path.join(data_path, data_set, 'boxjenkins_airline.csv'), delimiter=',')
    Y = data[:num_train, 1:2]
    X = data[:num_train, 0:1]
    Xtest = data[num_train:, 0:1]
    Ytest = data[num_train:, 1:2]
    return data_details_return({'X': X, 'Y': Y, 'Xtest': Xtest, 'Ytest': Ytest, 'info': "Montly airline passenger data from Box & Jenkins 1976."}, data_set)

def osu_run1(data_set='osu_run1', sample_every=4):
        zip = zipfile.ZipFile(os.path.join(data_path, data_set, 'run1TXT.ZIP'), 'r')
        for name in zip.namelist():
            zip.extract(name, path)
    Y, connect = GPy.util.mocap.load_text_data('Aug210106', path)
    Y = Y[0:-1:sample_every, :]
    return data_details_return({'Y': Y, 'connect' : connect}, data_set)

def swiss_roll_generated(num_samples=1000, sigma=0.0):
    with open(os.path.join(os.path.dirname(__file__), 'datasets', 'swiss_roll.pickle')) as f:
        data = pickle.load(f)
    Na = data['Y'].shape[0]
    perm = np.random.permutation(np.r_[:Na])[:num_samples]
    Y = data['Y'][perm, :]
    t = data['t'][perm]
    c = data['colors'][perm, :]
    so = np.argsort(t)
    Y = Y[so, :]
    t = t[so]
    c = c[so, :]
    return {'Y':Y, 't':t, 'colors':c}

def hapmap3(data_set='hapmap3'):
    """
    The HapMap phase three SNP dataset - 1184 samples out of 11 populations.

    SNP_matrix (A) encoding [see Paschou et all. 2007 (PCA-Correlated SNPs...)]:
    Let (B1,B2) be the alphabetically sorted bases, which occur in the j-th SNP, then

          /  1, iff SNPij==(B1,B1)
    Aij = |  0, iff SNPij==(B1,B2)
          \ -1, iff SNPij==(B2,B2)

    The SNP data and the meta information (such as iid, sex and phenotype) are
    stored in the dataframe datadf, index is the Individual ID,
    with following columns for metainfo:

        * family_id   -> Family ID
        * paternal_id -> Paternal ID
        * maternal_id -> Maternal ID
        * sex         -> Sex (1=male; 2=female; other=unknown)
        * phenotype   -> Phenotype (-9, or 0 for unknown)
        * population  -> Population string (e.g. 'ASW' - 'YRI')
        * rest are SNP rs (ids)

    More information is given in infodf:

        * Chromosome:
            - autosomal chromosemes                -> 1-22
            - X    X chromosome                    -> 23
            - Y    Y chromosome                    -> 24
            - XY   Pseudo-autosomal region of X    -> 25
            - MT   Mitochondrial                   -> 26
        * Relative Positon (to Chromosome) [base pairs]
    """
    try:
        from pandas import read_pickle, DataFrame
        from sys import stdout
        import bz2
    except ImportError as i:
        raise i, "Need pandas for hapmap dataset, make sure to install pandas (http://pandas.pydata.org/) before loading the hapmap dataset"

    dir_path = os.path.join(data_path,'hapmap3')
    hapmap_file_name = 'hapmap3_r2_b36_fwd.consensus.qc.poly'
    unpacked_files = [os.path.join(dir_path, hapmap_file_name+ending) for ending in ['.ped', '.map']]
    unpacked_files_exist = reduce(lambda a, b:a and b, map(os.path.exists, unpacked_files))

    if not unpacked_files_exist and not data_available(data_set):

    preprocessed_data_paths = [os.path.join(dir_path,hapmap_file_name + file_name) for file_name in \
                               ['.snps.pickle',
                                '.info.pickle',
                                '.nan.pickle']]

    if not reduce(lambda a,b: a and b, map(os.path.exists, preprocessed_data_paths)):
        if not overide_manual_authorize and not prompt_user("Preprocessing requires ~25GB "
                            "of memory and can take a (very) long time, continue? [Y/n]"):
            print "Preprocessing required for further usage."
            return
        status = "Preprocessing data, please be patient..."
        def write_status(message, progress, status):
            stdout.write(" "*len(status)); stdout.write("\r"); stdout.flush()
            status = r"[{perc: <{ll}}] {message: <13s}".format(message=message, ll=20,
                                                               perc="="*int(20.*progress/100.))
            stdout.write(status); stdout.flush()
            return status
        if not unpacked_files_exist:
            status=write_status('unpacking...', 0, '')
            curr = 0
            for newfilepath in unpacked_files:
                if not os.path.exists(newfilepath):
                    filepath = newfilepath + '.bz2'
                    file_size = os.path.getsize(filepath)
                    with open(newfilepath, 'wb') as new_file, open(filepath, 'rb') as f:
                        decomp = bz2.BZ2Decompressor()
                        file_processed = 0
                        buffsize = 100 * 1024
                        for data in iter(lambda : f.read(buffsize), b''):
                            new_file.write(decomp.decompress(data))
                            file_processed += len(data)
                            status=write_status('unpacking...', curr+12.*file_processed/(file_size), status)
                curr += 12
                status=write_status('unpacking...', curr, status)
                os.remove(filepath)
        status=write_status('reading .ped...', 25, status)
        # Preprocess data:
        snpstrnp = np.loadtxt(unpacked_files[0], dtype=str)
        status=write_status('reading .map...', 33, status)
        mapnp = np.loadtxt(unpacked_files[1], dtype=str)
        status=write_status('reading relationships.txt...', 42, status)
        # and metainfo:
        infodf = DataFrame.from_csv(os.path.join(dir_path,'./relationships_w_pops_121708.txt'), header=0, sep='\t')
        infodf.set_index('IID', inplace=1)
        status=write_status('filtering nan...', 45, status)
        snpstr = snpstrnp[:,6:].astype('S1').reshape(snpstrnp.shape[0], -1, 2)
        inan = snpstr[:,:,0] == '0'
        status=write_status('filtering reference alleles...', 55, status)
        ref = np.array(map(lambda x: np.unique(x)[-2:], snpstr.swapaxes(0,1)[:,:,:]))
        status=write_status('encoding snps...', 70, status)
        # Encode the information for each gene in {-1,0,1}:
        status=write_status('encoding snps...', 73, status)
        snps = (snpstr==ref[None,:,:])
        status=write_status('encoding snps...', 76, status)
        snps = (snps*np.array([1,-1])[None,None,:])
        status=write_status('encoding snps...', 78, status)
        snps = snps.sum(-1)
        status=write_status('encoding snps...', 81, status)
        snps = snps.astype('i8')
        status=write_status('marking nan values...', 88, status)
        # put in nan values (masked as -128):
        snps[inan] = -128
        status=write_status('setting up meta...', 94, status)
        # get meta information:
        metaheader = np.r_[['family_id', 'iid', 'paternal_id', 'maternal_id', 'sex', 'phenotype']]
        metadf = DataFrame(columns=metaheader, data=snpstrnp[:,:6])
        metadf.set_index('iid', inplace=1)
        metadf = metadf.join(infodf.population)
        metadf.to_pickle(preprocessed_data_paths[1])
        # put everything together:
        status=write_status('setting up snps...', 96, status)
        snpsdf = DataFrame(index=metadf.index, data=snps, columns=mapnp[:,1])
        with open(preprocessed_data_paths[0], 'wb') as f:
            pickle.dump(f, snpsdf, protocoll=-1)
        status=write_status('setting up snps...', 98, status)
        inandf = DataFrame(index=metadf.index, data=inan, columns=mapnp[:,1])
        inandf.to_pickle(preprocessed_data_paths[2])
        status=write_status('done :)', 100, status)
        print ''
    else:
        print "loading snps..."
        snpsdf = read_pickle(preprocessed_data_paths[0])
        print "loading metainfo..."
        metadf = read_pickle(preprocessed_data_paths[1])
        print "loading nan entries..."
        inandf = read_pickle(preprocessed_data_paths[2])
    snps = snpsdf.values
    populations = metadf.population.values.astype('S3')
    hapmap = dict(name=data_set,
                  description='The HapMap phase three SNP dataset - '
                  '1184 samples out of 11 populations. inan is a '
                  'boolean array, containing wheather or not the '
                  'given entry is nan (nans are masked as '
                  '-128 in snps).',
                  snpsdf=snpsdf,
                  metadf=metadf,
                  snps=snps,
                  inan=inandf.values,
                  inandf=inandf,
                  populations=populations)
    return hapmap

def singlecell(data_set='singlecell'):

    filename = os.path.join(dir_path, 'singlecell.csv')
    genes = Y.columns
    labels = Y.index
    # data = np.loadtxt(os.path.join(dir_path, 'singlecell.csv'), delimiter=",", dtype=str)
    return data_details_return({'Y': Y, 'info' : "qPCR singlecell experiment in Mouse, measuring 48 gene expressions in 1-64 cell states. The labels have been created as in Guo et al. [2010]",
                                'genes': genes, 'labels':labels,
                                }, data_set)

def singlecell_rna_seq_islam(dataset='singlecell_islam'):
    if not data_available(dataset):
        download_data(dataset)

    from pandas import read_csv, DataFrame, concat
    dir_path = os.path.join(data_path, dataset)
    filename = os.path.join(dir_path, 'GSE29087_L139_expression_tab.txt.gz')
    data = read_csv(filename, sep='\t', skiprows=6, compression='gzip', header=None)
    header1 = read_csv(filename, sep='\t', header=None, skiprows=5, nrows=1, compression='gzip')
    header2 = read_csv(filename, sep='\t', header=None, skiprows=3, nrows=1, compression='gzip')
    data.columns = np.concatenate((header1.ix[0, :], header2.ix[0, 7:]))
    Y = data.set_index("Feature").ix[8:, 6:-4].T.astype(float)

    # read the info .soft
    filename = os.path.join(dir_path, 'GSE29087_family.soft.gz')
    info = read_csv(filename, sep='\t', skiprows=0, compression='gzip', header=None)
    # split at ' = '
    info = DataFrame(info.ix[:,0].str.split(' = ').tolist())
    # only take samples:
    info = info[info[0].str.contains("!Sample")]
    info[0] = info[0].apply(lambda row: row[len("!Sample_"):])

    groups = info.groupby(0).groups
    # remove 'GGG' from barcodes
    barcode = info[1][groups['barcode']].apply(lambda row: row[:-3])

    title = info[1][groups['title']]
    title.index = barcode
    title.name = 'title'
    geo_accession = info[1][groups['geo_accession']]
    geo_accession.index = barcode
    geo_accession.name = 'geo_accession'
    case_id = info[1][groups['source_name_ch1']]
    case_id.index = barcode
    case_id.name = 'source_name_ch1'

    info = concat([title, geo_accession, case_id], axis=1)
    labels = info.join(Y).source_name_ch1[:-4]
    labels[labels=='Embryonic stem cell'] = "ES"
    labels[labels=='Embryonic fibroblast'] = "MEF"

    return data_details_return({'Y': Y,
                                'info': '92 single cells (48 mouse ES cells, 44 mouse embryonic fibroblasts and 4 negative controls) were analyzed by single-cell tagged reverse transcription (STRT)',
                                'genes': Y.columns,
                                'labels': labels,
                                'datadf': data,
                                'infodf': info}, dataset)

def singlecell_rna_seq_deng(dataset='singlecell_deng'):

    from pandas import read_csv, isnull

    filename = os.path.join(dir_path, 'GSE45719_series_matrix.txt.gz')
    info = read_csv(filename, sep='\t', skiprows=0, compression='gzip', header=None, nrows=29, index_col=0)
    summary = info.loc['!Series_summary'][1]
    design = info.loc['!Series_overall_design']

    sample_info = read_csv(filename, sep='\t', skiprows=30, compression='gzip', header=0, index_col=0).T
    sample_info.columns = sample_info.columns.to_series().apply(lambda row: row[len("!Sample_"):])
    sample_info.columns.name = sample_info.columns.name[len("!Sample_"):]
    sample_info = sample_info[['geo_accession', 'characteristics_ch1',  'description']]
    sample_info = sample_info.iloc[:, np.r_[0:4, 5:sample_info.shape[1]]]
    c = sample_info.columns.to_series()
    c[1:4] = ['strain', 'cross', 'developmental_stage']
    sample_info.columns = c

    # get the labels right:
    rep = re.compile('\(.*\)')
    def filter_dev_stage(row):
        if isnull(row):
            row = "2-cell stage embryo"
        if row.startswith("developmental stage: "):
            row = row[len("developmental stage: "):]
        if row == 'adult':
            row += " liver"
        row = row.replace(' stage ', ' ')
        row = rep.sub(' ', row)
        row = row.strip(' ')
        return row
    labels = sample_info.developmental_stage.apply(filter_dev_stage)

    # Extract the tar file
    filename = os.path.join(dir_path, 'GSE45719_Raw.tar')
    with tarfile.open(filename, 'r') as files:
        print "Extracting Archive {}...".format(files.name)
        data = None
        gene_info = None
        message = ''
        members = files.getmembers()
        overall = len(members)
        for i, file_info in enumerate(members):
            f = files.extractfile(file_info)
            inner = read_csv(f, sep='\t', header=0, compression='gzip', index_col=0)
            print ' '*(len(message)+1) + '\r',
            message = "{: >7.2%}: Extracting: {}".format(float(i+1)/overall, file_info.name[:20]+"...txt.gz")
            print message,
            if data is None:
                data = inner.RPKM.to_frame()
                data.columns = [file_info.name[:-18]]
                gene_info = inner.Refseq_IDs.to_frame()
                gene_info.columns = [file_info.name[:-18]]
            else:
                data[file_info.name[:-18]] = inner.RPKM
                gene_info[file_info.name[:-18]] = inner.Refseq_IDs

    # Strip GSM number off data index
    rep = re.compile('GSM\d+_')
    data.columns = data.columns.to_series().apply(lambda row: row[rep.match(row).end():])
    data = data.T

    # make sure the same index gets used
    sample_info.index = data.index

    # get the labels from the description
    #rep = re.compile('fibroblast|\d+-cell|embryo|liver|early blastocyst|mid blastocyst|late blastocyst|blastomere|zygote', re.IGNORECASE)

    sys.stdout.write(' '*len(message) + '\r')
    print
    print "Read Archive {}".format(files.name)

    return data_details_return({'Y': data,
                                'series_info': info,
                                'sample_info': sample_info,
                                'gene_info': gene_info,
                                'summary': summary,
                                'design': design,
                                'genes': data.columns,
                                }, dataset)

def swiss_roll_1000():
    return swiss_roll(num_samples=1000)

def swiss_roll(num_samples=3000, data_set='swiss_roll'):
    mat_data = scipy.io.loadmat(os.path.join(data_path, data_set, 'swiss_roll_data.mat'))
    Y = mat_data['X_data'][:, 0:num_samples].transpose()
    return data_details_return({'Y': Y, 'X': mat_data['X_data'], 'info': "The first " + str(num_samples) + " points from the swiss roll data of Tennenbaum, de Silva and Langford (2001)."}, data_set)

def isomap_faces(num_samples=698, data_set='isomap_face_data'):
    mat_data = scipy.io.loadmat(os.path.join(data_path, data_set, 'face_data.mat'))
    Y = mat_data['images'][:, 0:num_samples].transpose()
    return data_details_return({'Y': Y, 'poses' : mat_data['poses'], 'lights': mat_data['lights'], 'info': "The first " + str(num_samples) + " points from the face data of Tennenbaum, de Silva and Langford (2001)."}, data_set)

def simulation_BGPLVM():
    mat_data = scipy.io.loadmat(os.path.join(data_path, 'BGPLVMSimulation.mat'))
    Y = np.array(mat_data['Y'], dtype=float)
    S = np.array(mat_data['initS'], dtype=float)
    mu = np.array(mat_data['initMu'], dtype=float)
    #return data_details_return({'S': S, 'Y': Y, 'mu': mu}, data_set)
    return {'Y': Y, 'S': S,
            'mu' : mu,
            'info': "Simulated test dataset generated in MATLAB to compare BGPLVM between python and MATLAB"}

def toy_rbf_1d(seed=default_seed, num_samples=500):
    """
    Samples values of a function from an RBF covariance with very small noise for inputs uniformly distributed between -1 and 1.

    :param seed: seed to use for random sampling.
    :type seed: int
    :param num_samples: number of samples to sample in the function (default 500).
    :type num_samples: int

    """
    num_in = 1
    X = np.random.uniform(low= -1.0, high=1.0, size=(num_samples, num_in))
    X.sort(axis=0)
    rbf = GPy.kern.RBF(num_in, variance=1., lengthscale=np.array((0.25,)))
    white = GPy.kern.White(num_in, variance=1e-2)
    kernel = rbf + white
    K = kernel.K(X)
    y = np.reshape(np.random.multivariate_normal(np.zeros(num_samples), K), (num_samples, 1))
    return {'X':X, 'Y':y, 'info': "Sampled " + str(num_samples) + " values of a function from an RBF covariance with very small noise for inputs uniformly distributed between -1 and 1."}

def toy_rbf_1d_50(seed=default_seed):
    data = toy_rbf_1d()
    indices = np.random.permutation(data['X'].shape[0])
    indices = indices[0:50]
    indices.sort(axis=0)
    return {'X': X, 'Y': Y, 'info': "Subsamples the toy_rbf_sample with 50 values randomly taken from the original sample.", 'seed' : seed}

def toy_linear_1d_classification(seed=default_seed):
    x1 = np.random.normal(-3, 5, 20)
    x2 = np.random.normal(3, 5, 20)
    X = (np.r_[x1, x2])[:, None]
    return {'X': X, 'Y':  sample_class(2.*X), 'F': 2.*X, 'seed' : seed}

def olivetti_glasses(data_set='olivetti_glasses', num_training=200, seed=default_seed):
    y = np.load(os.path.join(path, 'has_glasses.np'))
    y = np.where(y=='y',1,0).reshape(-1,1)
    faces = scipy.io.loadmat(os.path.join(path, 'olivettifaces.mat'))['faces'].T
    index = np.random.permutation(faces.shape[0])
    X = faces[index[:num_training],:]
    Xtest = faces[index[num_training:],:]
    Y = y[index[:num_training],:]
    Ytest = y[index[num_training:]]
    return data_details_return({'X': X, 'Y': Y, 'Xtest': Xtest, 'Ytest': Ytest, 'seed' : seed, 'info': "ORL Faces with labels identifiying who is wearing glasses and who isn't. Data is randomly partitioned according to given seed. Presence or absence of glasses was labelled by James Hensman."}, 'olivetti_faces')

def olivetti_faces(data_set='olivetti_faces'):
        zip = zipfile.ZipFile(os.path.join(path, 'att_faces.zip'), 'r')
    Y = []
    lbls = []
    for subject in range(40):
        for image in range(10):
            image_path = os.path.join(path, 'orl_faces', 's'+str(subject+1), str(image+1) + '.pgm')
            from GPy.util import netpbmfile
            Y.append(netpbmfile.imread(image_path).flatten())
            lbls.append(subject)
    Y = np.asarray(Y)
    lbls = np.asarray(lbls)[:, None]
    return data_details_return({'Y': Y, 'lbls' : lbls, 'info': "ORL Faces processed to 64x64 images."}, data_set)

def xw_pen(data_set='xw_pen'):
    Y = np.loadtxt(os.path.join(data_path, data_set, 'xw_pen_15.csv'), delimiter=',')
    X = np.arange(485)[:, None]
    return data_details_return({'Y': Y, 'X': X, 'info': "Tilt data from a personalized digital assistant pen. Plot in original paper showed regression between time steps 175 and 275."}, data_set)

def download_rogers_girolami_data(data_set='rogers_girolami_data'):
    if not data_available('rogers_girolami_data'):
        tar_file = os.path.join(path, 'firstcoursemldata.tar.gz')
        tar = tarfile.open(tar_file)

def olympic_100m_men(data_set='rogers_girolami_data'):
    download_rogers_girolami_data()
    olympic_data = scipy.io.loadmat(os.path.join(data_path, data_set, 'data', 'olympics.mat'))['male100']

    X = olympic_data[:, 0][:, None]
    Y = olympic_data[:, 1][:, None]
    return data_details_return({'X': X, 'Y': Y, 'info': "Olympic sprint times for 100 m men from 1896 until 2008. Example is from Rogers and Girolami's First Course in Machine Learning."}, data_set)

def olympic_100m_women(data_set='rogers_girolami_data'):
    olympic_data = scipy.io.loadmat(os.path.join(data_path, data_set, 'data', 'olympics.mat'))['female100']

    return data_details_return({'X': X, 'Y': Y, 'info': "Olympic sprint times for 100 m women from 1896 until 2008. Example is from Rogers and Girolami's First Course in Machine Learning."}, data_set)

def olympic_200m_women(data_set='rogers_girolami_data'):
    olympic_data = scipy.io.loadmat(os.path.join(data_path, data_set, 'data', 'olympics.mat'))['female200']

    return data_details_return({'X': X, 'Y': Y, 'info': "Olympic 200 m winning times for women from 1896 until 2008. Data is from Rogers and Girolami's First Course in Machine Learning."}, data_set)

def olympic_200m_men(data_set='rogers_girolami_data'):
    olympic_data = scipy.io.loadmat(os.path.join(data_path, data_set, 'data', 'olympics.mat'))['male200']

    return data_details_return({'X': X, 'Y': Y, 'info': "Male 200 m winning times for women from 1896 until 2008. Data is from Rogers and Girolami's First Course in Machine Learning."}, data_set)

def olympic_400m_women(data_set='rogers_girolami_data'):
    olympic_data = scipy.io.loadmat(os.path.join(data_path, data_set, 'data', 'olympics.mat'))['female400']

    return data_details_return({'X': X, 'Y': Y, 'info': "Olympic 400 m winning times for women until 2008. Data is from Rogers and Girolami's First Course in Machine Learning."}, data_set)

def olympic_400m_men(data_set='rogers_girolami_data'):
    olympic_data = scipy.io.loadmat(os.path.join(data_path, data_set, 'data', 'olympics.mat'))['male400']

    return data_details_return({'X': X, 'Y': Y, 'info': "Male 400 m winning times for women until 2008. Data is from Rogers and Girolami's First Course in Machine Learning."}, data_set)

def olympic_marathon_men(data_set='olympic_marathon_men'):
    olympics = np.genfromtxt(os.path.join(data_path, data_set, 'olympicMarathonTimes.csv'), delimiter=',')
    X = olympics[:, 0:1]
    Y = olympics[:, 1:2]

def olympic_sprints(data_set='rogers_girolami_data'):
    """All olympics sprint winning times for multiple output prediction."""
    X = np.zeros((0, 2))
    Y = np.zeros((0, 1))
    for i, dataset in enumerate([olympic_100m_men,
                              olympic_100m_women,
                              olympic_200m_men,
                              olympic_200m_women,
                              olympic_400m_men,
                              olympic_400m_women]):
        data = dataset()
        year = data['X']
        time = data['Y']
        X = np.vstack((X, np.hstack((year, np.ones_like(year)*i))))
        Y = np.vstack((Y, time))
    data['X'] = X
    data['Y'] = Y
    data['info'] = "Olympics sprint event winning for men and women to 2008. Data is from Rogers and Girolami's First Course in Machine Learning."
    return data_details_return({
        'X': X,
        'Y': Y,
        'info': "Olympics sprint event winning for men and women to 2008. Data is from Rogers and Girolami's First Course in Machine Learning.",
        'output_info': {
          0:'100m Men',
          1:'100m Women',
          2:'200m Men',
          3:'200m Women',
          4:'400m Men',
          5:'400m Women'}
        }, data_set)

# def movielens_small(partNo=1,seed=default_seed):
#     np.random.seed(seed=seed)

#     fileName = os.path.join(data_path, 'movielens', 'small', 'u' + str(partNo) + '.base')
#     fid = open(fileName)
#     uTrain = np.fromfile(fid, sep='\t', dtype=np.int16).reshape((-1, 4))
#     fid.close()
#     maxVals = np.amax(uTrain, axis=0)
#     numUsers = maxVals[0]
#     numFilms = maxVals[1]
#     numRatings = uTrain.shape[0]

#     Y = scipy.sparse.lil_matrix((numFilms, numUsers), dtype=np.int8)
#     for i in range(numUsers):
#         ind = pb.mlab.find(uTrain[:, 0]==i+1)
#         Y[uTrain[ind, 1]-1, i] = uTrain[ind, 2]

#     fileName = os.path.join(data_path, 'movielens', 'small', 'u' + str(partNo) + '.test')
#     uTest = np.fromfile(fid, sep='\t', dtype=np.int16).reshape((-1, 4))
#     numTestRatings = uTest.shape[0]

#     Ytest = scipy.sparse.lil_matrix((numFilms, numUsers), dtype=np.int8)
#         ind = pb.mlab.find(uTest[:, 0]==i+1)
#         Ytest[uTest[ind, 1]-1, i] = uTest[ind, 2]

#     lbls = np.empty((1,1))
#     lblstest = np.empty((1,1))
#     return {'Y':Y, 'lbls':lbls, 'Ytest':Ytest, 'lblstest':lblstest}

def crescent_data(num_data=200, seed=default_seed):
    """
Data set formed from a mixture of four Gaussians. In each class two of the Gaussians are elongated at right angles to each other and offset to form an approximation to the crescent data that is popular in semi-supervised learning as a toy problem.

    :param num_data_part: number of data to be sampled (default is 200).
    :type num_data: int
    :param seed: random seed to be used for data generation.

    """
    sqrt2 = np.sqrt(2)
    # Rotation matrix
    R = np.array([[sqrt2 / 2, -sqrt2 / 2], [sqrt2 / 2, sqrt2 / 2]])
    # Scaling matrices
    scales = []
    scales.append(np.array([[3, 0], [0, 1]]))
    scales.append([[1, 0], [0, 3]])
    means = []
    means.append(np.array([4, 4]))
    means.append(np.array([0, 4]))
    means.append(np.array([-4, -4]))
    means.append(np.array([0, -4]))

    Xparts = []
    num_data_part = []
    num_data_total = 0
    for i in range(0, 4):
        num_data_part.append(round(((i + 1) * num_data) / 4.))
        num_data_part[i] -= num_data_total
        part = np.random.normal(size=(num_data_part[i], 2))
        part = np.dot(np.dot(part, scales[i]), R) + means[i]
        Xparts.append(part)
        num_data_total += num_data_part[i]
    X = np.vstack((Xparts[0], Xparts[1], Xparts[2], Xparts[3]))

    Y = np.vstack((np.ones((num_data_part[0] + num_data_part[1], 1)), -np.ones((num_data_part[2] + num_data_part[3], 1))))
    return {'X':X, 'Y':Y, 'info': "Two separate classes of data formed approximately in the shape of two crescents."}

def creep_data(data_set='creep_rupture'):
    """Brun and Yoshida's metal creep rupture data."""
        tar_file = os.path.join(path, 'creeprupt.tar')
    all_data = np.loadtxt(os.path.join(data_path, data_set, 'taka'))
    y = all_data[:, 1:2].copy()
    features = [0]
    features.extend(range(2, 31))
    X = all_data[:, features].copy()
    return data_details_return({'X': X, 'y': y}, data_set)

def cifar10_patches(data_set='cifar-10'):
    """The Candian Institute for Advanced Research 10 image data set. Code for loading in this data is taken from this Boris Babenko's blog post, original code available here: http://bbabenko.tumblr.com/post/86756017649/learning-low-level-vision-feautres-in-10-lines-of-code"""
    filename = os.path.join(dir_path, 'cifar-10-python.tar.gz')
        import tarfile
        # This code is from Boris Babenko's blog post.
        # http://bbabenko.tumblr.com/post/86756017649/learning-low-level-vision-feautres-in-10-lines-of-code
        tfile = tarfile.open(filename, 'r:gz')
        tfile.extractall(dir_path)

    with open(os.path.join(dir_path, 'cifar-10-batches-py','data_batch_1'),'rb') as f:

    images = data['data'].reshape((-1,3,32,32)).astype('float32')/255
    images = np.rollaxis(images, 1, 4)
    patches = np.zeros((0,5,5,3))
    for x in range(0,32-5,5):
        for y in range(0,32-5,5):
            patches = np.concatenate((patches, images[:,x:x+5,y:y+5,:]), axis=0)
    patches = patches.reshape((patches.shape[0],-1))
    return data_details_return({'Y': patches, "info" : "32x32 pixel patches extracted from the CIFAR-10 data by Boris Babenko to demonstrate k-means features."}, data_set)

def cmu_mocap_49_balance(data_set='cmu_mocap'):
    """Load CMU subject 49's one legged balancing motion that was used by Alvarez, Luengo and Lawrence at AISTATS 2009."""
    train_motions = ['18', '19']
    test_motions = ['20']
    data = cmu_mocap('49', train_motions, test_motions, sample_every=4, data_set=data_set)
    data['info'] = "One legged balancing motions from CMU data base subject 49. As used in Alvarez, Luengo and Lawrence at AISTATS 2009. It consists of " + data['info']

def cmu_mocap_35_walk_jog(data_set='cmu_mocap'):
    """Load CMU subject 35's walking and jogging motions, the same data that was used by Taylor, Roweis and Hinton at NIPS 2007. but without their preprocessing. Also used by Lawrence at AISTATS 2007."""
    train_motions = ['01', '02', '03', '04', '05', '06',
                '07', '08', '09', '10', '11', '12',
                '13', '14', '15', '16', '17', '19',
                '20', '21', '22', '23', '24', '25',
                '26', '28', '30', '31', '32', '33', '34']
    test_motions = ['18', '29']
    data = cmu_mocap('35', train_motions, test_motions, sample_every=4, data_set=data_set)
    data['info'] = "Walk and jog data from CMU data base subject 35. As used in Tayor, Roweis and Hinton at NIPS 2007, but without their pre-processing (i.e. as used by Lawrence at AISTATS 2007). It consists of " + data['info']

def cmu_mocap(subject, train_motions, test_motions=[], sample_every=4, data_set='cmu_mocap'):
    """Load a given subject's training and test motions from the CMU motion capture data."""
    # Load in subject skeleton.
    subject_dir = os.path.join(data_path, data_set)

    # Make sure the data is downloaded.
    all_motions = train_motions + test_motions
    resource = cmu_urls_files(([subject], [all_motions]))
    data_resources[data_set] = data_resources['cmu_mocap_full'].copy()
    data_resources[data_set]['files'] = resource['files']
    data_resources[data_set]['urls'] = resource['urls']
    if resource['urls']:

    skel = GPy.util.mocap.acclaim_skeleton(os.path.join(subject_dir, subject + '.asf'))

    # Set up labels for each sequence
    exlbls = np.eye(len(train_motions))

    # Load sequences
    tot_length = 0
    temp_Y = []
    temp_lbls = []
    for i in range(len(train_motions)):
        temp_chan = skel.load_channels(os.path.join(subject_dir, subject + '_' + train_motions[i] + '.amc'))
        temp_Y.append(temp_chan[::sample_every, :])
        temp_lbls.append(np.tile(exlbls[i, :], (temp_Y[i].shape[0], 1)))
        tot_length += temp_Y[i].shape[0]

    Y = np.zeros((tot_length, temp_Y[0].shape[1]))
    lbls = np.zeros((tot_length, temp_lbls[0].shape[1]))

    end_ind = 0
    for i in range(len(temp_Y)):
        start_ind = end_ind
        end_ind += temp_Y[i].shape[0]
        Y[start_ind:end_ind, :] = temp_Y[i]
        lbls[start_ind:end_ind, :] = temp_lbls[i]
    if len(test_motions) > 0:
        temp_Ytest = []
        temp_lblstest = []

        testexlbls = np.eye(len(test_motions))
        tot_test_length = 0
        for i in range(len(test_motions)):
            temp_chan = skel.load_channels(os.path.join(subject_dir, subject + '_' + test_motions[i] + '.amc'))
            temp_Ytest.append(temp_chan[::sample_every, :])
            temp_lblstest.append(np.tile(testexlbls[i, :], (temp_Ytest[i].shape[0], 1)))
            tot_test_length += temp_Ytest[i].shape[0]

        # Load test data
        Ytest = np.zeros((tot_test_length, temp_Ytest[0].shape[1]))
        lblstest = np.zeros((tot_test_length, temp_lblstest[0].shape[1]))

        end_ind = 0
        for i in range(len(temp_Ytest)):
            start_ind = end_ind
            end_ind += temp_Ytest[i].shape[0]
            Ytest[start_ind:end_ind, :] = temp_Ytest[i]
            lblstest[start_ind:end_ind, :] = temp_lblstest[i]
    else:
        Ytest = None
        lblstest = None

    info = 'Subject: ' + subject + '. Training motions: '
    for motion in train_motions:
        info += motion + ', '
    info = info[:-2]
        info += '. Test motions: '
        for motion in test_motions:
            info += motion + ', '
        info = info[:-2] + '.'
    else:
        info += '.'
    if sample_every != 1:
        info += ' Data is sub-sampled to every ' + str(sample_every) + ' frames.'
    return data_details_return({'Y': Y, 'lbls' : lbls, 'Ytest': Ytest, 'lblstest' : lblstest, 'info': info, 'skel': skel}, data_set)

'''
Created on Feb 13, 2014

@author: maxzwiessele
'''
import unittest
import GPy
from GPy.core.parameterization.parameter_core import HierarchyError
from GPy.core.parameterization.observable_array import ObsAr
from GPy.core.parameterization.transformations import NegativeLogexp, Logistic
from GPy.core.parameterization.parameterized import Parameterized
from GPy.core.parameterization.param import Param
from GPy.core.parameterization.index_operations import ParameterIndexOperations

class ArrayCoreTest(unittest.TestCase):
    def setUp(self):
        self.X = np.random.normal(1,1, size=(100,10))
        self.obsX = ObsAr(self.X)

    def test_init(self):
        X = ObsAr(self.X)
        X2 = ObsAr(X)
        self.assertIs(X, X2, "no new Observable array, when Observable is given")

    def test_slice(self):
        t1 = self.X[2:78]
        t2 = self.obsX[2:78]
        self.assertListEqual(t1.tolist(), t2.tolist(), "Slicing should be the exact same, as in ndarray")

class ParameterizedTest(unittest.TestCase):

        self.rbf = GPy.kern.RBF(20)
        self.white = GPy.kern.White(1)
        from GPy.core.parameterization import Param
        from GPy.core.parameterization.transformations import Logistic
        self.param = Param('param', np.random.uniform(0,1,(10,5)), Logistic(0, 1))

        self.test1 = GPy.core.Parameterized("test model")
        self.test1.param = self.param
        self.test1.kern = self.rbf+self.white
        self.test1.link_parameter(self.test1.kern)
        self.test1.link_parameter(self.param, 0)

        # print self.test1:
        #=============================================================================
        # test_model.          |    Value    |  Constraint   |  Prior  |  Tied to
        # param                |  (25L, 2L)  |   {0.0,1.0}   |         |
        # add.rbf.variance     |        1.0  |  0.0,1.0 +ve  |         |
        # add.rbf.lengthscale  |        1.0  |  0.0,1.0 +ve  |         |
        # add.white.variance   |        1.0  |  0.0,1.0 +ve  |         |

        x = np.linspace(-2,6,4)[:,None]
        y = np.sin(x)
        self.testmodel = GPy.models.GPRegression(x,y)
        # print self.testmodel:
        # GP_regression.           |  Value  |  Constraint  |  Prior  |  Tied to
        # rbf.variance             |    1.0  |     +ve      |         |
        # rbf.lengthscale          |    1.0  |     +ve      |         |
        # Gaussian_noise.variance  |    1.0  |     +ve      |         |

    def test_add_parameter(self):
        self.assertEquals(self.rbf._parent_index_, 0)
        self.assertEquals(self.white._parent_index_, 1)
        self.assertEquals(self.param._parent_index_, 0)
        pass

    def test_fixes(self):
        self.white.fix(warning=False)
        self.test1.unlink_parameter(self.param)
        self.assertTrue(self.test1._has_fixes())
        from GPy.core.parameterization.transformations import FIXED, UNFIXED
        self.assertListEqual(self.test1._fixes_.tolist(),[UNFIXED,UNFIXED,FIXED])
        self.test1.kern.link_parameter(self.white, 0)
        self.assertListEqual(self.test1._fixes_.tolist(),[FIXED,UNFIXED,UNFIXED])
        self.test1.kern.rbf.fix()
        self.assertListEqual(self.test1._fixes_.tolist(),[FIXED]*3)
        self.test1.fix()
        self.assertTrue(self.test1.is_fixed)
        self.assertListEqual(self.test1._fixes_.tolist(),[FIXED]*self.test1.size)

    def test_remove_parameter(self):
        from GPy.core.parameterization.transformations import FIXED, UNFIXED, __fixed__, Logexp
        self.white.fix()
        self.test1.kern.unlink_parameter(self.white)
        self.assertIs(self.test1._fixes_,None)

        self.assertIsInstance(self.white.constraints, ParameterIndexOperations)
        self.assertListEqual(self.white._fixes_.tolist(), [FIXED])
        self.assertIs(self.test1.constraints, self.rbf.constraints._param_index_ops)
        self.assertIs(self.test1.constraints, self.param.constraints._param_index_ops)

        self.test1.link_parameter(self.white, 0)
        self.assertIs(self.test1.constraints, self.white.constraints._param_index_ops)
        self.assertListEqual(self.test1.constraints[__fixed__].tolist(), [0])
        self.assertIs(self.white._fixes_,None)
        self.assertListEqual(self.test1._fixes_.tolist(),[FIXED] + [UNFIXED] * 52)

        self.test1.unlink_parameter(self.white)
        self.assertListEqual(self.test1.constraints[Logexp()].tolist(), range(self.param.size, self.param.size+self.rbf.size))

    def test_remove_parameter_param_array_grad_array(self):
        val = self.test1.kern.param_array.copy()
        self.assertListEqual(self.test1.kern.param_array.tolist(), val[:2].tolist())

    def test_add_parameter_already_in_hirarchy(self):
        self.assertRaises(HierarchyError, self.test1.link_parameter, self.white.parameters[0])

    def test_default_constraints(self):
        self.assertIs(self.rbf.variance.constraints._param_index_ops, self.rbf.constraints._param_index_ops)
        self.assertListEqual(self.rbf.constraints.indices()[0].tolist(), range(2))
        from GPy.core.parameterization.transformations import Logexp
        kern = self.test1.kern
        self.test1.unlink_parameter(kern)
        self.assertListEqual(kern.constraints[Logexp()].tolist(), range(3))

    def test_constraints(self):
        self.rbf.constrain(GPy.transformations.Square(), False)
        self.assertListEqual(self.test1.constraints[GPy.transformations.Square()].tolist(), range(self.param.size, self.param.size+self.rbf.size))
        self.assertListEqual(self.test1.constraints[GPy.transformations.Logexp()].tolist(), [self.param.size+self.rbf.size])

        self.test1.kern.unlink_parameter(self.rbf)
        self.assertListEqual(self.test1.constraints[GPy.transformations.Square()].tolist(), [])

    def test_constraints_link_unlink(self):
        self.test1.unlink_parameter(self.test1.kern)
        self.test1.kern.rbf.unlink_parameter(self.test1.kern.rbf.lengthscale)
        self.test1.kern.rbf.link_parameter(self.test1.kern.rbf.lengthscale)

    def test_constraints_views(self):
        self.assertEqual(self.white.constraints._offset, self.param.size+self.rbf.size)
        self.assertEqual(self.rbf.constraints._offset, self.param.size)
        self.assertEqual(self.param.constraints._offset, 0)

    def test_fixing_randomize(self):
        self.white.fix(warning=True)
        val = float(self.white.variance)
        self.test1.randomize()
        self.assertEqual(val, self.white.variance)

    def test_randomize(self):
        ps = self.test1.param.view(np.ndarray).copy()
        self.test1.param[2:5].fix()
        self.test1.param.randomize()
        self.assertFalse(np.all(ps==self.test1.param),str(ps)+str(self.test1.param))

    def test_fixing_randomize_parameter_handling(self):
        self.rbf.fix(warning=True)
        val = float(self.rbf.variance)
        self.test1.kern.randomize()
        self.assertEqual(val, self.rbf.variance)

    def test_updates(self):
        val = float(self.testmodel.log_likelihood())
        self.testmodel.update_model(False)
        self.testmodel.kern.randomize()
        self.testmodel.likelihood.randomize()
        self.assertEqual(val, self.testmodel.log_likelihood())
        self.testmodel.update_model(True)
        self.assertNotEqual(val, self.testmodel.log_likelihood())

    def test_fixing_optimize(self):
        self.testmodel.kern.lengthscale.fix()
        val = float(self.testmodel.kern.lengthscale)
        self.testmodel.randomize()
        self.assertEqual(val, self.testmodel.kern.lengthscale)

    def test_add_parameter_in_hierarchy(self):
        self.test1.kern.rbf.link_parameter(Param("NEW", np.random.rand(2), NegativeLogexp()), 1)
        self.assertListEqual(self.test1.constraints[NegativeLogexp()].tolist(), range(self.param.size+1, self.param.size+1 + 2))
        self.assertListEqual(self.test1.constraints[GPy.transformations.Logistic(0,1)].tolist(), range(self.param.size))
        self.assertListEqual(self.test1.constraints[GPy.transformations.Logexp(0,1)].tolist(), np.r_[50, 53:55].tolist())

    def test_regular_expression_misc(self):

        variances = self.testmodel['.*var'].values()
        self.testmodel['.*var'].fix()
        np.testing.assert_equal(variances, self.testmodel['.*var'].values())

    def test_fix_unfix(self):
        fixed = self.testmodel.kern.lengthscale.fix()
        self.assertListEqual(fixed.tolist(), [0])
        unfixed = self.testmodel.kern.lengthscale.unfix()
        self.testmodel.kern.lengthscale.constrain_positive()
        self.assertListEqual(unfixed.tolist(), [0])

        fixed = self.testmodel.kern.fix()
        self.assertListEqual(fixed.tolist(), [0,1])
        unfixed = self.testmodel.kern.unfix()
        self.assertListEqual(unfixed.tolist(), [0,1])

    def test_constraints_in_init(self):
        class Test(Parameterized):
            def __init__(self, name=None, parameters=[], *a, **kw):
                super(Test, self).__init__(name=name)
                self.x = Param('x', np.random.uniform(0,1,(3,4)))
                self.x[0].constrain_bounded(0,1)
                self.link_parameter(self.x)
                self.x[1].fix()
        t = Test()
        c = {Logistic(0,1): np.array([0, 1, 2, 3]), 'fixed': np.array([4, 5, 6, 7])}
        np.testing.assert_equal(t.x.constraints[Logistic(0,1)], c[Logistic(0,1)])
        np.testing.assert_equal(t.x.constraints['fixed'], c['fixed'])

    def test_parameter_modify_in_init(self):
        class TestLikelihood(Parameterized):
            def __init__(self, param1 = 2., param2 = 3.):
                super(TestLikelihood, self).__init__("TestLike")
                self.p1 = Param('param1', param1)
                self.p2 = Param('param2', param2)

                self.link_parameter(self.p1)
                self.link_parameter(self.p2)

                self.p1.fix()
                self.p1.unfix()
                self.p2.constrain_negative()
                self.p2.constrain_positive()
                self.p2.fix()

        m = TestLikelihood()
        print m
        val = m.p1.values.copy()
        self.assert_(m.p1.is_fixed)
        self.assert_(m.constraints[GPy.constraints.Logexp()].tolist(), [1])
        m.randomize()
        self.assertEqual(m.p1, val)

    def test_printing(self):
        print self.test1
        print self.param
        print self.test1['']

if __name__ == "__main__":
    #import sys;sys.argv = ['', 'Test.test_add_parameter']
    unittest.main()

# Copyright (c) 2012, James Hensman
# Licensed under the BSD 3-clause license (see LICENSE.txt)

from ..core import SparseGP
from ..core.sparse_gp_mpi import SparseGP_MPI
from .. import likelihoods
from .. import kern
from ..inference.latent_function_inference import VarDTC
from ..core.parameterization.variational import NormalPosterior
from GPy.inference.latent_function_inference.var_dtc_parallel import VarDTC_minibatch

class SparseGPRegression(SparseGP_MPI):
    """
    Gaussian Process model for regression

    This is a thin wrapper around the SparseGP class, with a set of sensible defalts

    :param X: input observations
    :param Y: observed values
    :param kernel: a GPy kernel, defaults to rbf+white
    :param Z: inducing inputs (optional, see note)
    :type Z: np.ndarray (num_inducing x input_dim) | None
    :param num_inducing: number of inducing points (ignored if Z is passed, see note)
    :type num_inducing: int
    :rtype: model object

    .. Note:: If no Z array is passed, num_inducing (default 10) points are selected from the data. Other wise num_inducing is ignored
    .. Note:: Multiple independent outputs are allowed using columns of Y

    """

    def __init__(self, X, Y, kernel=None, Z=None, num_inducing=10, X_variance=None, normalizer=None, mpi_comm=None):
        num_data, input_dim = X.shape

        # kern defaults to rbf (plus white for stability)
        if kernel is None:
            kernel = kern.RBF(input_dim)#  + kern.white(input_dim, variance=1e-3)

        # Z defaults to a subset of the data
        if Z is None:
            i = np.random.permutation(num_data)[:min(num_inducing, num_data)]
            Z = X.view(np.ndarray)[i].copy()
        else:
            assert Z.shape[1] == input_dim

        likelihood = likelihoods.Gaussian()

        if not (X_variance is None):
            X = NormalPosterior(X,X_variance)
            
        if mpi_comm is not None:
            from ..inference.latent_function_inference.var_dtc_parallel import VarDTC_minibatch
            infr = VarDTC_minibatch(mpi_comm=mpi_comm)
        else:
            infr = VarDTC()

        SparseGP_MPI.__init__(self, X, Y, Z, kernel, likelihood, inference_method=infr, normalizer=normalizer, mpi_comm=mpi_comm)

    def parameters_changed(self):
        from ..inference.latent_function_inference.var_dtc_parallel import update_gradients_sparsegp,VarDTC_minibatch
        if isinstance(self.inference_method,VarDTC_minibatch):
            update_gradients_sparsegp(self, mpi_comm=self.mpi_comm)
        else:
            super(SparseGPRegression, self).parameters_changed()

class SparseGPRegressionUncertainInput(SparseGP):
    """
    Gaussian Process model for regression with Gaussian variance on the inputs (X_variance)

    """

    def __init__(self, X, X_variance, Y, kernel=None, Z=None, num_inducing=10, normalizer=None):
        """
        :param X: input observations
        :type X: np.ndarray (num_data x input_dim)
        :param X_variance: The uncertainty in the measurements of X (Gaussian variance, optional)
        :type X_variance: np.ndarray (num_data x input_dim)
        :param Y: observed values
        :param kernel: a GPy kernel, defaults to rbf+white
        :param Z: inducing inputs (optional, see note)
        :type Z: np.ndarray (num_inducing x input_dim) | None
        :param num_inducing: number of inducing points (ignored if Z is passed, see note)
        :type num_inducing: int
        :rtype: model object

        .. Note:: If no Z array is passed, num_inducing (default 10) points are selected from the data. Other wise num_inducing is ignored
        .. Note:: Multiple independent outputs are allowed using columns of Y
        """

            kernel = kern.RBF(input_dim) + kern.White(input_dim, variance=1e-3)

            Z = X[i].copy()
        else:

        SparseGP.__init__(self, X, Y, Z, kernel, likelihood, X_variance=X_variance, inference_method=VarDTC(), normalizer=normalizer)
        self.ensure_default_constraints()

# Copyright (c) 2012, GPy authors (see AUTHORS.txt).

"""
The package for the Psi statistics computation of the linear kernel for SSGPLVM
"""

from ....util.linalg import tdot

def psicomputations(variance, Z, variational_posterior):
    """
    Compute psi-statistics for ss-linear kernel
    """
    # here are the "statistics" for psi0, psi1 and psi2
    # Produced intermediate results:
    # psi0    N
    # psi1    NxM
    # psi2    MxM
    mu = variational_posterior.mean
    S = variational_posterior.variance
    gamma = variational_posterior.binary_prob

    psi0 = (gamma*(np.square(mu)+S)*variance).sum(axis=-1)
    psi1 = np.inner(variance*gamma*mu,Z)
    psi2 = np.inner(np.square(variance)*(gamma*((1-gamma)*np.square(mu)+S)).sum(axis=0)*Z,Z)+tdot(psi1.T)

    return psi0, psi1, psi2

def psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior):

    dL_dvar, dL_dgamma, dL_dmu, dL_dS, dL_dZ = _psi2computations(dL_dpsi2, variance, Z, mu, S, gamma)

    # Compute for psi0 and psi1
    mu2S = np.square(mu)+S
    dL_dvar += np.einsum('n,nq,nq->q',dL_dpsi0,gamma,mu2S) + np.einsum('nm,nq,mq,nq->q',dL_dpsi1,gamma,Z,mu)
    dL_dgamma += np.einsum('n,q,nq->nq',dL_dpsi0,variance,mu2S) + np.einsum('nm,q,mq,nq->nq',dL_dpsi1,variance,Z,mu)
    dL_dmu += np.einsum('n,nq,q,nq->nq',dL_dpsi0,gamma,2.*variance,mu) + np.einsum('nm,nq,q,mq->nq',dL_dpsi1,gamma,variance,Z)
    dL_dS += np.einsum('n,nq,q->nq',dL_dpsi0,gamma,variance)
    dL_dZ +=  np.einsum('nm,nq,q,nq->mq',dL_dpsi1,gamma, variance,mu)
    
    return dL_dvar, dL_dZ, dL_dmu, dL_dS, dL_dgamma

def _psi2computations(dL_dpsi2, variance, Z, mu, S, gamma):
    """
    Z - MxQ
    mu - NxQ
    S - NxQ
    gamma - NxQ
    """
    # here are the "statistics" for psi1 and psi2
    # _psi2_dvariance      Q
    # _psi2_dZ             MxQ
    # _psi2_dgamma         NxQ
    # _psi2_dmu            NxQ
    # _psi2_dS             NxQ
    
    mu2 = np.square(mu)
    gamma2 = np.square(gamma)
    variance2 = np.square(variance)
    mu2S = mu2+S # NxQ
    gvm = np.einsum('nq,nq,q->nq',gamma,mu,variance)
    common_sum = np.einsum('nq,mq->nm',gvm,Z)
#     common_sum = np.einsum('nq,q,mq,nq->nm',gamma,variance,Z,mu) # NxM
    Z_expect = np.einsum('mo,mq,oq->q',dL_dpsi2,Z,Z)
    dL_dpsi2T = dL_dpsi2+dL_dpsi2.T
    tmp = np.einsum('mo,oq->mq',dL_dpsi2T,Z)
    common_expect = np.einsum('mq,nm->nq',tmp,common_sum)
#     common_expect = np.einsum('mo,mq,no->nq',dL_dpsi2+dL_dpsi2.T,Z,common_sum)
    Z2_expect = np.einsum('om,nm->no',dL_dpsi2T,common_sum)
    Z1_expect = np.einsum('om,mq->oq',dL_dpsi2T,Z)
    
    dL_dvar = np.einsum('nq,q,q->q',2.*(gamma*mu2S-gamma2*mu2),variance,Z_expect)+\
        np.einsum('nq,nq,nq->q',common_expect,gamma,mu)
        
    dL_dgamma = np.einsum('q,q,nq->nq',Z_expect,variance2,(mu2S-2.*gamma*mu2))+\
        np.einsum('nq,q,nq->nq',common_expect,variance,mu)
    
    dL_dmu = np.einsum('q,q,nq,nq->nq',Z_expect,variance2,mu,2.*(gamma-gamma2))+\
            np.einsum('nq,nq,q->nq',common_expect,gamma,variance)
                    
    dL_dS = np.einsum('q,nq,q->nq',Z_expect,gamma,variance2)
    
#     dL_dZ = 2.*(np.einsum('om,nq,q,mq,nq->oq',dL_dpsi2,gamma,variance2,Z,(mu2S-gamma*mu2))+np.einsum('om,nq,q,nq,nm->oq',dL_dpsi2,gamma,variance,mu,common_sum))
    dL_dZ = Z1_expect*np.einsum('nq,q,nq->q',gamma,variance2,(mu2S-gamma*mu2))+np.einsum('nq,q,nq,nm->mq',gamma,variance,mu,Z2_expect)

    return dL_dvar, dL_dgamma, dL_dmu, dL_dS, dL_dZ

# Copyright (c) 2013, GPy authors (see AUTHORS.txt).

from kern import Kern
from ...core.parameterization import Param
from ...core.parameterization.transformations import Logexp
from independent_outputs import index_to_slices

class ODE_UY(Kern):
    def __init__(self, input_dim, variance_U=3., variance_Y=1., lengthscale_U=1., lengthscale_Y=1., active_dims=None, name='ode_uy'):
        assert input_dim ==2, "only defined for 2 input dims"
        super(ODE_UY, self).__init__(input_dim, active_dims, name)

        self.variance_Y = Param('variance_Y', variance_Y, Logexp())
        self.variance_U = Param('variance_U', variance_Y, Logexp())
        self.lengthscale_Y = Param('lengthscale_Y', lengthscale_Y, Logexp())
        self.lengthscale_U = Param('lengthscale_U', lengthscale_Y, Logexp())

        self.link_parameters(self.variance_Y, self.variance_U, self.lengthscale_Y, self.lengthscale_U)

    def K(self, X, X2=None):
        # model :   a * dy/dt + b * y = U
        #lu=sqrt(3)/theta1  ly=1/theta2  theta2= a/b :thetay   sigma2=1/(2ab) :sigmay

        X,slices = X[:,:-1],index_to_slices(X[:,-1])
        if X2 is None:
            X2,slices2 = X,slices
            K = np.zeros((X.shape[0], X.shape[0]))
        else:
            X2,slices2 = X2[:,:-1],index_to_slices(X2[:,-1])
            K = np.zeros((X.shape[0], X2.shape[0]))

        #rdist = X[:,0][:,None] - X2[:,0][:,None].T
        rdist = X - X2.T
        ly=1/self.lengthscale_Y
        lu=np.sqrt(3)/self.lengthscale_U
        #iu=self.input_lengthU  #dimention of U
        Vu=self.variance_U
        Vy=self.variance_Y
        #Vy=ly/2
        #stop

        # kernel for kuu  matern3/2
        kuu = lambda dist:Vu * (1 + lu* np.abs(dist)) * np.exp(-lu * np.abs(dist))

        # kernel for kyy
        k1 = lambda dist:np.exp(-ly*np.abs(dist))*(2*lu+ly)/(lu+ly)**2
        k2 = lambda dist:(np.exp(-lu*dist)*(ly-2*lu+lu*ly*dist-lu**2*dist) + np.exp(-ly*dist)*(2*lu-ly) ) / (ly-lu)**2
        k3 = lambda dist:np.exp(-lu*dist) * ( (1+lu*dist)/(lu+ly) + (lu)/(lu+ly)**2 )
        kyy = lambda dist:Vu*Vy*(k1(dist) + k2(dist) + k3(dist))

        # cross covariance function
        kyu3 = lambda dist:np.exp(-lu*dist)/(lu+ly)*(1+lu*(dist+1/(lu+ly)))
        #kyu3 = lambda dist: 0

        k1cros = lambda dist:np.exp(ly*dist)/(lu-ly) * ( 1- np.exp( (lu-ly)*dist) + lu* ( dist*np.exp( (lu-ly)*dist ) + (1- np.exp( (lu-ly)*dist ) ) /(lu-ly)   )    )
        #k1cros = lambda dist:0

        k2cros = lambda dist:np.exp(ly*dist)*( 1/(lu+ly) + lu/(lu+ly)**2 )
        #k2cros = lambda dist:0

        Vyu=np.sqrt(Vy*ly*2)

        # cross covariance kuy
        kuyp = lambda dist:Vu*Vyu*(kyu3(dist))       #t>0 kuy
        kuyn = lambda dist:Vu*Vyu*(k1cros(dist)+k2cros(dist))      #t<0 kuy
        # cross covariance kyu
        kyup = lambda dist:Vu*Vyu*(k1cros(-dist)+k2cros(-dist))    #t>0 kyu
        kyun = lambda dist:Vu*Vyu*(kyu3(-dist))       #t<0 kyu

        for i, s1 in enumerate(slices):
            for j, s2 in enumerate(slices2):
                for ss1 in s1:
                    for ss2 in s2:
                        if i==0 and j==0:
                            K[ss1,ss2] = kuu(np.abs(rdist[ss1,ss2]))
                        elif i==0 and j==1:
                            #K[ss1,ss2]=  np.where(  rdist[ss1,ss2]>0 , kuyp(np.abs(rdist[ss1,ss2])), kuyn(np.abs(rdist[ss1,ss2]) )   )
                            K[ss1,ss2]=  np.where(  rdist[ss1,ss2]>0 , kuyp(rdist[ss1,ss2]), kuyn(rdist[ss1,ss2] )   )
                        elif i==1 and j==1:
                            K[ss1,ss2] = kyy(np.abs(rdist[ss1,ss2]))
                        else:
                            #K[ss1,ss2]= 0
                            #K[ss1,ss2]= np.where(  rdist[ss1,ss2]>0 , kyup(np.abs(rdist[ss1,ss2])), kyun(np.abs(rdist[ss1,ss2]) )   )
                            K[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , kyup(rdist[ss1,ss2]), kyun(rdist[ss1,ss2] )   )
        return K

    def Kdiag(self, X):
        """Compute the diagonal of the covariance matrix associated to X."""
        Kdiag = np.zeros(X.shape[0])

        Vu = self.variance_U

        k1 = (2*lu+ly)/(lu+ly)**2
        k2 = (ly-2*lu + 2*lu-ly ) / (ly-lu)**2
        k3 = 1/(lu+ly) + (lu)/(lu+ly)**2

        slices = index_to_slices(X[:,-1])

        for i, ss1 in enumerate(slices):
            for s1 in ss1:
                if i==0:
                    Kdiag[s1]+= self.variance_U
                elif i==1:
                    Kdiag[s1]+= Vu*Vy*(k1+k2+k3)
                else:
                    raise ValueError, "invalid input/output index"
        #Kdiag[slices[0][0]]+= self.variance_U   #matern32 diag
        #Kdiag[slices[1][0]]+= self.variance_U*self.variance_Y*(k1+k2+k3)  #  diag
        return Kdiag

    def update_gradients_full(self, dL_dK, X, X2=None):
        """derivative of the covariance matrix with respect to the parameters."""
        else:

        Vyu = np.sqrt(Vy*ly*2)
        dVdly = 0.5/np.sqrt(ly)*np.sqrt(2*Vy)
        dVdVy = 0.5/np.sqrt(Vy)*np.sqrt(2*ly)

        rd=rdist.shape
        dktheta1 = np.zeros(rd)
        dktheta2 = np.zeros(rd)
        dkUdvar = np.zeros(rd)
        dkYdvar = np.zeros(rd)

        # dk dtheta for UU
        UUdtheta1 = lambda dist: np.exp(-lu* dist)*dist + (-dist)*np.exp(-lu* dist)*(1+lu*dist)
        UUdtheta2 = lambda dist: 0
        #UUdvar = lambda dist: (1 + lu*dist)*np.exp(-lu*dist)
        UUdvar = lambda dist: (1 + lu* np.abs(dist)) * np.exp(-lu * np.abs(dist))

        # dk dtheta for YY

        dk1theta1 = lambda dist: np.exp(-ly*dist)*2*(-lu)/(lu+ly)**3

        dk2theta1 = lambda dist: (1.0)*(
            np.exp(-lu*dist)*dist*(-ly+2*lu-lu*ly*dist+dist*lu**2)*(ly-lu)**(-2) + np.exp(-lu*dist)*(-2+ly*dist-2*dist*lu)*(ly-lu)**(-2)
            +np.exp(-dist*lu)*(ly-2*lu+ly*lu*dist-dist*lu**2)*2*(ly-lu)**(-3)
            +np.exp(-dist*ly)*2*(ly-lu)**(-2)
            +np.exp(-dist*ly)*2*(2*lu-ly)*(ly-lu)**(-3)
            )

        dk3theta1 = lambda dist: np.exp(-dist*lu)*(lu+ly)**(-2)*((2*lu+ly+dist*lu**2+lu*ly*dist)*(-dist-2/(lu+ly))+2+2*lu*dist+ly*dist)

        #dktheta1 = lambda dist: self.variance_U*self.variance_Y*(dk1theta1+dk2theta1+dk3theta1)

        dk1theta2 = lambda dist: np.exp(-ly*dist) * ((lu+ly)**(-2)) * (  (-dist)*(2*lu+ly)  +  1  +  (-2)*(2*lu+ly)/(lu+ly)  )

        dk2theta2 =lambda dist:  1*(
            np.exp(-dist*lu)*(ly-lu)**(-2) * ( 1+lu*dist+(-2)*(ly-2*lu+lu*ly*dist-dist*lu**2)*(ly-lu)**(-1) )
            +np.exp(-dist*ly)*(ly-lu)**(-2) * ( (-dist)*(2*lu-ly) -1+(2*lu-ly)*(-2)*(ly-lu)**(-1) )
            )

        dk3theta2 = lambda dist: np.exp(-dist*lu) * (-3*lu-ly-dist*lu**2-lu*ly*dist)/(lu+ly)**3

        #dktheta2 = lambda dist: self.variance_U*self.variance_Y*(dk1theta2 + dk2theta2 +dk3theta2)

        # kyy kernel

        k1 = lambda dist: np.exp(-ly*dist)*(2*lu+ly)/(lu+ly)**2
        k2 = lambda dist: (np.exp(-lu*dist)*(ly-2*lu+lu*ly*dist-lu**2*dist) + np.exp(-ly*dist)*(2*lu-ly) ) / (ly-lu)**2
        k3 = lambda dist: np.exp(-lu*dist) * ( (1+lu*dist)/(lu+ly) + (lu)/(lu+ly)**2 )
        #dkdvar = k1+k2+k3

        kuyp = lambda dist:(kyu3(dist))       #t>0 kuy
        kuyn = lambda dist:(k1cros(dist)+k2cros(dist))      #t<0 kuy
        kyup = lambda dist:(k1cros(-dist)+k2cros(-dist))    #t>0 kyu
        kyun = lambda dist:(kyu3(-dist))       #t<0 kyu

        # dk dtheta for UY

        dkyu3dtheta2 = lambda dist: np.exp(-lu*dist) * ( (-1)*(lu+ly)**(-2)*(1+lu*dist+lu*(lu+ly)**(-1)) + (lu+ly)**(-1)*(-lu)*(lu+ly)**(-2) )
        dkyu3dtheta1 = lambda dist: np.exp(-lu*dist)*(lu+ly)**(-1)* ( (-dist)*(1+dist*lu+lu*(lu+ly)**(-1)) -\
         (lu+ly)**(-1)*(1+dist*lu+lu*(lu+ly)**(-1)) +dist+(lu+ly)**(-1)-lu*(lu+ly)**(-2) )

        dkcros2dtheta1 = lambda dist: np.exp(ly*dist)* ( -(ly+lu)**(-2) + (ly+lu)**(-2) + (-2)*lu*(lu+ly)**(-3)  )
        dkcros2dtheta2 = lambda dist: np.exp(ly*dist)*dist* ( (ly+lu)**(-1) + lu*(lu+ly)**(-2) ) + \
                                      np.exp(ly*dist)*( -(lu+ly)**(-2) + lu*(-2)*(lu+ly)**(-3)  )

        dkcros1dtheta1 = lambda dist: np.exp(ly*dist)*(     -(lu-ly)**(-2)*(  1-np.exp((lu-ly)*dist) + lu*dist*np.exp((lu-ly)*dist)+ \
          lu*(1-np.exp((lu-ly)*dist))/(lu-ly)  )  +  (lu-ly)**(-1)*(  -np.exp( (lu-ly)*dist )*dist + dist*np.exp( (lu-ly)*dist)+\
          lu*dist**2*np.exp((lu-ly)*dist)+(1-np.exp((lu-ly)*dist))/(lu-ly) - lu*np.exp((lu-ly)*dist)*dist/(lu-ly) -\
          lu*(1-np.exp((lu-ly)*dist))/(lu-ly)**2  )   )

        dkcros1dtheta2 = lambda t: np.exp(ly*t)*t/(lu-ly)*( 1-np.exp((lu-ly)*t) +lu*t*np.exp((lu-ly)*t)+\
            lu*(1-np.exp((lu-ly)*t))/(lu-ly)  )+\
            np.exp(ly*t)/(lu-ly)**2* ( 1-np.exp((lu-ly)*t) +lu*t*np.exp((lu-ly)*t) + lu*( 1-np.exp((lu-ly)*t) )/(lu-ly)  )+\
            np.exp(ly*t)/(lu-ly)*( np.exp((lu-ly)*t)*t -lu*t*t*np.exp((lu-ly)*t) +lu*t*np.exp((lu-ly)*t)/(lu-ly)+\
            lu*( 1-np.exp((lu-ly)*t) )/(lu-ly)**2 )

        dkuypdtheta1 = lambda dist:(dkyu3dtheta1(dist))       #t>0 kuy
        dkuyndtheta1 = lambda dist:(dkcros1dtheta1(dist)+dkcros2dtheta1(dist))      #t<0 kuy
        dkyupdtheta1 = lambda dist:(dkcros1dtheta1(-dist)+dkcros2dtheta1(-dist))    #t>0 kyu
        dkyundtheta1 = lambda dist:(dkyu3dtheta1(-dist))       #t<0 kyu

        dkuypdtheta2 = lambda dist:(dkyu3dtheta2(dist))       #t>0 kuy
        dkuyndtheta2 = lambda dist:(dkcros1dtheta2(dist)+dkcros2dtheta2(dist))      #t<0 kuy
        dkyupdtheta2 = lambda dist:(dkcros1dtheta2(-dist)+dkcros2dtheta2(-dist))    #t>0 kyu
        dkyundtheta2 = lambda dist:(dkyu3dtheta2(-dist))       #t<0 kyu

                            #target[ss1,ss2] = kuu(np.abs(rdist[ss1,ss2]))
                            dktheta1[ss1,ss2] = Vu*UUdtheta1(np.abs(rdist[ss1,ss2]))
                            dktheta2[ss1,ss2] = 0
                            dkUdvar[ss1,ss2] = UUdvar(np.abs(rdist[ss1,ss2]))
                            dkYdvar[ss1,ss2] = 0
                            ########target[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , kuyp(np.abs(rdist[ss1,ss2])), kuyn(np.abs(rdist[s1[0],s2[0]]) )   )
                            #np.where(  rdist[ss1,ss2]>0 , kuyp(np.abs(rdist[ss1,ss2])), kuyn(np.abs(rdist[s1[0],s2[0]]) )   )
                            #dktheta1[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , self.variance_U*self.variance_Y*dkcrtheta1(np.abs(rdist[ss1,ss2])) ,self.variance_U*self.variance_Y*(dk1theta1(np.abs(rdist[ss1,ss2]))+dk2theta1(np.abs(rdist[ss1,ss2])))    )
                            #dktheta2[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , self.variance_U*self.variance_Y*dkcrtheta2(np.abs(rdist[ss1,ss2])) ,self.variance_U*self.variance_Y*(dk1theta2(np.abs(rdist[ss1,ss2]))+dk2theta2(np.abs(rdist[ss1,ss2])))    )
                            dktheta1[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , Vu*Vyu*dkuypdtheta1(rdist[ss1,ss2]),Vu*Vyu*dkuyndtheta1(rdist[ss1,ss2]) )
                            dkUdvar[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , Vyu*kuyp(rdist[ss1,ss2]), Vyu* kuyn(rdist[ss1,ss2])  )
                            dktheta2[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , Vu*Vyu*dkuypdtheta2(rdist[ss1,ss2])+Vu*dVdly*kuyp(rdist[ss1,ss2]),Vu*Vyu*dkuyndtheta2(rdist[ss1,ss2])+Vu*dVdly*kuyn(rdist[ss1,ss2]) )
                            dkYdvar[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , Vu*dVdVy*kuyp(rdist[ss1,ss2]), Vu*dVdVy* kuyn(rdist[ss1,ss2])  )
                            #target[ss1,ss2] = kyy(np.abs(rdist[ss1,ss2]))
                            dktheta1[ss1,ss2] = self.variance_U*self.variance_Y*(dk1theta1(np.abs(rdist[ss1,ss2]))+dk2theta1(np.abs(rdist[ss1,ss2]))+dk3theta1(np.abs(rdist[ss1,ss2])))
                            dktheta2[ss1,ss2] = self.variance_U*self.variance_Y*(dk1theta2(np.abs(rdist[ss1,ss2])) + dk2theta2(np.abs(rdist[ss1,ss2])) +dk3theta2(np.abs(rdist[ss1,ss2])))
                            dkUdvar[ss1,ss2] = self.variance_Y*(k1(np.abs(rdist[ss1,ss2]))+k2(np.abs(rdist[ss1,ss2]))+k3(np.abs(rdist[ss1,ss2])) )
                            dkYdvar[ss1,ss2] = self.variance_U*(k1(np.abs(rdist[ss1,ss2]))+k2(np.abs(rdist[ss1,ss2]))+k3(np.abs(rdist[ss1,ss2])) )
                        else:
                            #######target[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , kyup(np.abs(rdist[ss1,ss2])), kyun(np.abs(rdist[s1[0],s2[0]]) )   )
                            #dktheta1[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 ,self.variance_U*self.variance_Y*(dk1theta1(np.abs(rdist[ss1,ss2]))+dk2theta1(np.abs(rdist[ss1,ss2]))) , self.variance_U*self.variance_Y*dkcrtheta1(np.abs(rdist[ss1,ss2])) )
                            #dktheta2[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 ,self.variance_U*self.variance_Y*(dk1theta2(np.abs(rdist[ss1,ss2]))+dk2theta2(np.abs(rdist[ss1,ss2]))) , self.variance_U*self.variance_Y*dkcrtheta2(np.abs(rdist[ss1,ss2])) )
                            dktheta1[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , Vu*Vyu*dkyupdtheta1(rdist[ss1,ss2]),Vu*Vyu*dkyundtheta1(rdist[ss1,ss2])  )
                            dkUdvar[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , Vyu*kyup(rdist[ss1,ss2]),Vyu*kyun(rdist[ss1,ss2]))
                            dktheta2[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , Vu*Vyu*dkyupdtheta2(rdist[ss1,ss2])+Vu*dVdly*kyup(rdist[ss1,ss2]),Vu*Vyu*dkyundtheta2(rdist[ss1,ss2])+Vu*dVdly*kyun(rdist[ss1,ss2])  )
                            dkYdvar[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , Vu*dVdVy*kyup(rdist[ss1,ss2]), Vu*dVdVy*kyun(rdist[ss1,ss2]))

        #stop
        self.variance_U.gradient = np.sum(dkUdvar * dL_dK)     # Vu

        self.variance_Y.gradient = np.sum(dkYdvar * dL_dK)     # Vy

        self.lengthscale_U.gradient = np.sum(dktheta1*(-np.sqrt(3)*self.lengthscale_U**(-2))* dL_dK)     #lu

        self.lengthscale_Y.gradient = np.sum(dktheta2*(-self.lengthscale_Y**(-2)) * dL_dK)              #ly

# Copyright (c) 2014, James Hensman, Alan Saul
# Distributed under the terms of the GNU General public License, see LICENSE.txt

from ..core.model import Model
from ..core.parameterization import ObsAr

class GPKroneckerGaussianRegression(Model):
    """
    Kronecker GP regression

    Take two kernels computed on separate spaces K1(X1), K2(X2), and a data
    matrix Y which is f size (N1, N2).

    The effective covaraince is np.kron(K2, K1)
    The effective data is vec(Y) = Y.flatten(order='F')

    The noise must be iid Gaussian.

    See Stegle et al.
    @inproceedings{stegle2011efficient,
      title={Efficient inference in matrix-variate gaussian models with $\\backslash$ iid observation noise},
      author={Stegle, Oliver and Lippert, Christoph and Mooij, Joris M and Lawrence, Neil D and Borgwardt, Karsten M},
      booktitle={Advances in Neural Information Processing Systems},
      pages={630--638},
      year={2011}
    }

    """
    def __init__(self, X1, X2, Y, kern1, kern2, noise_var=1., name='KGPR'):
        Model.__init__(self, name=name)
        # accept the construction arguments
        self.X1 = ObsAr(X1)
        self.X2 = ObsAr(X2)
        self.Y = Y
        self.kern1, self.kern2 = kern1, kern2
        self.link_parameter(self.kern1)
        self.link_parameter(self.kern2)

        self.likelihood = likelihoods.Gaussian()
        self.likelihood.variance = noise_var
        self.link_parameter(self.likelihood)

        self.num_data1, self.input_dim1 = self.X1.shape
        self.num_data2, self.input_dim2 = self.X2.shape

        assert kern1.input_dim == self.input_dim1
        assert kern2.input_dim == self.input_dim2
        assert Y.shape == (self.num_data1, self.num_data2)

    def log_likelihood(self):
        return self._log_marginal_likelihood

        (N1, D1), (N2, D2) = self.X1.shape, self.X2.shape
        K1, K2 = self.kern1.K(self.X1), self.kern2.K(self.X2)

        # eigendecompositon
        S1, U1 = np.linalg.eigh(K1)
        S2, U2 = np.linalg.eigh(K2)
        W = np.kron(S2, S1) + self.likelihood.variance

        Y_ = U1.T.dot(self.Y).dot(U2)

        # store these quantities: needed for prediction
        Wi = 1./W
        Ytilde = Y_.flatten(order='F')*Wi

        self._log_marginal_likelihood = -0.5*self.num_data1*self.num_data2*np.log(2*np.pi)\
                                        -0.5*np.sum(np.log(W))\
                                        -0.5*np.dot(Y_.flatten(order='F'), Ytilde)

        # gradients for data fit part
        Yt_reshaped = Ytilde.reshape(N1, N2, order='F')
        tmp = U1.dot(Yt_reshaped)
        dL_dK1 = .5*(tmp*S2).dot(tmp.T)
        tmp = U2.dot(Yt_reshaped.T)
        dL_dK2 = .5*(tmp*S1).dot(tmp.T)

        # gradients for logdet
        Wi_reshaped = Wi.reshape(N1, N2, order='F')
        tmp = np.dot(Wi_reshaped, S2)
        dL_dK1 += -0.5*(U1*tmp).dot(U1.T)
        tmp = np.dot(Wi_reshaped.T, S1)
        dL_dK2 += -0.5*(U2*tmp).dot(U2.T)

        self.kern1.update_gradients_full(dL_dK1, self.X1)
        self.kern2.update_gradients_full(dL_dK2, self.X2)

        # gradients for noise variance
        dL_dsigma2 = -0.5*Wi.sum() + 0.5*np.sum(np.square(Ytilde))
        self.likelihood.variance.gradient = dL_dsigma2

        # store these quantities for prediction:
        self.Wi, self.Ytilde, self.U1, self.U2 = Wi, Ytilde, U1, U2

    def predict(self, X1new, X2new):
        """
        Return the predictive mean and variance at a series of new points X1new, X2new
        Only returns the diagonal of the predictive variance, for now.

        :param X1new: The points at which to make a prediction
        :type X1new: np.ndarray, Nnew x self.input_dim1
        :param X2new: The points at which to make a prediction
        :type X2new: np.ndarray, Nnew x self.input_dim2

        """
        k1xf = self.kern1.K(X1new, self.X1)
        k2xf = self.kern2.K(X2new, self.X2)
        A = k1xf.dot(self.U1)
        B = k2xf.dot(self.U2)
        mu = A.dot(self.Ytilde.reshape(self.num_data1, self.num_data2, order='F')).dot(B.T).flatten(order='F')
        k1xx = self.kern1.Kdiag(X1new)
        k2xx = self.kern2.Kdiag(X2new)
        BA = np.kron(B, A)
        var = np.kron(k2xx, k1xx) - np.sum(BA**2*self.Wi, 1) + self.likelihood.variance

        return mu[:, None], var[:, None]

from ..core.mapping import Mapping

class MLP(Mapping):
    """
    Mapping based on a multi-layer perceptron neural network model.

    .. math::

       f(\\mathbf{x}*) = \\mathbf{W}^0\\boldsymbol{\\phi}(\\mathbf{W}^1\\mathbf{x}+\\mathbf{b}^1)^* + \\mathbf{b}^0

    where

    .. math::

      \\phi(\\cdot) = \\text{tanh}(\\cdot)

    :type X: ndarray
    :param output_dim: dimension of output.
    :type output_dim: int
    :param hidden_dim: dimension of hidden layer. If it is an int, there is one hidden layer of the given dimension. If it is a list of ints there are as manny hidden layers as the length of the list, each with the given number of hidden nodes in it.
    :type hidden_dim: int or list of ints. 

    """

    def __init__(self, input_dim=1, output_dim=1, hidden_dim=3):
        Mapping.__init__(self, input_dim=input_dim, output_dim=output_dim)
        self.name = 'mlp'
        if isinstance(hidden_dim, int):
            hidden_dim = [hidden_dim]
        self.hidden_dim = hidden_dim
        self.activation = [None]*len(self.hidden_dim)
        self.W = []
        self._dL_dW = []
        self.bias = []
        self._dL_dbias = []
        self.W.append(np.zeros((self.input_dim, self.hidden_dim[0])))
        self._dL_dW.append(np.zeros((self.input_dim, self.hidden_dim[0])))
        self.bias.append(np.zeros(self.hidden_dim[0]))
        self._dL_dbias.append(np.zeros(self.hidden_dim[0]))
        self.num_params = self.hidden_dim[0]*(self.input_dim+1)
        for h1, h0 in zip(hidden_dim[1:], hidden_dim[0:-1]):
            self.W.append(np.zeros((h0, h1)))
            self._dL_dW.append(np.zeros((h0, h1)))
            self.bias.append(np.zeros(h1))
            self._dL_dbias.append(np.zeros(h1))
            self.num_params += h1*(h0+1)
        self.W.append(np.zeros((self.hidden_dim[-1], self.output_dim)))
        self._dL_dW.append(np.zeros((self.hidden_dim[-1], self.output_dim)))
        self.bias.append(np.zeros(self.output_dim))
        self._dL_dbias.append(np.zeros(self.output_dim))
        self.num_params += self.output_dim*(self.hidden_dim[-1]+1)
        self.randomize()

    def _get_param_names(self):
        return sum([['W%i_%i_%i' % (i, n, d)  for n in range(self.W[i].shape[0]) for d in range(self.W[i].shape[1])] + ['bias%i_%i' % (i, d) for d in range(self.W[i].shape[1])] for i in range(len(self.W))], [])

    def _get_params(self):
        param = np.array([])
        for W, bias in zip(self.W, self.bias):
            param = np.hstack((param, W.flatten(), bias))
        return param
    
    def _set_params(self, x):
        start = 0
            end = W.shape[0]*W.shape[1]+start
            W[:] = x[start:end].reshape(W.shape[0], W.shape[1]).copy()
            start = end
            end = W.shape[1]+end
            bias[:] = x[start:end].copy()
            start = end

    def randomize(self):
            W[:] = np.random.randn(W.shape[0], W.shape[1])/np.sqrt(W.shape[0]+1)
            bias[:] = np.random.randn(W.shape[1])/np.sqrt(W.shape[0]+1)

    def f(self, X):
        self._f_computations(X)
        return np.dot(np.tanh(self.activation[-1]), self.W[-1]) + self.bias[-1]

    def _f_computations(self, X):
        W = self.W[0]
        bias = self.bias[0]
        self.activation[0] = np.dot(X,W) + bias
        for W, bias, index in zip(self.W[1:-1], self.bias[1:-1], range(1, len(self.activation))):
            self.activation[index] = np.dot(np.tanh(self.activation[index-1]), W)+bias

    def df_dtheta(self, dL_df, X):
        self._df_computations(dL_df, X)
        g = np.array([])
        for gW, gbias in zip(self._dL_dW, self._dL_dbias):
            g = np.hstack((g, gW.flatten(), gbias))
        return g

    def _df_computations(self, dL_df, X):
        a0 = self.activation[-1]
        W = self.W[-1]
        self._dL_dW[-1] = (dL_df[:, :, None]*np.tanh(a0[:, None, :])).sum(0).T
        dL_dta=(dL_df[:, None, :]*W[None, :, :]).sum(2)
        self._dL_dbias[-1] = (dL_df.sum(0))
        for dL_dW, dL_dbias, W, bias, a0, a1 in zip(self._dL_dW[-2:0:-1],
                                                    self._dL_dbias[-2:0:-1],
                                                    self.W[-2:0:-1],
                                                    self.bias[-2:0:-1],
                                                    self.activation[-2::-1],
                                                    self.activation[-1:0:-1]):
            ta = np.tanh(a1)
            dL_da = dL_dta*(1-ta*ta)
            dL_dW[:] = (dL_da[:, :, None]*np.tanh(a0[:, None, :])).sum(0).T
            dL_dbias[:] = (dL_da.sum(0))
            dL_dta = (dL_da[:, None, :]*W[None, :, :]).sum(2)
        ta = np.tanh(self.activation[0])
        dL_da = dL_dta*(1-ta*ta)
        self._dL_dW[0] = (dL_da[:, :, None]*X[:, None, :]).sum(0).T
        self._dL_dbias[0] = (dL_da.sum(0))
        self._dL_dX = (dL_da[:, None, :]*W[None, :, :]).sum(2)

    def df_dX(self, dL_df, X):
        return self._dL_dX
    
from scipy.special import erf, erfc, erfcx
import sys
epsilon = sys.float_info.epsilon
lim_val = -np.log(epsilon) 

def logisticln(x):
    return np.where(x<lim_val, np.where(x>-lim_val, -np.log(1+np.exp(-x)), -x), -np.log(1+epsilon))

def logistic(x):
    return np.where(x<lim_val, np.where(x>-lim_val, 1/(1+np.exp(-x)), epsilon/(epsilon+1)), 1/(1+epsilon))

def normcdf(x):
    g=0.5*erfc(-x/np.sqrt(2))
    return np.where(g==0, epsilon, np.where(g==1, 1-epsilon, g)) 

def normcdfln(x):
    return np.where(x < 0, -.5*x*x + np.log(.5) + np.log(erfcx(-x/np.sqrt(2))), np.log(normcdf(x)))

def clip_exp(x):
    return np.where(x<lim_val, np.where(x>-lim_val, np.exp(x), epsilon), 1/epsilon)

def differfln(x0, x1):
    # this is a, hopefully!, a numerically more stable variant of log(erf(x0)-erf(x1)) = log(erfc(x1)-erfc(x0)).
    return np.where(x0>x1, -x1*x1 + np.log(erfcx(x1)-np.exp(-x0**2+x1**2)*erfcx(x0)), -x0*x0 + np.log(np.exp(-x1**2+x0**2)*erfcx(x1) - erfcx(x0)))

class Brownian(Kern):
    """
    Brownian motion in 1D only.

    Negative times are treated as a separate (backwards!) Brownian motion.

    :param input_dim: the number of input dimensions
    :type input_dim: int
    :param variance:
    :type variance: float
    """
    def __init__(self, input_dim=1, variance=1., active_dims=None, name='Brownian'):
        assert input_dim==1, "Brownian motion in 1D only"
        super(Brownian, self).__init__(input_dim, active_dims, name)

        self.variance = Param('variance', variance, Logexp())
        self.link_parameters(self.variance)

    def K(self,X,X2=None):
            X2 = X
        return self.variance*np.where(np.sign(X)==np.sign(X2.T),np.fmin(np.abs(X),np.abs(X2.T)), 0.)

    def Kdiag(self,X):
        return self.variance*np.abs(X.flatten())

            X2 = X
        self.variance.gradient = np.sum(dL_dK * np.where(np.sign(X)==np.sign(X2.T),np.fmin(np.abs(X),np.abs(X2.T)), 0.))

    #def update_gradients_diag(self, dL_dKdiag, X):
        #self.variance.gradient = np.dot(np.abs(X.flatten()), dL_dKdiag)

    #def gradients_X(self, dL_dK, X, X2=None):
        #if X2 is None:
            #return np.sum(self.variance*dL_dK*np.abs(X),1)[:,None]
        #else:
            #return np.sum(np.where(np.logical_and(np.abs(X)<np.abs(X2.T), np.sign(X)==np.sign(X2)), self.variance*dL_dK,0.),1)[:,None]

# Copyright (c) 2012-2014, Max Zwiessele

import numpy

class GDUpdateRule():
    _gradnat = None
    _gradnatold = None
    def __init__(self, initgrad, initgradnat=None):
        self.grad = initgrad
        if initgradnat:
            self.gradnat = initgradnat
        else:
            self.gradnat = initgrad
        # self.grad, self.gradnat
    def _gamma(self):
        raise NotImplemented("""Implement gamma update rule here, 
        you can use self.grad and self.gradold for parameters, as well as
        self.gradnat and self.gradnatold for natural gradients.""")
    def __call__(self, grad, gradnat=None, si=None, *args, **kw):
        """
        Return gamma for given gradients and optional natural gradients
        """
        if not gradnat:
            gradnat = grad
        self.gradold = self.grad
        self.gradnatold = self.gradnat
        self.grad = grad
        self.gradnat = gradnat
        self.si = si
        return self._gamma(*args, **kw)

class FletcherReeves(GDUpdateRule):
    '''
    Fletcher Reeves update rule for gamma
    '''
    def _gamma(self, *a, **kw):
        tmp = numpy.dot(self.grad.T, self.gradnat)
        if tmp:
            return tmp / numpy.dot(self.gradold.T, self.gradnatold)
        return tmp

class PolakRibiere(GDUpdateRule):
    '''
    '''
        tmp = numpy.dot((self.grad - self.gradold).T, self.gradnat)
        if tmp:
        return tmp

from scipy import stats
from scipy.special import erf
from ..core.parameterization.param import Param
from ..util.linalg import pdinv

log_2_pi = np.log(2*np.pi)

class GPVariationalGaussianApproximation(Model):
    """
    The Variational Gaussian Approximation revisited implementation for regression

    @article{Opper:2009,
        title = {The Variational Gaussian Approximation Revisited},
        author = {Opper, Manfred and Archambeau, C{\'e}dric},
        journal = {Neural Comput.},
        year = {2009},
        pages = {786--792},
    }
    """
    def __init__(self, X, Y, kernel=None):
        Model.__init__(self,'Variational GP classification')
        self.X = ObsAr(X)
            kernel = kern.RBF(X.shape[1]) + kern.White(X.shape[1], 0.01)
        self.kern = kernel
        self.link_parameter(self.kern)
        self.num_data, self.input_dim = self.X.shape

        self.alpha = Param('alpha', np.zeros(self.num_data))
        self.beta = Param('beta', np.ones(self.num_data))
        self.link_parameter(self.alpha)
        self.link_parameter(self.beta)

        self.gh_x, self.gh_w = np.polynomial.hermite.hermgauss(20)
        self.Ysign = np.where(Y==1, 1, -1).flatten()

        """
        Marginal log likelihood evaluation
        """
        return self._log_lik

    def likelihood_quadrature(self, m, v):
        """
        Perform Gauss-Hermite quadrature over the log of the likelihood, with a fixed weight
        """
        # assume probit for now.
        X = self.gh_x[None, :]*np.sqrt(2.*v[:, None]) + (m*self.Ysign)[:, None]
        p = stats.norm.cdf(X)
        N = stats.norm.pdf(X)
        F = np.log(p).dot(self.gh_w)
        NoverP = N/p
        dF_dm = (NoverP*self.Ysign[:,None]).dot(self.gh_w)
        dF_dv = -0.5*(NoverP**2 + NoverP*X).dot(self.gh_w)
        return F, dF_dm, dF_dv

        K = self.kern.K(self.X)
        m = K.dot(self.alpha)
        KB = K*self.beta[:, None]
        BKB = KB*self.beta[None, :]
        A = np.eye(self.num_data) + BKB
        Ai, LA, _, Alogdet = pdinv(A)
        Sigma = np.diag(self.beta**-2) - Ai/self.beta[:, None]/self.beta[None, :]  # posterior coavairance: need full matrix for gradients
        var = np.diag(Sigma)

        F, dF_dm, dF_dv = self.likelihood_quadrature(m, var)
        dF_da = np.dot(K, dF_dm)
        SigmaB = Sigma*self.beta
        dF_db = -np.diag(Sigma.dot(np.diag(dF_dv)).dot(SigmaB))*2
        KL = 0.5*(Alogdet + np.trace(Ai) - self.num_data + m.dot(self.alpha))
        dKL_da = m
        A_A2 = Ai - Ai.dot(Ai)
        dKL_db = np.diag(np.dot(KB.T, A_A2))
        self._log_lik = F.sum() - KL
        self.alpha.gradient = dF_da - dKL_da
        self.beta.gradient = dF_db - dKL_db

        # K-gradients
        dKL_dK = 0.5*(self.alpha[None, :]*self.alpha[:, None] + self.beta[:, None]*self.beta[None, :]*A_A2)
        tmp = Ai*self.beta[:, None]/self.beta[None, :]
        dF_dK = self.alpha[:, None]*dF_dm[None, :] + np.dot(tmp*dF_dv, tmp.T)
        self.kern.update_gradients_full(dF_dK - dKL_dK, self.X)

    def predict(self, Xnew):
        """
        Predict the function(s) at the new point(s) Xnew.

        :param Xnew: The points at which to make a prediction
        :type Xnew: np.ndarray, Nnew x self.input_dim
        """
        Wi, _, _, _ = pdinv(self.kern.K(self.X) + np.diag(self.beta**-2))
        Kux = self.kern.K(self.X, Xnew)
        mu = np.dot(Kux.T, self.alpha)
        WiKux = np.dot(Wi, Kux)
        Kxx = self.kern.Kdiag(Xnew)
        var = Kxx - np.sum(WiKux*Kux, 0)

        return 0.5*(1+erf(mu/np.sqrt(2.*(var+1))))

"""
A new kernel
"""

from kern import Kern,CombinationKernel
from .independent_outputs import index_to_slices
import itertools

class DiffGenomeKern(Kern):

    def __init__(self, kernel, idx_p, Xp, index_dim=-1, name='DiffGenomeKern'):
        self.idx_p = idx_p
        self.index_dim=index_dim
        self.kern = SplitKern(kernel,Xp, index_dim=index_dim)
        super(DiffGenomeKern, self).__init__(input_dim=kernel.input_dim+1, active_dims=None, name=name)
        self.add_parameter(self.kern)
    
        assert X2==None
        K = self.kern.K(X,X2)
        
        if self.idx_p<=0 or self.idx_p>X.shape[0]/2:
            return K
        
        slices = index_to_slices(X[:,self.index_dim])
        idx_start = slices[1][0].start
        idx_end = idx_start+self.idx_p
        K_c = K[idx_start:idx_end,idx_start:idx_end].copy()
        K[idx_start:idx_end,:] = K[:self.idx_p,:]
        K[:,idx_start:idx_end] = K[:,:self.idx_p]
        K[idx_start:idx_end,idx_start:idx_end] = K_c
        
        return K
    
        Kdiag = self.kern.Kdiag(X)

            return Kdiag

        Kdiag[idx_start:idx_end] = Kdiag[:self.idx_p]
        
    def update_gradients_full(self,dL_dK,X,X2=None):
            self.kern.update_gradients_full(dL_dK, X)
            return
        
        self.kern.update_gradients_full(dL_dK[idx_start:idx_end,:], X[:self.idx_p],X)
        grad_p1 = self.kern.gradient.copy()
        self.kern.update_gradients_full(dL_dK[:,idx_start:idx_end], X, X[:self.idx_p])
        grad_p2 = self.kern.gradient.copy()
        self.kern.update_gradients_full(dL_dK[idx_start:idx_end,idx_start:idx_end], X[:self.idx_p],X[idx_start:idx_end])
        grad_p3 = self.kern.gradient.copy()
        self.kern.update_gradients_full(dL_dK[idx_start:idx_end,idx_start:idx_end], X[idx_start:idx_end], X[:self.idx_p])
        grad_p4 = self.kern.gradient.copy()

        self.kern.update_gradients_full(dL_dK[idx_start:idx_end,:], X[idx_start:idx_end],X)
        grad_n1 = self.kern.gradient.copy()
        self.kern.update_gradients_full(dL_dK[:,idx_start:idx_end], X, X[idx_start:idx_end])
        grad_n2 = self.kern.gradient.copy()
        self.kern.update_gradients_full(dL_dK[idx_start:idx_end,idx_start:idx_end], X[idx_start:idx_end], X[idx_start:idx_end])
        grad_n3 = self.kern.gradient.copy()

        self.kern.update_gradients_full(dL_dK, X)
        self.kern.gradient += grad_p1+grad_p2-grad_p3-grad_p4-grad_n1-grad_n2+2*grad_n3

    def update_gradients_diag(self, dL_dKdiag, X):
        pass

class SplitKern(CombinationKernel):

    def __init__(self, kernel, Xp, index_dim=-1, name='SplitKern'):
        assert isinstance(index_dim, int), "The index dimension must be an integer!"
        self.kern_cross = SplitKern_cross(kernel,Xp)
        super(SplitKern, self).__init__(kernels=[self.kern, self.kern_cross], extra_dims=[index_dim], name=name)
        self.index_dim = index_dim

    def K(self,X ,X2=None):
        assert len(slices)<=2, 'The Split kernel only support two different indices'
            target = np.zeros((X.shape[0], X.shape[0]))
            # diagonal blocks
            [[target.__setitem__((s,ss), self.kern.K(X[s,:], X[ss,:])) for s,ss in itertools.product(slices_i, slices_i)] for slices_i in slices]
            if len(slices)>1:
                # cross blocks
                [target.__setitem__((s,ss), self.kern_cross.K(X[s,:], X[ss,:])) for s,ss in itertools.product(slices[0], slices[1])]
                [target.__setitem__((s,ss), self.kern_cross.K(X[s,:], X[ss,:])) for s,ss in itertools.product(slices[1], slices[0])]
        else:
            slices2 = index_to_slices(X2[:,self.index_dim])
            assert len(slices2)<=2, 'The Split kernel only support two different indices'
            target = np.zeros((X.shape[0], X2.shape[0]))
            [[target.__setitem__((s,s2), self.kern.K(X[s,:],X2[s2,:])) for s,s2 in itertools.product(slices[i], slices2[i])] for i in xrange(min(len(slices),len(slices2)))]
                [target.__setitem__((s,s2), self.kern_cross.K(X[s,:],X2[s2,:])) for s,s2 in itertools.product(slices[1], slices2[0])]
            if len(slices2)>1:
                [target.__setitem__((s,s2), self.kern_cross.K(X[s,:],X2[s2,:])) for s,s2 in itertools.product(slices[0], slices2[1])]                
        return target

        return self.kern.Kdiag(X)

        target = np.zeros(self.kern.size)

        def collate_grads(dL, X, X2, cross=False):
            if cross:
                self.kern_cross.update_gradients_full(dL,X,X2)
                target[:] += self.kern_cross.kern.gradient
            else:
                self.kern.update_gradients_full(dL,X,X2)
                target[:] += self.kern.gradient
    
            assert dL_dK.shape==(X.shape[0],X.shape[0])
            [[collate_grads(dL_dK[s,ss], X[s], X[ss]) for s,ss in itertools.product(slices_i, slices_i)] for slices_i in slices]
                [collate_grads(dL_dK[s,ss], X[s], X[ss], True) for s,ss in itertools.product(slices[0], slices[1])]
                [collate_grads(dL_dK[s,ss], X[s], X[ss], True) for s,ss in itertools.product(slices[1], slices[0])]
        else:
            assert dL_dK.shape==(X.shape[0],X2.shape[0])
            [[collate_grads(dL_dK[s,s2],X[s],X2[s2]) for s,s2 in itertools.product(slices[i], slices2[i])] for i in xrange(min(len(slices),len(slices2)))]
                [collate_grads(dL_dK[s,s2], X[s], X2[s2], True) for s,s2 in itertools.product(slices[1], slices2[0])]
                [collate_grads(dL_dK[s,s2], X[s], X2[s2], True) for s,s2 in itertools.product(slices[0], slices2[1])]
        self.kern.gradient = target

        self.kern.update_gradients_diag(self, dL_dKdiag, X)

class SplitKern_cross(Kern):

    def __init__(self, kernel, Xp, name='SplitKern_cross'):
        assert isinstance(kernel, Kern)
        if not isinstance(Xp,np.ndarray):
            Xp = np.array([[Xp]])
        self.Xp = Xp
        super(SplitKern_cross, self).__init__(input_dim=kernel.input_dim, active_dims=None, name=name)
        
            return np.dot(self.kern.K(X,self.Xp),self.kern.K(self.Xp,X))/self.kern.K(self.Xp,self.Xp)
        else:
            return np.dot(self.kern.K(X,self.Xp),self.kern.K(self.Xp,X2))/self.kern.K(self.Xp,self.Xp)
        
        return np.inner(self.kern.K(X,self.Xp),self.kern.K(self.Xp,X).T)/self.kern.K(self.Xp,self.Xp)

            X2 = X
                        
        k1 = self.kern.K(X,self.Xp)
        k2 = self.kern.K(self.Xp,X2)
        k3 = self.kern.K(self.Xp,self.Xp)
        dL_dk1 = np.einsum('ij,j->i',dL_dK,k2[0])/k3[0,0]
        dL_dk2 = np.einsum('ij,i->j',dL_dK,k1[:,0])/k3[0,0]
        dL_dk3 = np.einsum('ij,ij->',dL_dK,-np.dot(k1,k2)/(k3[0,0]*k3[0,0]))

        self.kern.update_gradients_full(dL_dk1[:,None],X,self.Xp)
        grad = self.kern.gradient.copy()
        self.kern.update_gradients_full(dL_dk2[None,:],self.Xp,X2)
        grad += self.kern.gradient.copy()
        self.kern.update_gradients_full(np.array([[dL_dk3]]),self.Xp,self.Xp)
        
        self.kern.gradient = grad

        k2 = self.kern.K(self.Xp,X)
        dL_dk1 = dL_dKdiag*k2[0]/k3
        dL_dk2 = dL_dKdiag*k1[:,0]/k3
        dL_dk3 = -dL_dKdiag*(k1[:,0]*k2[0]).sum()/(k3*k3)
        
        grad1 = self.kern.gradient.copy()
        self.kern.update_gradients_full(dL_dk2[None,:],self.Xp,X)
        grad2 = self.kern.gradient.copy()
        grad3 = self.kern.gradient.copy()
        
        self.kern.gradient = grad1+grad2+grad3
        
import pylab as pb, numpy as np

def plot(parameterized, fignum=None, ax=None, colors=None):
    """
    Plot latent space X in 1D:

        - if fig is given, create input_dim subplots in fig and plot in these
        - if ax is given plot input_dim 1D latent space plots of X into each `axis`
        - if neither fig nor ax is given create a figure with fignum and plot in there

    colors:
        colors of different latent space dimensions input_dim

    """
    if ax is None:
        fig = pb.figure(num=fignum, figsize=(8, min(12, (2 * parameterized.mean.shape[1]))))
    if colors is None:
        colors = pb.gca()._get_lines.color_cycle
        pb.clf()
    else:
        colors = iter(colors)
    plots = []
    means, variances = parameterized.mean, parameterized.variance
    x = np.arange(means.shape[0])
    for i in range(means.shape[1]):
        if ax is None:
            a = fig.add_subplot(means.shape[1], 1, i + 1)
        elif isinstance(ax, (tuple, list)):
            a = ax[i]
        else:
            raise ValueError("Need one ax per latent dimension input_dim")
        a.plot(means, c='k', alpha=.3)
        plots.extend(a.plot(x, means.T[i], c=colors.next(), label=r"$\mathbf{{X_{{{}}}}}$".format(i)))
        a.fill_between(x,
                        means.T[i] - 2 * np.sqrt(variances.T[i]),
                        means.T[i] + 2 * np.sqrt(variances.T[i]),
                        facecolor=plots[-1].get_color(),
                        alpha=.3)
        a.legend(borderaxespad=0.)
        a.set_xlim(x.min(), x.max())
        if i < means.shape[1] - 1:
            a.set_xticklabels('')
    pb.draw()
    fig.tight_layout(h_pad=.01) # , rect=(0, 0, 1, .95))
    return fig

def plot_SpikeSlab(parameterized, fignum=None, ax=None, colors=None, side_by_side=True):
    """

    colors:

    """
        if side_by_side:
            fig = pb.figure(num=fignum, figsize=(16, min(12, (2 * parameterized.mean.shape[1]))))
        else:
            fig = pb.figure(num=fignum, figsize=(8, min(12, (2 * parameterized.mean.shape[1]))))
        pb.clf()
    else:
    plots = []
    means, variances, gamma = parameterized.mean, parameterized.variance, parameterized.binary_prob
            sub1 = (means.shape[1],2,2*i+1)
            sub2 = (means.shape[1],2,2*i+2)
        else:
            sub1 = (means.shape[1]*2,1,2*i+1)
            sub2 = (means.shape[1]*2,1,2*i+2)

        # mean and variance plot
        a = fig.add_subplot(*sub1)
                        alpha=.3)
        # binary prob plot
        a = fig.add_subplot(*sub2)
        a.bar(x,gamma[:,i],bottom=0.,linewidth=0,width=1.0,align='center')
        a.set_ylim([0.,1.])
    pb.draw()
    return fig

# Copyright (c) 2014, James Hensman

class Poly(Kern):
    """
    Polynomial kernel
    """

    def __init__(self, input_dim, variance=1., order=3., active_dims=None, name='poly'):
        super(Poly, self).__init__(input_dim, active_dims, name)
        self.link_parameter(self.variance)
        self.order=order

        return (self._dot_product(X, X2) + 1.)**self.order * self.variance

    def _dot_product(self, X, X2=None):
            return np.dot(X, X.T)
        else:
            return np.dot(X, X2.T)

        return self.variance*(np.square(X).sum(1) + 1.)**self.order

        self.variance.gradient = np.sum(dL_dK * (self._dot_product(X, X2) + 1.)**self.order)

        raise NotImplementedError

    def gradients_X(self, dL_dK, X, X2=None):

    def gradients_X_diag(self, dL_dKdiag, X):

from stationary import Stationary
from psi_comp import PSICOMP_RBF
from psi_comp.rbf_psi_gpucomp import PSICOMP_RBF_GPU
from ...util.config import *

class RBF(Stationary):
    """
    Radial Basis Function kernel, aka squared-exponential, exponentiated quadratic or Gaussian kernel:

    .. math::

       k(r) = \sigma^2 \exp \\bigg(- \\frac{1}{2} r^2 \\bigg)

    """
    _support_GPU = True
    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=False, active_dims=None, name='rbf', useGPU=False):
        super(RBF, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name, useGPU=useGPU)
        self.psicomp = PSICOMP_RBF()
        if self.useGPU:
            self.psicomp = PSICOMP_RBF_GPU()
        else:
            self.psicomp = PSICOMP_RBF()

    def K_of_r(self, r):
        return self.variance * np.exp(-0.5 * r**2)

    def dK_dr(self, r):
        return -r*self.K_of_r(r)

    def __getstate__(self):
        dc = super(RBF, self).__getstate__()
            dc['psicomp'] = PSICOMP_RBF()
        return dc

    def __setstate__(self, state):
        return super(RBF, self).__setstate__(state)

    def spectrum(self, omega):
        assert self.input_dim == 1 #TODO: higher dim spectra?
        return self.variance*np.sqrt(2*np.pi)*self.lengthscale*np.exp(-self.lengthscale*2*omega**2/2)

    #---------------------------------------#
    #             PSI statistics            #

    def psi0(self, Z, variational_posterior):
        return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[0]

    def psi1(self, Z, variational_posterior):
        return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[1]

    def psi2(self, Z, variational_posterior):
        return self.psicomp.psicomputations(self.variance, self.lengthscale, Z, variational_posterior)[2]

    def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
        dL_dvar, dL_dlengscale = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[:2]
        self.variance.gradient = dL_dvar
        self.lengthscale.gradient = dL_dlengscale

    def gradients_Z_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
        return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[2]

    def gradients_qX_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
        return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variance, self.lengthscale, Z, variational_posterior)[3:]

# Copyright (c) 2012-2014, GPy authors (see AUTHORS.txt).

from ..inference import optimization
from ..util.misc import opt_wrapper
from parameterization import Parameterized
import multiprocessing as mp
from numpy.linalg.linalg import LinAlgError
# import numdifftools as ndt

class Model(Parameterized):
    _fail_count = 0  # Count of failed optimization steps (see objective)
    _allowed_failures = 10  # number of allowed failures

    def __init__(self, name):
        super(Model, self).__init__(name)  # Parameterized.__init__(self)
        self.optimization_runs = []
        self.sampling_runs = []
        self.preferred_optimizer = 'bfgs'
        from .parameterization.ties_and_remappings import Tie
        self.tie = Tie()
        self.link_parameter(self.tie, -1)
        self.add_observer(self.tie, self.tie._parameters_changed_notification, priority=-500)

        raise NotImplementedError, "this needs to be implemented to use the model class"
    def _log_likelihood_gradients(self):
        return self.gradient

    def optimize_restarts(self, num_restarts=10, robust=False, verbose=True, parallel=False, num_processes=None, **kwargs):
        """
        Perform random restarts of the model, and set the model to the best
        seen solution.

        If the robust flag is set, exceptions raised during optimizations will
        be handled silently.  If _all_ runs fail, the model is reset to the
        existing parameter values.

        **Notes**

        :param num_restarts: number of restarts to use (default 10)
        :type num_restarts: int
        :param robust: whether to handle exceptions silently or not (default False)
        :type robust: bool
        :param parallel: whether to run each restart as a separate process. It relies on the multiprocessing module.
        :type parallel: bool
        :param num_processes: number of workers in the multiprocessing pool
        :type numprocesses: int

        \*\*kwargs are passed to the optimizer. They can be:

        :param max_f_eval: maximum number of function evaluations
        :type max_f_eval: int
        :param max_iters: maximum number of iterations
        :type max_iters: int
        :param messages: whether to display during optimisation
        :type messages: bool

        .. note:: If num_processes is None, the number of workes in the
        multiprocessing pool is automatically set to the number of processors
        on the current machine.

        """
        initial_parameters = self.optimizer_array.copy()

        if parallel:
            try:
                jobs = []
                pool = mp.Pool(processes=num_processes)
                for i in range(num_restarts):
                    self.randomize()
                    job = pool.apply_async(opt_wrapper, args=(self,), kwds=kwargs)
                    jobs.append(job)

                pool.close()  # signal that no more data coming in
                pool.join()  # wait for all the tasks to complete
            except KeyboardInterrupt:
                print "Ctrl+c received, terminating and joining pool."
                pool.terminate()
                pool.join()

        for i in range(num_restarts):
            try:
                if not parallel:
                    self.optimize(**kwargs)
                else:
                    self.optimization_runs.append(jobs[i].get())

                if verbose:
                    print("Optimization restart {0}/{1}, f = {2}".format(i + 1, num_restarts, self.optimization_runs[-1].f_opt))
            except Exception as e:
                if robust:
                    print("Warning - optimization restart {0}/{1} failed".format(i + 1, num_restarts))
                else:
                    raise e

        if len(self.optimization_runs):
            i = np.argmin([o.f_opt for o in self.optimization_runs])
            self.optimizer_array = self.optimization_runs[i].x_opt
        else:
            self.optimizer_array = initial_parameters

    def ensure_default_constraints(self, warning=True):
        """
        Ensure that any variables which should clearly be positive
        have been constrained somehow. The method performs a regular
        expression search on parameter names looking for the terms
        'variance', 'lengthscale', 'precision' and 'kappa'. If any of
        these terms are present in the name the parameter is
        constrained positive.

        DEPRECATED.
        """
        raise DeprecationWarning, 'parameters now have default constraints'

    def objective_function(self):
        """
        The objective function for the given algorithm.

        This function is the true objective, which wants to be minimized.
        Note that all parameters are already set and in place, so you just need
        to return the objective function here.

        For probabilistic models this is the negative log_likelihood
        (including the MAP prior), so we return it here. If your model is not
        probabilistic, just return your objective to minimize here!
        """
        return -float(self.log_likelihood()) - self.log_prior()

    def objective_function_gradients(self):
        """
        The gradients for the objective function for the given algorithm.
        The gradients are w.r.t. the *negative* objective function, as
        this framework works with *negative* log-likelihoods as a default.

        You can find the gradient for the parameters in self.gradient at all times.
        This is the place, where gradients get stored for parameters.

        to return the gradient here.

        For probabilistic models this is the gradient of the negative log_likelihood
        probabilistic, just return your *negative* gradient here!
        """
        return -(self._log_likelihood_gradients() + self._log_prior_gradients())

    def _grads(self, x):
        """
        Gets the gradients from the likelihood and the priors.

        Failures are handled robustly. The algorithm will try several times to
        return the gradients, and will raise the original exception if
        the objective cannot be computed.

        :param x: the parameters of the model.
        :type x: np.array
        """
        try:
            # self._set_params_transformed(x)
            self.optimizer_array = x
            obj_grads = self._transform_gradients(self.objective_function_gradients())
            self._fail_count = 0
        except (LinAlgError, ZeroDivisionError, ValueError):
            if self._fail_count >= self._allowed_failures:
                raise
            self._fail_count += 1
            obj_grads = np.clip(self._transform_gradients(self.objective_function_gradients()), -1e100, 1e100)
        return obj_grads

    def _objective(self, x):
        """
        The objective function passed to the optimizer. It combines
        the likelihood and the priors.

        return the objective, and will raise the original exception if

        :parameter type: np.array
        """
        try:
            obj = self.objective_function()
                raise
            return np.inf
        return obj

    def _objective_grads(self, x):
        try:
            obj_f, obj_grads = self.objective_function(), self._transform_gradients(self.objective_function_gradients())
                raise
            obj_f = np.inf
        return obj_f, obj_grads

    def optimize(self, optimizer=None, start=None, **kwargs):
        """
        Optimize the model using self.log_likelihood and self.log_likelihood_gradient, as well as self.priors.

        kwargs are passed to the optimizer. They can be:

        :messages: whether to display during optimisation
        :param optimizer: which optimizer to use (defaults to self.preferred optimizer)
        :type optimizer: string

        Valid optimizers are:
          - 'scg': scaled conjugate gradient method, recommended for stability.
                   See also GPy.inference.optimization.scg
          - 'fmin_tnc': truncated Newton method (see scipy.optimize.fmin_tnc)
          - 'simplex': the Nelder-Mead simplex method (see scipy.optimize.fmin),
          - 'lbfgsb': the l-bfgs-b method (see scipy.optimize.fmin_l_bfgs_b),
          - 'sgd': stochastic gradient decsent (see scipy.optimize.sgd). For experts only!

        """
        if self.is_fixed:
            print 'nothing to optimize'
        if self.size == 0:

        if not self.update_model():
            print "setting updates on again"
            self.update_model(True)

        if start == None:
            start = self.optimizer_array

        if optimizer is None:
            optimizer = self.preferred_optimizer

        if isinstance(optimizer, optimization.Optimizer):
            opt = optimizer
            opt.model = self
        else:
            optimizer = optimization.get_optimizer(optimizer)
            opt = optimizer(start, model=self, **kwargs)

        opt.run(f_fp=self._objective_grads, f=self._objective, fp=self._grads)

        self.optimization_runs.append(opt)

        self.optimizer_array = opt.x_opt

    def optimize_SGD(self, momentum=0.1, learning_rate=0.01, iterations=20, **kwargs):
        # assert self.Y.shape[1] > 1, "SGD only works with D > 1"
        sgd = SGD.StochasticGD(self, iterations, learning_rate, momentum, **kwargs)  # @UndefinedVariable
        sgd.run()
        self.optimization_runs.append(sgd)

    def _checkgrad(self, target_param=None, verbose=False, step=1e-6, tolerance=1e-3, df_tolerance=1e-12):
        """
        Check the gradient of the ,odel by comparing to a numerical
        estimate.  If the verbose flag is passed, individual
        components are tested (and printed)

        :param verbose: If True, print a "full" checking of each parameter
        :type verbose: bool
        :param step: The size of the step around which to linearise the objective
        :type step: float (default 1e-6)
        :param tolerance: the tolerance allowed (see note)
        :type tolerance: float (default 1e-3)

        Note:-
           The gradient is considered correct if the ratio of the analytical
           and numerical gradients is within <tolerance> of unity.

           The *dF_ratio* indicates the limit of numerical accuracy of numerical gradients.
           If it is too small, e.g., smaller than 1e-12, the numerical gradients are usually
           not accurate enough for the tests (shown with blue).
        """
        x = self.optimizer_array.copy()

        if not verbose:
            # make sure only to test the selected parameters
            if target_param is None:
                transformed_index = range(len(x))
            else:
                transformed_index = self._raveled_index_for(target_param)
                if self._has_fixes():
                    indices = np.r_[:self.size]
                    which = (transformed_index[:, None] == indices[self._fixes_][None, :]).nonzero()
                    transformed_index = (indices - (~self._fixes_).cumsum())[transformed_index[which[0]]]

                if transformed_index.size == 0:
                    print "No free parameters to check"
                    return

            # just check the global ratio
            dx = np.zeros(x.shape)
            dx[transformed_index] = step * (np.sign(np.random.uniform(-1, 1, transformed_index.size)) if transformed_index.size != 2 else 1.)

            # evaulate around the point x
            f1 = self._objective(x + dx)
            f2 = self._objective(x - dx)
            gradient = self._grads(x)

            dx = dx[transformed_index]
            gradient = gradient[transformed_index]

            denominator = (2 * np.dot(dx, gradient))
            global_ratio = (f1 - f2) / np.where(denominator == 0., 1e-32, denominator)
            global_diff = np.abs(f1 - f2) < tolerance and np.allclose(gradient, 0, atol=tolerance)
            if global_ratio is np.nan:
                global_ratio = 0
            return np.abs(1. - global_ratio) < tolerance or global_diff
        else:
            # check the gradient of each parameter individually, and do some pretty printing
            try:
                names = self._get_param_names()
            except NotImplementedError:
                names = ['Variable %i' % i for i in range(len(x))]
            # Prepare for pretty-printing
            header = ['Name', 'Ratio', 'Difference', 'Analytical', 'Numerical', 'dF_ratio']
            max_names = max([len(names[i]) for i in range(len(names))] + [len(header[0])])
            float_len = 10
            cols = [max_names]
            cols.extend([max(float_len, len(header[i])) for i in range(1, len(header))])
            cols = np.array(cols) + 5
            header_string = ["{h:^{col}}".format(h=header[i], col=cols[i]) for i in range(len(cols))]
            header_string = map(lambda x: '|'.join(x), [header_string])
            separator = '-' * len(header_string[0])
            print '\n'.join([header_string[0], separator])
                param_index = range(len(x))
                transformed_index = param_index
            else:
                param_index = self._raveled_index_for(target_param)
                    which = (param_index[:, None] == indices[self._fixes_][None, :]).nonzero()
                    param_index = param_index[which[0]]
                    transformed_index = (indices - (~self._fixes_).cumsum())[param_index]
                    # print param_index, transformed_index
                else:
                    transformed_index = param_index

                if param_index.size == 0:
                    return

            gradient = self._grads(x).copy()
            np.where(gradient == 0, 1e-312, gradient)
            ret = True
            for nind, xind in itertools.izip(param_index, transformed_index):
                xx = x.copy()
                xx[xind] += step
                f1 = self._objective(xx)
                xx[xind] -= 2.*step
                f2 = self._objective(xx)
                df_ratio = np.abs((f1-f2)/min(f1,f2))
                df_unstable = df_ratio<df_tolerance
                numerical_gradient = (f1 - f2) / (2 * step)
                if np.all(gradient[xind] == 0): ratio = (f1 - f2) == gradient[xind]
                else: ratio = (f1 - f2) / (2 * step * gradient[xind])
                difference = np.abs(numerical_gradient - gradient[xind])

                if (np.abs(1. - ratio) < tolerance) or np.abs(difference) < tolerance:
                    formatted_name = "\033[92m {0} \033[0m".format(names[nind])
                    ret &= True
                else:
                    formatted_name = "\033[91m {0} \033[0m".format(names[nind])
                    ret &= False
                if df_unstable:
                    formatted_name = "\033[94m {0} \033[0m".format(names[nind])

                r = '%.6f' % float(ratio)
                d = '%.6f' % float(difference)
                g = '%.6f' % gradient[xind]
                ng = '%.6f' % float(numerical_gradient)
                df = '%1.e' % float(df_ratio)
                grad_string = "{0:<{c0}}|{1:^{c1}}|{2:^{c2}}|{3:^{c3}}|{4:^{c4}}|{5:^{c5}}".format(formatted_name, r, d, g, ng, df, c0=cols[0] + 9, c1=cols[1], c2=cols[2], c3=cols[3], c4=cols[4], c5=cols[5])
                print grad_string

            return ret

    def _repr_html_(self):
        """Representation of the model in html for notebook display."""
        model_details = [['<b>Model</b>', self.name + '<br>'],
                         ['<b>Log-likelihood</b>', '{}<br>'.format(float(self.log_likelihood()))],
                         ["<b>Number of Parameters</b>", '{}<br>'.format(self.size)]]
        from operator import itemgetter
        to_print = [""] + ["{}: {}".format(name, detail) for name, detail in model_details] + ["<br><b>Parameters</b>:"]
        to_print.append(super(Model, self)._repr_html_())
        return "\n".join(to_print)

    def __str__(self):
        model_details = [['Name', self.name],
                         ['Log-likelihood', '{}'.format(float(self.log_likelihood()))],
                         ["Number of Parameters", '{}'.format(self.size)]]
        max_len = reduce(lambda a, b: max(len(b[0]), a), model_details, 0)
        to_print = [""] + ["{0:{l}} : {1}".format(name, detail, l=max_len) for name, detail in model_details] + ["Parameters:"]
        to_print.append(super(Model, self).__str__())

class ODE_t(Kern):

        def __init__(self, input_dim, a=1., c=1.,variance_Yt=3., lengthscale_Yt=1.5,ubias =1., active_dims=None, name='ode_st'):
                assert input_dim ==2, "only defined for 2 input dims"
                super(ODE_t, self).__init__(input_dim, active_dims, name)

                self.variance_Yt = Param('variance_Yt', variance_Yt, Logexp())
                self.lengthscale_Yt = Param('lengthscale_Yt', lengthscale_Yt, Logexp())        

                self.a= Param('a', a, Logexp())
                self.c = Param('c', c, Logexp())
                self.ubias = Param('ubias', ubias, Logexp())
                self.add_parameters(self.a, self.c, self.variance_Yt, self.lengthscale_Yt,self.ubias)

        def K(self, X, X2=None):
                """Compute the covariance matrix between X and X2."""        
                X,slices = X[:,:-1],index_to_slices(X[:,-1])
                if X2 is None:
                        X2,slices2 = X,slices
                        K = np.zeros((X.shape[0], X.shape[0]))
                else:
                        X2,slices2 = X2[:,:-1],index_to_slices(X2[:,-1])
                        K = np.zeros((X.shape[0], X2.shape[0]))

                tdist = (X[:,0][:,None] - X2[:,0][None,:])**2
                ttdist = (X[:,0][:,None] - X2[:,0][None,:])
                
                vyt = self.variance_Yt
                
                lyt=1/(2*self.lengthscale_Yt)

                a = -self.a
                c = self.c

                kyy = lambda tdist: np.exp(-lyt*(tdist))

                k1 = lambda tdist: (2*lyt - 4*lyt**2 *(tdist) )

                k4 = lambda tdist: 2*lyt*(tdist)

                for i, s1 in enumerate(slices):
                        for j, s2 in enumerate(slices2):
                                for ss1 in s1:
                                    for ss2 in s2:
                                        if i==0 and j==0:
                                            K[ss1,ss2] = vyt*kyy(tdist[ss1,ss2])
                                        elif i==0 and j==1:
                                            K[ss1,ss2] = (k4(ttdist[ss1,ss2])+1)*vyt*kyy(tdist[ss1,ss2])
                                            #K[ss1,ss2] = (2*lyt*(ttdist[ss1,ss2])+1)*vyt*kyy(tdist[ss1,ss2])
                                        elif i==1 and j==1:
                                            K[ss1,ss2] = ( k1(tdist[ss1,ss2]) + 1. )*vyt* kyy(tdist[ss1,ss2])+self.ubias
                                        else:
                                            K[ss1,ss2] = (-k4(ttdist[ss1,ss2])+1)*vyt*kyy(tdist[ss1,ss2])
                                            #K[ss1,ss2] = (-2*lyt*(ttdist[ss1,ss2])+1)*vyt*kyy(tdist[ss1,ss2])
                #stop
                return K

        def Kdiag(self, X):

                lyt = 1./(2*self.lengthscale_Yt)

                a = -self.a
                c = self.c        
                
                k1 = (2*lyt )*vyt
                
                Kdiag = np.zeros(X.shape[0])
                slices = index_to_slices(X[:,-1])

                for i, ss1 in enumerate(slices):
                    for s1 in ss1:
                        if i==0:
                            Kdiag[s1]+= vyt
                        elif i==1:
                            #i=1
                            Kdiag[s1]+= k1 + vyt+self.ubias
                            #Kdiag[s1]+= Vu*Vy*(k1+k2+k3)
                        else:
                            raise ValueError, "invalid input/output index"

                return Kdiag

        def update_gradients_full(self, dL_dK, X, X2=None):
                """derivative of the covariance matrix with respect to the parameters."""
                    X2,slices2 = X,slices
                    K = np.zeros((X.shape[0], X.shape[0]))
                else:
                    X2,slices2 = X2[:,:-1],index_to_slices(X2[:,-1])

                #rdist = [tdist,xdist]
                
                rd=tdist.shape[0]

                dka = np.zeros([rd,rd])
                dkc = np.zeros([rd,rd])
                dkYdvart = np.zeros([rd,rd])
                dkYdlent = np.zeros([rd,rd])

                dkdubias = np.zeros([rd,rd])

                dkyydlyt = lambda tdist: kyy(tdist)*(-tdist)

                k1 = lambda tdist: (2*lyt - 4*lyt**2 * (tdist) )

                k4 = lambda ttdist: 2*lyt*(ttdist)

                dk1dlyt = lambda tdist: 2. - 4*2.*lyt*tdist

                dk4dlyt = lambda ttdist: 2*(ttdist)

                    for j, s2 in enumerate(slices2):
                        for ss1 in s1:
                            for ss2 in s2:
                                if i==0 and j==0:
                                    dkYdvart[ss1,ss2] = kyy(tdist[ss1,ss2])
                                    dkYdlent[ss1,ss2] = vyt*dkyydlyt(tdist[ss1,ss2])
                                    dkdubias[ss1,ss2] = 0
                                elif i==0 and j==1:
                                    dkYdvart[ss1,ss2] = (k4(ttdist[ss1,ss2])+1)*kyy(tdist[ss1,ss2])
                                    #dkYdvart[ss1,ss2] = ((2*lyt*ttdist[ss1,ss2])+1)*kyy(tdist[ss1,ss2])
                                    dkYdlent[ss1,ss2] = vyt*dkyydlyt(tdist[ss1,ss2])* (k4(ttdist[ss1,ss2])+1.)+\
                                    vyt*kyy(tdist[ss1,ss2])*(dk4dlyt(ttdist[ss1,ss2]))
                                    #dkYdlent[ss1,ss2] = vyt*dkyydlyt(tdist[ss1,ss2])* (2*lyt*(ttdist[ss1,ss2])+1.)+\
                                    #vyt*kyy(tdist[ss1,ss2])*(2*ttdist[ss1,ss2])
                                elif i==1 and j==1:
                                    dkYdvart[ss1,ss2] = (k1(tdist[ss1,ss2]) + 1. )* kyy(tdist[ss1,ss2])
                                    dkYdlent[ss1,ss2] = vyt*dkyydlyt(tdist[ss1,ss2])*( k1(tdist[ss1,ss2]) + 1. ) +\
                          			vyt*kyy(tdist[ss1,ss2])*dk1dlyt(tdist[ss1,ss2])
                                    dkdubias[ss1,ss2] = 1
                                else:
                                    dkYdvart[ss1,ss2] = (-k4(ttdist[ss1,ss2])+1)*kyy(tdist[ss1,ss2])
                                    #dkYdvart[ss1,ss2] = (-2*lyt*(ttdist[ss1,ss2])+1)*kyy(tdist[ss1,ss2])
                                    dkYdlent[ss1,ss2] = vyt*dkyydlyt(tdist[ss1,ss2])* (-k4(ttdist[ss1,ss2])+1.)+\
                                    vyt*kyy(tdist[ss1,ss2])*(-dk4dlyt(ttdist[ss1,ss2]) )
                                    #dkYdlent[ss1,ss2] = vyt*dkyydlyt(tdist[ss1,ss2])* (-2*lyt*(ttdist[ss1,ss2])+1.)+\
                                    #vyt*kyy(tdist[ss1,ss2])*(-2)*(ttdist[ss1,ss2])
   
                self.variance_Yt.gradient = np.sum(dkYdvart * dL_dK)

                self.lengthscale_Yt.gradient =  np.sum(dkYdlent*(-0.5*self.lengthscale_Yt**(-2)) * dL_dK)

                self.ubias.gradient = np.sum(dkdubias * dL_dK) 

'''
Created on 24 Feb 2014

@author: maxz
'''

from GPy.util.pca import PCA

def initialize_latent(init, input_dim, Y):
    Xr = np.asfortranarray(np.random.randn(Y.shape[0], input_dim))
    if init == 'PCA':
        p = PCA(Y)
        PC = p.project(Y, min(input_dim, Y.shape[1]))
        Xr[:PC.shape[0], :PC.shape[1]] = PC
        var = .1*p.fracs[:input_dim]
    else:
        var = Xr.var(0)

    Xr -= Xr.mean(0)
    Xr /= Xr.std(0)

    return Xr, var/var.max()

'''
.. module:: GPy.util.subarray_and_sorting

.. moduleauthor:: Max Zwiessele <ibinbei@gmail.com>

'''
__updated__ = '2014-05-21'

import numpy as np, logging

def common_subarrays(X, axis=0):
    """
    Find common subarrays of 2 dimensional X, where axis is the axis to apply the search over.
    Common subarrays are returned as a dictionary of <subarray, [index]> pairs, where
    the subarray is a tuple representing the subarray and the index is the index
    for the subarray in X, where index is the index to the remaining axis.

    :param :class:`np.ndarray` X: 2d array to check for common subarrays in
    :param int axis: axis to apply subarray detection over.
        When the index is 0, compare rows -- columns, otherwise.

    Examples:
    =========

    In a 2d array:
    >>> import numpy as np
    >>> X = np.zeros((3,6), dtype=bool)
    >>> X[[1,1,1],[0,4,5]] = 1; X[1:,[2,3]] = 1
    >>> X
    array([[False, False, False, False, False, False],
           [ True, False,  True,  True,  True,  True],
           [False, False,  True,  True, False, False]], dtype=bool)
    >>> d = common_subarrays(X,axis=1)
    >>> len(d)
    3
    >>> X[:, d[tuple(X[:,0])]]
    array([[False, False, False],
           [ True,  True,  True],
           [False, False, False]], dtype=bool)
    >>> d[tuple(X[:,4])] == d[tuple(X[:,0])] == [0, 4, 5]
    True
    >>> d[tuple(X[:,1])]
    [1]
    """
    from collections import defaultdict
    from itertools import count
    from operator import iadd
    assert X.ndim == 2 and axis in (0,1), "Only implemented for 2D arrays"
    subarrays = defaultdict(list)
    cnt = count()
    def accumulate(x, s, c):
        t = tuple(x)
        col = c.next()
        iadd(s[t], [col])
        return None
    if axis == 0: [accumulate(x, subarrays, cnt) for x in X]
    else: [accumulate(x, subarrays, cnt) for x in X.T]
    return subarrays

if __name__ == '__main__':
    import doctest
    doctest.testmod()

"""
The module plotting results for SSGPLVM
"""

import pylab

from ...models import SSGPLVM
from img_plots import plot_2D_images

class SSGPLVM_plot(object):
    def __init__(self,model, imgsize):
        assert isinstance(model,SSGPLVM)
        self.model = model
        self.imgsize= imgsize
        assert model.Y.shape[1] == imgsize[0]*imgsize[1]
        
    def plot_inducing(self):
        fig1 = pylab.figure()
        mean = self.model.posterior.mean
        arr = mean.reshape(*(mean.shape[0],self.imgsize[1],self.imgsize[0]))
        plot_2D_images(fig1, arr)
        fig1.gca().set_title('The mean of inducing points')
        
        fig2 = pylab.figure()
        covar = self.model.posterior.covariance
        plot_2D_images(fig2, covar)
        fig2.gca().set_title('The variance of inducing points')
        
# ## Copyright (c) 2013, GPy authors (see AUTHORS.txt).

import itertools, logging

from ..kern import Kern
from ..core.parameterization.variational import NormalPosterior, NormalPrior
from ..core.parameterization import Param, Parameterized
from ..core.parameterization.observable_array import ObsAr
from ..inference.latent_function_inference.var_dtc import VarDTC
from ..inference.latent_function_inference import InferenceMethodList
from ..likelihoods import Gaussian
from ..util.initialization import initialize_latent
from ..core.sparse_gp import SparseGP, GP
from GPy.core.parameterization.variational import VariationalPosterior
from GPy.models.bayesian_gplvm_minibatch import BayesianGPLVMMiniBatch
from GPy.models.sparse_gp_minibatch import SparseGPMiniBatch

class MRD(BayesianGPLVMMiniBatch):
    """
    !WARNING: This is bleeding edge code and still in development.
    Functionality may change fundamentally during development!

    Apply MRD to all given datasets Y in Ylist.

    Y_i in [n x p_i]

    If Ylist is a dictionary, the keys of the dictionary are the names, and the
    values are the different datasets to compare.

    The samples n in the datasets need
    to match up, whereas the dimensionality p_d can differ.

    :param [array-like] Ylist: List of datasets to apply MRD on
    :param input_dim: latent dimensionality
    :param array-like X: mean of starting latent space q in [n x q]
    :param array-like X_variance: variance of starting latent space q in [n x q]
    :param initx: initialisation method for the latent space :

        * 'concat' - PCA on concatenation of all datasets
        * 'single' - Concatenation of PCA on datasets, respectively
        * 'random' - Random draw from a Normal(0,1)

    :type initx: ['concat'|'single'|'random']
    :param initz: initialisation method for inducing inputs
    :type initz: 'permute'|'random'
    :param num_inducing: number of inducing inputs to use
    :param Z: initial inducing inputs
    :param kernel: list of kernels or kernel to copy for each output
    :type kernel: [GPy.kernels.kernels] | GPy.kernels.kernels | None (default)
    :param :class:`~GPy.inference.latent_function_inference inference_method:
        InferenceMethodList of inferences, or one inference method for all
    :param :class:`~GPy.likelihoodss.likelihoods.likelihoods` likelihoods: the likelihoods to use
    :param str name: the name of this model
    :param [str] Ynames: the names for the datasets given, must be of equal length as Ylist or None
    :param bool|Norm normalizer: How to normalize the data?
    :param bool stochastic: Should this model be using stochastic gradient descent over the dimensions?
    :param bool|[bool] batchsize: either one batchsize for all, or one batchsize per dataset.
    """
    def __init__(self, Ylist, input_dim, X=None, X_variance=None,
                 initx = 'PCA', initz = 'permute',
                 num_inducing=10, Z=None, kernel=None,
                 inference_method=None, likelihoods=None, name='mrd',
                 Ynames=None, normalizer=False, stochastic=False, batchsize=10):

        self.logger = logging.getLogger(self.__class__.__name__)
        self.input_dim = input_dim
        self.num_inducing = num_inducing

        if isinstance(Ylist, dict):
            Ynames, Ylist = zip(*Ylist.items())

        self.logger.debug("creating observable arrays")
        self.Ylist = [ObsAr(Y) for Y in Ylist]

        if Ynames is None:
            self.logger.debug("creating Ynames")
            Ynames = ['Y{}'.format(i) for i in range(len(Ylist))]
        self.names = Ynames
        assert len(self.names) == len(self.Ylist), "one name per dataset, or None if Ylist is a dict"

        if inference_method is None:
            self.inference_method = InferenceMethodList([VarDTC() for _ in xrange(len(self.Ylist))])
        else:
            assert isinstance(inference_method, InferenceMethodList), "please provide one inference method per Y in the list and provide it as InferenceMethodList, inference_method given: {}".format(inference_method)
            self.inference_method = inference_method

        if X is None:
            X, fracs = self._init_X(initx, Ylist)
        else:
            fracs = [X.var(0)]*len(Ylist)

        Z = self._init_Z(initz, X)
        self.Z = Param('inducing inputs', Z)
        self.num_inducing = self.Z.shape[0] # ensure M==N if M>N

        # sort out the kernels
        self.logger.info("building kernels")
            from ..kern import RBF
            kernels = [RBF(input_dim, ARD=1, lengthscale=1./fracs[i]) for i in range(len(Ylist))]
        elif isinstance(kernel, Kern):
            kernels = []
            for i in range(len(Ylist)):
                k = kernel.copy()
                kernels.append(k)
        else:
            assert len(kernel) == len(Ylist), "need one kernel per output"
            assert all([isinstance(k, Kern) for k in kernel]), "invalid kernel object detected!"
            kernels = kernel

        if X_variance is None:
            X_variance = np.random.uniform(0.1, 0.2, X.shape)

        self.variational_prior = NormalPrior()
        #self.X = NormalPosterior(X, X_variance)

        if likelihoods is None:
            likelihoods = [Gaussian(name='Gaussian_noise'.format(i)) for i in range(len(Ylist))]
        else: likelihoods = likelihoods

        self.logger.info("adding X and Z")
        super(MRD, self).__init__(Y, input_dim, X=X, X_variance=X_variance, num_inducing=num_inducing,
                 Z=self.Z, kernel=None, inference_method=self.inference_method, likelihood=Gaussian(),
                 name='manifold relevance determination', normalizer=None,
                 missing_data=False, stochastic=False, batchsize=1)

        self._log_marginal_likelihood = 0

        self.unlink_parameter(self.likelihood)
        self.unlink_parameter(self.kern)
        del self.kern
        del self.likelihood

        self.num_data = Ylist[0].shape[0]
        if isinstance(batchsize, int):
            batchsize = itertools.repeat(batchsize)

        self.bgplvms = []

        for i, n, k, l, Y, im, bs in itertools.izip(itertools.count(), Ynames, kernels, likelihoods, Ylist, self.inference_method, batchsize):
            assert Y.shape[0] == self.num_data, "All datasets need to share the number of datapoints, and those have to correspond to one another"
            md = np.isnan(Y).any()
            spgp = BayesianGPLVMMiniBatch(Y, input_dim, X, X_variance,
                                          Z=Z, kernel=k, likelihood=l,
                                          inference_method=im, name=n,
                                          normalizer=normalizer,
                                          missing_data=md,
                                          stochastic=stochastic,
                                          batchsize=bs)
            spgp.kl_factr = 1./len(Ynames)
            spgp.unlink_parameter(spgp.Z)
            spgp.unlink_parameter(spgp.X)
            del spgp.Z
            del spgp.X
            spgp.Z = self.Z
            spgp.X = self.X
            self.link_parameter(spgp, i+2)
            self.bgplvms.append(spgp)

        self.posterior = None
        self.logger.info("init done")

        self.Z.gradient[:] = 0.
        self.X.gradient[:] = 0.
        for b, i in itertools.izip(self.bgplvms, self.inference_method):
            self._log_marginal_likelihood += b._log_marginal_likelihood

            self.logger.info('working on im <{}>'.format(hex(id(i))))
            self.Z.gradient[:] += b.full_values['Zgrad']
            grad_dict = b.full_values

            self.X.mean.gradient += grad_dict['meangrad']
            self.X.variance.gradient += grad_dict['vargrad']

        if isinstance(self.X, VariationalPosterior):
            # update for the KL divergence
            self.variational_prior.update_gradients_KL(self.X)
            self._log_marginal_likelihood -= self.variational_prior.KL_divergence(self.X)
            pass

    def _init_X(self, init='PCA', Ylist=None):
        if Ylist is None:
            Ylist = self.Ylist
        if init in "PCA_concat":
            X, fracs = initialize_latent('PCA', self.input_dim, np.hstack(Ylist))
            fracs = [fracs]*len(Ylist)
        elif init in "PCA_single":
            X = np.zeros((Ylist[0].shape[0], self.input_dim))
            fracs = []
            for qs, Y in itertools.izip(np.array_split(np.arange(self.input_dim), len(Ylist)), Ylist):
                x,frcs = initialize_latent('PCA', len(qs), Y)
                X[:, qs] = x
                fracs.append(frcs)
        else: # init == 'random':
            X = np.random.randn(Ylist[0].shape[0], self.input_dim)
            fracs = X.var(0)
        X -= X.mean()
        X /= X.std()
        return X, fracs

    def _init_Z(self, init="permute", X=None):
            X = self.X
        if init in "permute":
            Z = np.random.permutation(X.copy())[:self.num_inducing]
        elif init in "random":
            Z = np.random.randn(self.num_inducing, self.input_dim) * X.var()
        return Z

    def _handle_plotting(self, fignum, axes, plotf, sharex=False, sharey=False):
        import matplotlib.pyplot as plt
        if axes is None:
            fig = plt.figure(num=fignum)
        sharex_ax = None
        sharey_ax = None
        plots = []
        for i, g in enumerate(self.bgplvms):
            try:
                if sharex:
                    sharex_ax = ax # @UndefinedVariable
                    sharex = False # dont set twice
                if sharey:
                    sharey_ax = ax # @UndefinedVariable
                    sharey = False # dont set twice
            except:
                pass
            if axes is None:
                ax = fig.add_subplot(1, len(self.bgplvms), i + 1, sharex=sharex_ax, sharey=sharey_ax)
            elif isinstance(axes, (tuple, list, np.ndarray)):
                ax = axes[i]
            else:
                raise ValueError("Need one axes per latent dimension input_dim")
            plots.append(plotf(i, g, ax))
            if sharey_ax is not None:
                plt.setp(ax.get_yticklabels(), visible=False)
        plt.draw()
            try:
                fig.tight_layout()
            except:
                pass
        return plots

    def predict(self, Xnew, full_cov=False, Y_metadata=None, kern=None, Yindex=0):
        """
        Prediction for data set Yindex[default=0].
        This predicts the output mean and variance for the dataset given in Ylist[Yindex]
        """
        b = self.bgplvms[Yindex]
        self.posterior = b.posterior
        self.kern = b.kern
        self.likelihood = b.likelihood
        return super(MRD, self).predict(Xnew, full_cov, Y_metadata, kern)

    #===============================================================================
    # TODO: Predict! Maybe even change to several bgplvms, which share an X?
    #     def plot_predict(self, fignum=None, ax=None, sharex=False, sharey=False, **kwargs):
    #         fig = self._handle_plotting(fignum,
    #                                     ax,
    #                                     lambda i, g, ax: ax.imshow(g.predict(g.X)[0], **kwargs),
    #                                     sharex=sharex, sharey=sharey)
    #         return fig

    def plot_scales(self, fignum=None, ax=None, titles=None, sharex=False, sharey=True, *args, **kwargs):
        """

        TODO: Explain other parameters

        :param titles: titles for axes of datasets

        """
        if titles is None:
            titles = [r'${}$'.format(name) for name in self.names]
        ymax = reduce(max, [np.ceil(max(g.kern.input_sensitivity())) for g in self.bgplvms])
        def plotf(i, g, ax):
            #ax.set_ylim([0,ymax])
            return g.kern.plot_ARD(ax=ax, title=titles[i], *args, **kwargs)
        fig = self._handle_plotting(fignum, ax, plotf, sharex=sharex, sharey=sharey)
        return fig

    def plot_latent(self, labels=None, which_indices=None,
                resolution=50, ax=None, marker='o', s=40,
                fignum=None, plot_inducing=True, legend=True,
                plot_limits=None,
                aspect='auto', updates=False, predict_kwargs={}, imshow_kwargs={}):
        """
        see plotting.matplot_dep.dim_reduction_plots.plot_latent
        if predict_kwargs is None, will plot latent spaces for 0th dataset (and kernel), otherwise give
        predict_kwargs=dict(Yindex='index') for plotting only the latent space of dataset with 'index'.
        """
        import sys
        assert "matplotlib" in sys.modules, "matplotlib package has not been imported."
        from matplotlib import pyplot as plt
        from ..plotting.matplot_dep import dim_reduction_plots
        if "Yindex" not in predict_kwargs:
            predict_kwargs['Yindex'] = 0

        Yindex = predict_kwargs['Yindex']
            ax = fig.add_subplot(111)
        else:
            fig = ax.figure
        self.kern = self.bgplvms[Yindex].kern
        self.likelihood = self.bgplvms[Yindex].likelihood
        plot = dim_reduction_plots.plot_latent(self, labels, which_indices,
                                        resolution, ax, marker, s,
                                        fignum, plot_inducing, legend,
                                        plot_limits, aspect, updates, predict_kwargs, imshow_kwargs)
        ax.set_title(self.bgplvms[Yindex].name)
        try:
            fig.tight_layout()
        except:
            pass

        return plot

        state = super(MRD, self).__getstate__()
        if state.has_key('kern'):
            del state['kern']
        if state.has_key('likelihood'):
            del state['likelihood']
        return state

        # TODO:
        super(MRD, self).__setstate__(state)
        self.kern = self.bgplvms[0].kern
        self.likelihood = self.bgplvms[0].likelihood
        self.parameters_changed()

# Copyright I. Nabney, N.Lawrence and James Hensman (1996 - 2014)

# Scaled Conjuagte Gradients, originally in Matlab as part of the Netlab toolbox by I. Nabney, converted to python N. Lawrence and given a pythonic interface by James Hensman

#      THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT
#      HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
#      EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
#      NOT LIMITED TO, THE IMPLIED WARRANTIES OF
#      MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#      PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
#      REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
#      DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#      EXEMPLARY, OR CONSEQUENTIAL DAMAGES
#      (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
#      OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#      DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
#      HOWEVER CAUSED AND ON ANY THEORY OF
#      LIABILITY, WHETHER IN CONTRACT, STRICT
#      LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
#      OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#      OF THIS SOFTWARE, EVEN IF ADVISED OF THE
#      POSSIBILITY OF SUCH DAMAGE.

import sys

def print_out(len_maxiters, fnow, current_grad, beta, iteration):
    print '\r',
    print '{0:>0{mi}g}  {1:> 12e}  {2:< 12.6e}  {3:> 12e}'.format(iteration, float(fnow), float(beta), float(current_grad), mi=len_maxiters), # print 'Iteration:', iteration, ' Objective:', fnow, '  Scale:', beta, '\r',

def exponents(fnow, current_grad):
    exps = [np.abs(np.float(fnow)), current_grad]
    return np.sign(exps) * np.log10(exps).astype(int)

def SCG(f, gradf, x, optargs=(), maxiters=500, max_f_eval=np.inf, display=True, xtol=None, ftol=None, gtol=None):
    """
    Optimisation through Scaled Conjugate Gradients (SCG)

    f: the objective function
    gradf : the gradient function (should return a 1D np.ndarray)
    x : the initial condition

    Returns
    x the optimal value for x
    flog : a list of all the objective values
    function_eval number of fn evaluations
    status: string describing convergence status
    """
    if xtol is None:
        xtol = 1e-6
    if ftol is None:
        ftol = 1e-6
    if gtol is None:
        gtol = 1e-5

    sigma0 = 1.0e-7
    fold = f(x, *optargs) # Initial function value.
    function_eval = 1
    fnow = fold
    gradnew = gradf(x, *optargs) # Initial gradient.
    #if any(np.isnan(gradnew)):
    #    raise UnexpectedInfOrNan, "Gradient contribution resulted in a NaN value"
    current_grad = np.dot(gradnew, gradnew)
    gradold = gradnew.copy()
    d = -gradnew # Initial search direction.
    success = True # Force calculation of directional derivs.
    nsuccess = 0 # nsuccess counts number of successes.
    beta = 1.0 # Initial scale parameter.
    betamin = 1.0e-15 # Lower bound on scale.
    betamax = 1.0e15 # Upper bound on scale.
    status = "Not converged"

    flog = [fold]

    iteration = 0

    len_maxiters = len(str(maxiters))
    if display:
        print ' {0:{mi}s}   {1:11s}    {2:11s}    {3:11s}'.format("I", "F", "Scale", "|g|", mi=len_maxiters)
        exps = exponents(fnow, current_grad)
        p_iter = iteration

    # Main optimization loop.
    while iteration < maxiters:

        # Calculate first and second directional derivatives.
        if success:
            mu = np.dot(d, gradnew)
            if mu >= 0:
                d = -gradnew
                mu = np.dot(d, gradnew)
            kappa = np.dot(d, d)
            sigma = sigma0 / np.sqrt(kappa)
            xplus = x + sigma * d
            gplus = gradf(xplus, *optargs)
            theta = np.dot(d, (gplus - gradnew)) / sigma

        # Increase effective curvature and evaluate step size alpha.
        delta = theta + beta * kappa
        if delta <= 0:
            delta = beta * kappa
            beta = beta - theta / kappa

        alpha = -mu / delta

        # Calculate the comparison ratio.
        xnew = x + alpha * d
        fnew = f(xnew, *optargs)
        function_eval += 1

#         if function_eval >= max_f_eval:
#             status = "maximum number of function evaluations exceeded"
#             break
#             return x, flog, function_eval, status

        Delta = 2.*(fnew - fold) / (alpha * mu)
        if Delta >= 0.:
            success = True
            nsuccess += 1
            x = xnew
            fnow = fnew
        else:
            success = False
            fnow = fold

        # Store relevant variables
        flog.append(fnow) # Current function value

        iteration += 1
        if display:
            print_out(len_maxiters, fnow, current_grad, beta, iteration)
            n_exps = exponents(fnow, current_grad)
            if iteration - p_iter >= 20 * np.random.rand():
                a = iteration >= p_iter * 2.78
                b = np.any(n_exps < exps)
                if a or b:
                    p_iter = iteration
                    print ''
                if b:
                    exps = n_exps

            # Test for termination

            if (np.abs(fnew - fold) < ftol):
                status = 'converged - relative reduction in objective'
                break
#                 return x, flog, function_eval, status
            elif (np.max(np.abs(alpha * d)) < xtol):
                status = 'converged - relative stepsize'
                break
            else:
                # Update variables for new position
                gradold = gradnew
                gradnew = gradf(x, *optargs)
                current_grad = np.dot(gradnew, gradnew)
                fold = fnew
                # If the gradient is zero then we are done.
                if current_grad <= gtol:
                    status = 'converged - relative reduction in gradient'
                    break
                    # return x, flog, function_eval, status

        # Adjust beta according to comparison ratio.
        if Delta < 0.25:
            beta = min(4.0 * beta, betamax)
        if Delta > 0.75:
            beta = max(0.25 * beta, betamin)

        # Update search direction using Polak-Ribiere formula, or re-start
        # in direction of negative gradient after nparams steps.
        if nsuccess == x.size:
            d = -gradnew
            beta = 1. # This is not in the original paper
            nsuccess = 0
        elif success:
            Gamma = np.dot(gradold - gradnew, gradnew) / (mu)
            d = Gamma * d - gradnew
    else:
        # If we get here, then we haven't terminated in the given number of
        # iterations.
        status = "maxiter exceeded"

        print_out(len_maxiters, fnow, current_grad, beta, iteration)
        print ""
    return x, flog, function_eval, status

# Copyright (c) 2014, Max Zwiessele
"""

MaxZ

"""
import sys

def deepTest(reason):
    if reason:
        return lambda x:x
    return unittest.skip("Not deep scanning, enable deepscan by adding 'deep' argument")

#
# Module implementing synchronization primitives
#
# multiprocessing/synchronize.py
#
# Copyright (c) 2006-2008, R Oudkerk
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. Neither the name of author nor the names of any contributors may be
#    used to endorse or promote products derived from this software
#    without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#

__all__ = [
    'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', 'Event'
    ]

import threading
import os
import sys

from time import time as _time, sleep as _sleep

import _multiprocessing
from multiprocessing.process import current_process
from multiprocessing.util import Finalize, register_after_fork, debug
from multiprocessing.forking import assert_spawning, Popen

# Try to import the mp.synchronize module cleanly, if it fails
# raise ImportError for platforms lacking a working sem_open implementation.
# See issue 3770
try:
    from _multiprocessing import SemLock
except (ImportError):
    raise ImportError("This platform lacks a functioning sem_open" +
                      " implementation, therefore, the required" +
                      " synchronization primitives needed will not" +
                      " function, see issue 3770.")

#
# Constants
#

RECURSIVE_MUTEX, SEMAPHORE = range(2)
SEM_VALUE_MAX = _multiprocessing.SemLock.SEM_VALUE_MAX

#
# Base class for semaphores and mutexes; wraps `_multiprocessing.SemLock`
#

class SemLock(object):

    def __init__(self, kind, value, maxvalue):
        sl = self._semlock = _multiprocessing.SemLock(kind, value, maxvalue)
        debug('created semlock with handle %s' % sl.handle)
        self._make_methods()

        if sys.platform != 'win32':
            def _after_fork(obj):
                obj._semlock._after_fork()
            register_after_fork(self, _after_fork)

    def _make_methods(self):
        self.acquire = self._semlock.acquire
        self.release = self._semlock.release

    def __enter__(self):
        return self._semlock.__enter__()

    def __exit__(self, *args):
        return self._semlock.__exit__(*args)

        assert_spawning(self)
        sl = self._semlock
        return (Popen.duplicate_for_child(sl.handle), sl.kind, sl.maxvalue)

        self._semlock = _multiprocessing.SemLock._rebuild(*state)
        debug('recreated blocker with handle %r' % state[0])

#
# Semaphore
#

class Semaphore(SemLock):

    def __init__(self, value=1):
        SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX)

    def get_value(self):
        return self._semlock._get_value()

    def __repr__(self):
        try:
            value = self._semlock._get_value()
        except Exception:
            value = 'unknown'
        return '<Semaphore(value=%s)>' % value

#
# Bounded semaphore
#

class BoundedSemaphore(Semaphore):

        SemLock.__init__(self, SEMAPHORE, value, value)

        try:
        return '<BoundedSemaphore(value=%s, maxvalue=%s)>' % \
               (value, self._semlock.maxvalue)

#
# Non-recursive lock
#

class Lock(SemLock):

    def __init__(self):
        SemLock.__init__(self, SEMAPHORE, 1, 1)

        try:
            if self._semlock._is_mine():
                name = current_process().name
                if threading.current_thread().name != 'MainThread':
                    name += '|' + threading.current_thread().name
            elif self._semlock._get_value() == 1:
                name = 'None'
            elif self._semlock._count() > 0:
                name = 'SomeOtherThread'
            else:
                name = 'SomeOtherProcess'
            name = 'unknown'
        return '<Lock(owner=%s)>' % name

#
# Recursive lock
#

class RLock(SemLock):

        SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1)

        try:
                count = self._semlock._count()
                name, count = 'None', 0
                name, count = 'SomeOtherThread', 'nonzero'
            else:
                name, count = 'SomeOtherProcess', 'nonzero'
            name, count = 'unknown', 'unknown'
        return '<RLock(%s, %s)>' % (name, count)

#
# Condition variable
#

class Condition(object):

    def __init__(self, lock=None):
        self._lock = lock or RLock()
        self._sleeping_count = Semaphore(0)
        self._woken_count = Semaphore(0)
        self._wait_semaphore = Semaphore(0)

        return (self._lock, self._sleeping_count,
                self._woken_count, self._wait_semaphore)

        (self._lock, self._sleeping_count,
         self._woken_count, self._wait_semaphore) = state

        return self._lock.__enter__()

        return self._lock.__exit__(*args)

        self.acquire = self._lock.acquire
        self.release = self._lock.release

        try:
            num_waiters = (self._sleeping_count._semlock._get_value() -
                           self._woken_count._semlock._get_value())
            num_waiters = 'unknown'
        return '<Condition(%s, %s)>' % (self._lock, num_waiters)

    def wait(self, timeout=None):
        assert self._lock._semlock._is_mine(), \
               'must acquire() condition before using wait()'

        # indicate that this thread is going to sleep
        self._sleeping_count.release()

        # release lock
        count = self._lock._semlock._count()
        for i in xrange(count):
            self._lock.release()

        try:
            # wait for notification or timeout
            self._wait_semaphore.acquire(True, timeout)
        finally:
            # indicate that this thread has woken
            self._woken_count.release()

            # reacquire lock
            for i in xrange(count):
                self._lock.acquire()

    def notify(self):
        assert self._lock._semlock._is_mine(), 'lock is not owned'
        assert not self._wait_semaphore.acquire(False)

        # to take account of timeouts since last notify() we subtract
        # woken_count from sleeping_count and rezero woken_count
        while self._woken_count.acquire(False):
            res = self._sleeping_count.acquire(False)
            assert res

        if self._sleeping_count.acquire(False): # try grabbing a sleeper
            self._wait_semaphore.release()      # wake up one sleeper
            self._woken_count.acquire()         # wait for the sleeper to wake

            # rezero _wait_semaphore in case a timeout just happened
            self._wait_semaphore.acquire(False)

    def notify_all(self):

        # to take account of timeouts since last notify*() we subtract
            assert res

        sleepers = 0
        while self._sleeping_count.acquire(False):
            self._wait_semaphore.release()        # wake up one sleeper
            sleepers += 1

        if sleepers:
            for i in xrange(sleepers):
                self._woken_count.acquire()       # wait for a sleeper to wake

            # rezero wait_semaphore in case some timeouts just happened
            while self._wait_semaphore.acquire(False):
                pass

#
# Event
#

class Event(object):

        self._cond = Condition(Lock())
        self._flag = Semaphore(0)

    def is_set(self):
        self._cond.acquire()
        try:
            if self._flag.acquire(False):
                self._flag.release()
                return True
            return False
        finally:
            self._cond.release()

    def set(self):
        try:
            self._flag.acquire(False)
            self._flag.release()
            self._cond.notify_all()
        finally:

    def clear(self):
        try:
        finally:

        try:
            else:
                self._cond.wait(timeout)

        finally:

"""
The test cases for various inference algorithms
"""

import unittest, itertools
import GPy

class InferenceXTestCase(unittest.TestCase):
    
    def genData(self):
        D1,D2,N = 12,12,50
        np.random.seed(1234)
    
        x = np.linspace(0, 4 * np.pi, N)[:, None]
        s1 = np.vectorize(lambda x: np.sin(x))
        s2 = np.vectorize(lambda x: np.cos(x)**2)
        s3 = np.vectorize(lambda x:-np.exp(-np.cos(2 * x)))
        sS = np.vectorize(lambda x: np.cos(x))
    
        s1 = s1(x)
        s2 = s2(x)
        s3 = s3(x)
        sS = sS(x)
    
        s1 -= s1.mean(); s1 /= s1.std(0)
        s2 -= s2.mean(); s2 /= s2.std(0)
        s3 -= s3.mean(); s3 /= s3.std(0)
        sS -= sS.mean(); sS /= sS.std(0)
    
        S1 = np.hstack([s1, sS])
        S2 = np.hstack([s3, sS])
    
        P1 = np.random.randn(S1.shape[1], D1)
        P2 = np.random.randn(S2.shape[1], D2)
    
        Y1 = S1.dot(P1)
        Y2 = S2.dot(P2)
    
        Y1 += .01 * np.random.randn(*Y1.shape)
        Y2 += .01 * np.random.randn(*Y2.shape)
    
        Y1 -= Y1.mean(0)
        Y2 -= Y2.mean(0)
        Y1 /= Y1.std(0)
        Y2 /= Y2.std(0)

        slist = [s1, s2, s3, sS]
        slist_names = ["s1", "s2", "s3", "sS"]
        Ylist = [Y1, Y2]
        
        return Ylist
    
    def test_inferenceX_BGPLVM(self):
        Ys = self.genData()
        m = GPy.models.BayesianGPLVM(Ys[0],5,kernel=GPy.kern.Linear(5,ARD=True))
        
        x,mi = m.infer_newX(m.Y, optimize=False)
        self.assertTrue(mi.checkgrad())
        
        m.optimize(max_iters=10000)
        x,mi = m.infer_newX(m.Y)

        self.assertTrue(np.allclose(m.X.mean, mi.X.mean))
        self.assertTrue(np.allclose(m.X.variance, mi.X.variance))

    def test_inferenceX_GPLVM(self):
        m = GPy.models.GPLVM(Ys[0],3,kernel=GPy.kern.RBF(3,ARD=True))
        
#         m.optimize(max_iters=10000)
#         x,mi = m.infer_newX(m.Y)
#         self.assertTrue(np.allclose(m.X, x))
        
from sparse_gp import SparseGP
from ..inference.latent_function_inference.var_dtc_parallel import update_gradients, VarDTC_minibatch

import logging
logger = logging.getLogger("sparse gp mpi")

class SparseGP_MPI(SparseGP):
    """
    A general purpose Sparse GP model with MPI parallelization support

    This model allows (approximate) inference using variational DTC or FITC
    (Gaussian likelihoods) as well as non-conjugate sparse methods based on
    these.

    :param X: inputs
    :type X: np.ndarray (num_data x input_dim)
    :param likelihood: a likelihood instance, containing the observed data
    :type likelihood: GPy.likelihood.(Gaussian | EP | Laplace)
    :param kernel: the kernel (covariance function). See link kernels
    :type kernel: a GPy.kern.kern instance
    :param X_variance: The uncertainty in the measurements of X (Gaussian variance)
    :type X_variance: np.ndarray (num_data x input_dim) | None
    :param Z: inducing inputs
    :type Z: np.ndarray (num_inducing x input_dim)
    :param num_inducing: Number of inducing points (optional, default 10. Ignored if Z is not None)
    :param mpi_comm: The communication group of MPI, e.g. mpi4py.MPI.COMM_WORLD
    :type mpi_comm: mpi4py.MPI.Intracomm

    """

    def __init__(self, X, Y, Z, kernel, likelihood, variational_prior=None, inference_method=None, name='sparse gp mpi', Y_metadata=None, mpi_comm=None, normalizer=False):
        self._IN_OPTIMIZATION_ = False
        if mpi_comm != None:
            if inference_method is None:
                inference_method = VarDTC_minibatch(mpi_comm=mpi_comm)
            else:
                assert isinstance(inference_method, VarDTC_minibatch), 'inference_method has to support MPI!'

        super(SparseGP_MPI, self).__init__(X, Y, Z, kernel, likelihood, inference_method=inference_method, name=name, Y_metadata=Y_metadata, normalizer=normalizer)
        self.update_model(False)
        
        if variational_prior is not None:
            self.link_parameter(variational_prior)
            
        self.mpi_comm = mpi_comm
        # Manage the data (Y) division
            from ..util.parallel import divide_data
            N_start, N_end, N_list = divide_data(Y.shape[0], mpi_comm.rank, mpi_comm.size)
            self.N_range = (N_start, N_end)
            self.N_list = np.array(N_list)
            self.Y_local = self.Y[N_start:N_end]
            print 'MPI RANK '+str(self.mpi_comm.rank)+' with the data range '+str(self.N_range)
            mpi_comm.Bcast(self.param_array, root=0)
        self.update_model(True)

        dc = super(SparseGP_MPI, self).__getstate__()
        dc['mpi_comm'] = None
        if self.mpi_comm != None:
            del dc['N_range']
            del dc['N_list']
            del dc['Y_local']
        if 'normalizer' not in dc:
            dc['normalizer'] = None
            dc['Y_normalized'] = dc['Y']
        return dc

    #=====================================================
    # The MPI parallelization
    #     - can move to model at some point

    @SparseGP.optimizer_array.setter
    def optimizer_array(self, p):
            if self._IN_OPTIMIZATION_ and self.mpi_comm.rank==0:
                self.mpi_comm.Bcast(np.int32(1),root=0)
            self.mpi_comm.Bcast(p, root=0)
        SparseGP.optimizer_array.fset(self,p)

        self._IN_OPTIMIZATION_ = True
        if self.mpi_comm==None:
            super(SparseGP_MPI, self).optimize(optimizer,start,**kwargs)
        elif self.mpi_comm.rank==0:
            self.mpi_comm.Bcast(np.int32(-1),root=0)
        elif self.mpi_comm.rank>0:
            x = self.optimizer_array.copy()
            flag = np.empty(1,dtype=np.int32)
            while True:
                self.mpi_comm.Bcast(flag,root=0)
                if flag==1:
                    try:
                        self.optimizer_array = x
                        self._fail_count = 0
                    except (LinAlgError, ZeroDivisionError, ValueError):
                        if self._fail_count >= self._allowed_failures:
                            raise
                        self._fail_count += 1
                elif flag==-1:
                    break
                else:
                    self._IN_OPTIMIZATION_ = False
                    raise Exception("Unrecognizable flag for synchronization!")

            update_gradients(self, mpi_comm=self.mpi_comm)
        else:
            super(SparseGP_MPI,self).parameters_changed()

from ...util.linalg import pdinv,jitchol,DSYR,tdot,dtrtrs, dpotrs
from posterior import Posterior
from . import LatentFunctionInference

class EP(LatentFunctionInference):
    def __init__(self, epsilon=1e-6, eta=1., delta=1.):
        """
        The expectation-propagation algorithm.
        For nomenclature see Rasmussen & Williams 2006.

        :param epsilon: Convergence criterion, maximum squared difference allowed between mean updates to stop iterations (float)
        :type epsilon: float
        :param eta: parameter for fractional EP updates.
        :type eta: float64
        :param delta: damping EP updates factor.
        :type delta: float64
        """
        self.epsilon, self.eta, self.delta = epsilon, eta, delta
        self.reset()

    def reset(self):
        self.old_mutilde, self.old_vtilde = None, None
        self._ep_approximation = None

    def on_optimization_start(self):

    def on_optimization_end(self):
        # TODO: update approximation in the end as well? Maybe even with a switch?
        pass

    def inference(self, kern, X, likelihood, Y, Y_metadata=None, Z=None):
        num_data, output_dim = Y.shape
        assert output_dim ==1, "ep in 1D only (for now!)"

        K = kern.K(X)

        if self._ep_approximation is None:
            mu, Sigma, mu_tilde, tau_tilde, Z_hat = self._ep_approximation = self.expectation_propagation(K, Y, likelihood, Y_metadata)
        else:
            mu, Sigma, mu_tilde, tau_tilde, Z_hat = self._ep_approximation

        Wi, LW, LWi, W_logdet = pdinv(K + np.diag(1./tau_tilde))

        alpha, _ = dpotrs(LW, mu_tilde, lower=1)

        log_marginal =  0.5*(-num_data * log_2_pi - W_logdet - np.sum(alpha * mu_tilde)) # TODO: add log Z_hat??

        dL_dK = 0.5 * (tdot(alpha[:,None]) - Wi)

        dL_dthetaL = np.zeros(likelihood.size)#TODO: derivatives of the likelihood parameters

        return Posterior(woodbury_inv=Wi, woodbury_vector=alpha, K=K), log_marginal, {'dL_dK':dL_dK, 'dL_dthetaL':dL_dthetaL}

    def expectation_propagation(self, K, Y, likelihood, Y_metadata):

        num_data, data_dim = Y.shape
        assert data_dim == 1, "This EP methods only works for 1D outputs"

        #Initial values - Posterior distribution parameters: q(f|X,Y) = N(f|mu,Sigma)
        mu = np.zeros(num_data)
        Sigma = K.copy()

        #Initial values - Marginal moments
        Z_hat = np.empty(num_data,dtype=np.float64)
        mu_hat = np.empty(num_data,dtype=np.float64)
        sigma2_hat = np.empty(num_data,dtype=np.float64)

        #initial values - Gaussian factors
        if self.old_mutilde is None:
            tau_tilde, mu_tilde, v_tilde = np.zeros((3, num_data))
        else:
            assert old_mutilde.size == num_data, "data size mis-match: did you change the data? try resetting!"
            mu_tilde, v_tilde = self.old_mutilde, self.old_vtilde
            tau_tilde = v_tilde/mu_tilde

        #Approximation
        tau_diff = self.epsilon + 1.
        v_diff = self.epsilon + 1.
       	iterations = 0
        while (tau_diff > self.epsilon) or (v_diff > self.epsilon):
            update_order = np.random.permutation(num_data)
            for i in update_order:
                #Cavity distribution parameters
                tau_cav = 1./Sigma[i,i] - self.eta*tau_tilde[i]
                v_cav = mu[i]/Sigma[i,i] - self.eta*v_tilde[i]
                #Marginal moments
                Z_hat[i], mu_hat[i], sigma2_hat[i] = likelihood.moments_match_ep(Y[i], tau_cav, v_cav)#, Y_metadata=None)#=(None if Y_metadata is None else Y_metadata[i]))
                #Site parameters update
                delta_tau = self.delta/self.eta*(1./sigma2_hat[i] - 1./Sigma[i,i])
                delta_v = self.delta/self.eta*(mu_hat[i]/sigma2_hat[i] - mu[i]/Sigma[i,i])
                tau_tilde[i] += delta_tau
                v_tilde[i] += delta_v
                #Posterior distribution parameters update
                DSYR(Sigma, Sigma[:,i].copy(), -delta_tau/(1.+ delta_tau*Sigma[i,i]))
                mu = np.dot(Sigma, v_tilde)

            #(re) compute Sigma and mu using full Cholesky decompy
            tau_tilde_root = np.sqrt(tau_tilde)
            Sroot_tilde_K = tau_tilde_root[:,None] * K
            B = np.eye(num_data) + Sroot_tilde_K * tau_tilde_root[None,:]
            L = jitchol(B)
            V, _ = dtrtrs(L, Sroot_tilde_K, lower=1)
            Sigma = K - np.dot(V.T,V)
            mu = np.dot(Sigma,v_tilde)

            #monitor convergence
            if iterations>0:
                tau_diff = np.mean(np.square(tau_tilde-tau_tilde_old))
                v_diff = np.mean(np.square(v_tilde-v_tilde_old))
            tau_tilde_old = tau_tilde.copy()
            v_tilde_old = v_tilde.copy()

            iterations += 1

        mu_tilde = v_tilde/tau_tilde
        return mu, Sigma, mu_tilde, tau_tilde, Z_hat

"""
The Maniforld Relevance Determination model with the spike-and-slab prior
"""

from ..core import Model
from .ss_gplvm import SSGPLVM

class SSMRD(Model):
    
                 inference_method=None, likelihoods=None, name='ss_mrd', Ynames=None):
        super(SSMRD, self).__init__(name)
        
        self.updates = False
        self.models = [SSGPLVM(y, input_dim, X=X, X_variance=X_variance, num_inducing=num_inducing,Z=Z,init=initx,
                               kernel=kernel.copy() if kernel else None,inference_method=inference_method,likelihood=likelihoods,
                               name='model_'+str(i)) for i,y in enumerate(Ylist)]
        self.add_parameters(*(self.models))
        
        [[[self.models[m].X.mean[i,j:j+1].tie('mean_'+str(i)+'_'+str(j)) for m in xrange(len(self.models))] for j in xrange(self.models[0].X.mean.shape[1])] 
         for i in xrange(self.models[0].X.mean.shape[0])]
        [[[self.models[m].X.variance[i,j:j+1].tie('var_'+str(i)+'_'+str(j)) for m in xrange(len(self.models))] for j in xrange(self.models[0].X.variance.shape[1])] 
         for i in xrange(self.models[0].X.variance.shape[0])]
        
        self.updates = True
        
        super(SSMRD, self).parameters_changed()        
        self._log_marginal_likelihood = sum([m._log_marginal_likelihood for m in self.models])

# Copyright (c) 2012-2014 The GPy authors (see AUTHORS.txt)

from scipy import stats,special
import scipy as sp
import link_functions
from ..util.misc import chain_1, chain_2, chain_3
from scipy.integrate import quad
import warnings
from ..core.parameterization import Parameterized

class Likelihood(Parameterized):
    """
    Likelihood base class, used to defing p(y|f).

    All instances use _inverse_ link functions, which can be swapped out. It is
    expected that inheriting classes define a default inverse link function

    To use this class, inherit and define missing functionality.

    Inheriting classes *must* implement:
       pdf_link : a bound method which turns the output of the link function into the pdf
       logpdf_link : the logarithm of the above

    To enable use with EP, inheriting classes *must* define:
       TODO: a suitable derivative function for any parameters of the class
    It is also desirable to define:
       moments_match_ep : a function to compute the EP moments If this isn't defined, the moments will be computed using 1D quadrature.

    To enable use with Laplace approximation, inheriting classes *must* define:
       Some derivative functions *AS TODO*

    For exact Gaussian inference, define *JH TODO*

    """
    def __init__(self, gp_link, name):
        super(Likelihood, self).__init__(name)
        assert isinstance(gp_link,link_functions.GPTransformation), "gp_link is not a valid GPTransformation."
        self.gp_link = gp_link
        self.log_concave = False

    def _gradients(self,partial):
        return np.zeros(0)

    def update_gradients(self, partial):
        if self.size > 0:
            raise NotImplementedError('Must be implemented for likelihoods with parameters to be optimized')

    def _preprocess_values(self,Y):
        """
        In case it is needed, this function assess the output values or makes any pertinent transformation on them.

        :param Y: observed output
        :type Y: Nx1 numpy.darray

        """
        return Y

    def conditional_mean(self, gp):
        """
        The mean of the random variable conditioned on one value of the GP
        """

    def conditional_variance(self, gp):
        """
        The variance of the random variable conditioned on one value of the GP
        """

    def log_predictive_density(self, y_test, mu_star, var_star):
        """
        Calculation of the log predictive density

        .. math:
            p(y_{*}|D) = p(y_{*}|f_{*})p(f_{*}|\mu_{*}\\sigma^{2}_{*})

        :param y_test: test observations (y_{*})
        :type y_test: (Nx1) array
        :param mu_star: predictive mean of gaussian p(f_{*}|mu_{*}, var_{*})
        :type mu_star: (Nx1) array
        :param var_star: predictive variance of gaussian p(f_{*}|mu_{*}, var_{*})
        :type var_star: (Nx1) array
        """
        assert y_test.shape==mu_star.shape
        assert y_test.shape==var_star.shape
        assert y_test.shape[1] == 1
        def integral_generator(y, m, v):
            """Generate a function which can be integrated to give p(Y*|Y) = int p(Y*|f*)p(f*|Y) df*"""
            def f(f_star):
                return self.pdf(f_star, y)*np.exp(-(1./(2*v))*np.square(m-f_star))
            return f

        scaled_p_ystar, accuracy = zip(*[quad(integral_generator(y, m, v), -np.inf, np.inf) for y, m, v in zip(y_test.flatten(), mu_star.flatten(), var_star.flatten())])
        scaled_p_ystar = np.array(scaled_p_ystar).reshape(-1,1)
        p_ystar = scaled_p_ystar/np.sqrt(2*np.pi*var_star)
        return np.log(p_ystar)

    def _moments_match_ep(self,obs,tau,v):
        """
        Calculation of moments using quadrature

        :param obs: observed output
        :param tau: cavity distribution 1st natural parameter (precision)
        :param v: cavity distribution 2nd natural paramenter (mu*precision)
        """
        #Compute first integral for zeroth moment.
        #NOTE constant np.sqrt(2*pi/tau) added at the end of the function
        mu = v/tau
        def int_1(f):
            return self.pdf(f, obs)*np.exp(-0.5*tau*np.square(mu-f))
        z_scaled, accuracy = quad(int_1, -np.inf, np.inf)

        #Compute second integral for first moment
        def int_2(f):
            return f*self.pdf(f, obs)*np.exp(-0.5*tau*np.square(mu-f))
        mean, accuracy = quad(int_2, -np.inf, np.inf)
        mean /= z_scaled

        #Compute integral for variance
        def int_3(f):
            return (f**2)*self.pdf(f, obs)*np.exp(-0.5*tau*np.square(mu-f))
        Ef2, accuracy = quad(int_3, -np.inf, np.inf)
        Ef2 /= z_scaled
        variance = Ef2 - mean**2

        #Add constant to the zeroth moment
        #NOTE: this constant is not needed in the other moments because it cancells out.
        z = z_scaled/np.sqrt(2*np.pi/tau)

        return z, mean, variance

    def variational_expectations(self, Y, m, v, gh_points=None):
        """
        Use Gauss-Hermite Quadrature to compute 

           E_p(f) [ log p(y|f) ]
           d/dm E_p(f) [ log p(y|f) ]
           d/dv E_p(f) [ log p(y|f) ]

        where p(f) is a Gaussian with mean m and variance v. The shapes of Y, m and v should match.

        if no gh_points are passed, we construct them using defualt options
        """

        if gh_points is None:
            gh_x, gh_w = np.polynomial.hermite.hermgauss(12)
        else:
            gh_x, gh_w = gh_points

        shape = m.shape
        m,v,Y = m.flatten(), v.flatten(), Y.flatten()

        #make a grid of points
        X = gh_x[None,:]*np.sqrt(2.*v[:,None]) + m[:,None]

        #evaluate the likelhood for the grid. First ax indexes the data (and mu, var) and the second indexes the grid.
        # broadcast needs to be handled carefully. 
        logp = self.logpdf(X,Y[:,None])
        dlogp_dx = self.dlogpdf_df(X, Y[:,None])
        d2logp_dx2 = self.d2logpdf_df2(X, Y[:,None])

        #clipping for numerical stability
        logp = np.clip(logp,-1e6,1e6)
        dlogp_dx = np.clip(dlogp_dx,-1e6,1e6)
        d2logp_dx2 = np.clip(d2logp_dx2,-1e6,1e6)

        #average over the gird to get derivatives of the Gaussian's parameters
        F = np.dot(logp, gh_w)
        dF_dm = np.dot(dlogp_dx, gh_w)
        dF_dv = np.dot(d2logp_dx2, gh_w)/2.

        if np.any(np.isnan(dF_dv)) or np.any(np.isinf(dF_dv)):
            stop
        if np.any(np.isnan(dF_dm)) or np.any(np.isinf(dF_dm)):
            stop

        return F.reshape(*shape), dF_dm.reshape(*shape), dF_dv.reshape(*shape)

    def predictive_mean(self, mu, variance, Y_metadata=None):
        """
        Quadrature calculation of the predictive mean: E(Y_star|Y) = E( E(Y_star|f_star, Y) )

        :param mu: mean of posterior
        :param sigma: standard deviation of posterior

        """
        #conditional_mean: the edpected value of y given some f, under this likelihood
        def int_mean(f,m,v):
            p = np.exp(-(0.5/v)*np.square(f - m))
            #If p is zero then conditional_mean will overflow
            if p < 1e-10:
                return 0.
            else:
                return self.conditional_mean(f)*p
        scaled_mean = [quad(int_mean, -np.inf, np.inf,args=(mj,s2j))[0] for mj,s2j in zip(mu,variance)]
        mean = np.array(scaled_mean)[:,None] / np.sqrt(2*np.pi*(variance))

        return mean

    def _conditional_mean(self, f):
        """Quadrature calculation of the conditional mean: E(Y_star|f)"""
        raise NotImplementedError, "implement this function to make predictions"

    def predictive_variance(self, mu,variance, predictive_mean=None, Y_metadata=None):
        """
        Approximation to the predictive variance: V(Y_star)

        The following variance decomposition is used:
        V(Y_star) = E( V(Y_star|f_star) ) + V( E(Y_star|f_star) )

        :predictive_mean: output's predictive mean, if None _predictive_mean function will be called.

        """
        #sigma2 = sigma**2
        normalizer = np.sqrt(2*np.pi*variance)

        # E( V(Y_star|f_star) )
        def int_var(f,m,v):
            #If p is zero then conditional_variance will overflow
                return 0.
            else:
                return self.conditional_variance(f)*p
        scaled_exp_variance = [quad(int_var, -np.inf, np.inf,args=(mj,s2j))[0] for mj,s2j in zip(mu,variance)]
        exp_var = np.array(scaled_exp_variance)[:,None] / normalizer

        #V( E(Y_star|f_star) ) =  E( E(Y_star|f_star)**2 ) - E( E(Y_star|f_star) )**2

        #E( E(Y_star|f_star) )**2
        if predictive_mean is None:
            predictive_mean = self.predictive_mean(mu,variance)
        predictive_mean_sq = predictive_mean**2

        #E( E(Y_star|f_star)**2 )
        def int_pred_mean_sq(f,m,v,predictive_mean_sq):
            #If p is zero then conditional_mean**2 will overflow
                return 0.
            else:
                return self.conditional_mean(f)**2*p

        scaled_exp_exp2 = [quad(int_pred_mean_sq, -np.inf, np.inf,args=(mj,s2j,pm2j))[0] for mj,s2j,pm2j in zip(mu,variance,predictive_mean_sq)]
        exp_exp2 = np.array(scaled_exp_exp2)[:,None] / normalizer

        var_exp = exp_exp2 - predictive_mean_sq

        # V(Y_star) = E[ V(Y_star|f_star) ] + V[ E(Y_star|f_star) ]
        # V(Y_star) = E[ V(Y_star|f_star) ] + E(Y_star**2|f_star) - E[Y_star|f_star]**2
        return exp_var + var_exp

    def pdf_link(self, inv_link_f, y, Y_metadata=None):

    def logpdf_link(self, inv_link_f, y, Y_metadata=None):

    def dlogpdf_dlink(self, inv_link_f, y, Y_metadata=None):

    def d2logpdf_dlink2(self, inv_link_f, y, Y_metadata=None):

    def d3logpdf_dlink3(self, inv_link_f, y, Y_metadata=None):

    def dlogpdf_link_dtheta(self, inv_link_f, y, Y_metadata=None):

    def dlogpdf_dlink_dtheta(self, inv_link_f, y, Y_metadata=None):

    def d2logpdf_dlink2_dtheta(self, inv_link_f, y, Y_metadata=None):

    def pdf(self, f, y, Y_metadata=None):
        """
        Evaluates the link function link(f) then computes the likelihood (pdf) using it

        .. math:
            p(y|\\lambda(f))

        :param f: latent variables f
        :type f: Nx1 array
        :param y: data
        :type y: Nx1 array
        :param Y_metadata: Y_metadata which is not used in student t distribution - not used
        :returns: likelihood evaluated for this point
        :rtype: float
        """
        inv_link_f = self.gp_link.transf(f)
        return self.pdf_link(inv_link_f, y, Y_metadata=Y_metadata)

    def logpdf(self, f, y, Y_metadata=None):
        """
        Evaluates the link function link(f) then computes the log likelihood (log pdf) using it

        .. math:
            \\log p(y|\\lambda(f))

        :returns: log likelihood evaluated for this point
        """
        return self.logpdf_link(inv_link_f, y, Y_metadata=Y_metadata)

    def dlogpdf_df(self, f, y, Y_metadata=None):
        """
        Evaluates the link function link(f) then computes the derivative of log likelihood using it
        Uses the Faa di Bruno's formula for the chain rule

        .. math::
            \\frac{d\\log p(y|\\lambda(f))}{df} = \\frac{d\\log p(y|\\lambda(f))}{d\\lambda(f)}\\frac{d\\lambda(f)}{df}

        :returns: derivative of log likelihood evaluated for this point
        :rtype: 1xN array
        """
        dlogpdf_dlink = self.dlogpdf_dlink(inv_link_f, y, Y_metadata=Y_metadata)
        dlink_df = self.gp_link.dtransf_df(f)
        return chain_1(dlogpdf_dlink, dlink_df)

    def d2logpdf_df2(self, f, y, Y_metadata=None):
        """
        Evaluates the link function link(f) then computes the second derivative of log likelihood using it

        .. math::
            \\frac{d^{2}\\log p(y|\\lambda(f))}{df^{2}} = \\frac{d^{2}\\log p(y|\\lambda(f))}{d^{2}\\lambda(f)}\\left(\\frac{d\\lambda(f)}{df}\\right)^{2} + \\frac{d\\log p(y|\\lambda(f))}{d\\lambda(f)}\\frac{d^{2}\\lambda(f)}{df^{2}}

        :returns: second derivative of log likelihood evaluated for this point (diagonal only)
        """
        d2logpdf_dlink2 = self.d2logpdf_dlink2(inv_link_f, y, Y_metadata=Y_metadata)
        d2link_df2 = self.gp_link.d2transf_df2(f)
        return chain_2(d2logpdf_dlink2, dlink_df, dlogpdf_dlink, d2link_df2)

    def d3logpdf_df3(self, f, y, Y_metadata=None):
        """
        Evaluates the link function link(f) then computes the third derivative of log likelihood using it

        .. math::
            \\frac{d^{3}\\log p(y|\\lambda(f))}{df^{3}} = \\frac{d^{3}\\log p(y|\\lambda(f)}{d\\lambda(f)^{3}}\\left(\\frac{d\\lambda(f)}{df}\\right)^{3} + 3\\frac{d^{2}\\log p(y|\\lambda(f)}{d\\lambda(f)^{2}}\\frac{d\\lambda(f)}{df}\\frac{d^{2}\\lambda(f)}{df^{2}} + \\frac{d\\log p(y|\\lambda(f)}{d\\lambda(f)}\\frac{d^{3}\\lambda(f)}{df^{3}}

        :returns: third derivative of log likelihood evaluated for this point
        """
        d3logpdf_dlink3 = self.d3logpdf_dlink3(inv_link_f, y, Y_metadata=Y_metadata)
        d3link_df3 = self.gp_link.d3transf_df3(f)
        return chain_3(d3logpdf_dlink3, dlink_df, d2logpdf_dlink2, d2link_df2, dlogpdf_dlink, d3link_df3)

    def dlogpdf_dtheta(self, f, y, Y_metadata=None):
        """
        TODO: Doc strings
        """
            inv_link_f = self.gp_link.transf(f)
            return self.dlogpdf_link_dtheta(inv_link_f, y, Y_metadata=Y_metadata)
        else:
            # There are no parameters so return an empty array for derivatives
            return np.zeros([1, 0])

    def dlogpdf_df_dtheta(self, f, y, Y_metadata=None):
        """
        """
            dlink_df = self.gp_link.dtransf_df(f)
            dlogpdf_dlink_dtheta = self.dlogpdf_dlink_dtheta(inv_link_f, y, Y_metadata=Y_metadata)
            return chain_1(dlogpdf_dlink_dtheta, dlink_df)
        else:
            return np.zeros([f.shape[0], 0])

    def d2logpdf_df2_dtheta(self, f, y, Y_metadata=None):
        """
        """
            d2link_df2 = self.gp_link.d2transf_df2(f)
            d2logpdf_dlink2_dtheta = self.d2logpdf_dlink2_dtheta(inv_link_f, y, Y_metadata=Y_metadata)
            return chain_2(d2logpdf_dlink2_dtheta, dlink_df, dlogpdf_dlink_dtheta, d2link_df2)
        else:

    def _laplace_gradients(self, f, y, Y_metadata=None):
        dlogpdf_dtheta = self.dlogpdf_dtheta(f, y, Y_metadata=Y_metadata)
        dlogpdf_df_dtheta = self.dlogpdf_df_dtheta(f, y, Y_metadata=Y_metadata)
        d2logpdf_df2_dtheta = self.d2logpdf_df2_dtheta(f, y, Y_metadata=Y_metadata)

        #Parameters are stacked vertically. Must be listed in same order as 'get_param_names'
        # ensure we have gradients for every parameter we want to optimize
        assert len(dlogpdf_dtheta) == self.size #1 x num_param array
        assert dlogpdf_df_dtheta.shape[1] == self.size #f x num_param matrix
        assert d2logpdf_df2_dtheta.shape[1] == self.size #f x num_param matrix

        return dlogpdf_dtheta, dlogpdf_df_dtheta, d2logpdf_df2_dtheta

    def predictive_values(self, mu, var, full_cov=False, Y_metadata=None):
        """
        Compute  mean, variance of the  predictive distibution.

        :param mu: mean of the latent variable, f, of posterior
        :param var: variance of the latent variable, f, of posterior
        :param full_cov: whether to use the full covariance or just the diagonal
        :type full_cov: Boolean
        """

        pred_mean = self.predictive_mean(mu, var, Y_metadata)
        pred_var = self.predictive_variance(mu, var, pred_mean, Y_metadata)

        return pred_mean, pred_var

    def predictive_quantiles(self, mu, var, quantiles, Y_metadata=None):
        #compute the quantiles by sampling!!!
        N_samp = 1000
        s = np.random.randn(mu.shape[0], N_samp)*np.sqrt(var) + mu
        #ss_f = s.flatten()
        #ss_y = self.samples(ss_f, Y_metadata)
        ss_y = self.samples(s, Y_metadata)
        #ss_y = ss_y.reshape(mu.shape[0], N_samp)

        return [np.percentile(ss_y ,q, axis=1)[:,None] for q in quantiles]

    def samples(self, gp, Y_metadata=None):
        """
        Returns a set of samples of observations based on a given value of the latent variable.

        :param gp: latent variable
        """

from functools import wraps

def silence_errors(f):
    """
    This wraps a function and it silences numpy errors that
    happen during the execution. After the function has exited, it restores
    the previous state of the warnings.
    """
    @wraps(f)
    def wrapper(*args, **kwds):
        status = np.seterr(all='ignore')
        result = f(*args, **kwds)
        np.seterr(**status)
        return result
    return wrapper

from ...util.linalg import mdot, jitchol, backsub_both_sides, tdot, dtrtrs, dtrtri, dpotri, dpotrs, symmetrify
from ...util import diag
from ...core.parameterization.variational import VariationalPosterior
import logging, itertools
logger = logging.getLogger('vardtc')

class VarDTC(LatentFunctionInference):
    """
    An object for inference when the likelihood is Gaussian, but we want to do sparse inference.

    The function self.inference returns a Posterior object, which summarizes
    the posterior.

    For efficiency, we sometimes work with the cholesky of Y*Y.T. To save repeatedly recomputing this, we cache it.

    """
    const_jitter = 1e-6
    def __init__(self, limit=1):
        #self._YYTfactor_cache = caching.cache()
        from ...util.caching import Cacher
        self.limit = limit
        self.get_trYYT = Cacher(self._get_trYYT, limit)
        self.get_YYTfactor = Cacher(self._get_YYTfactor, limit)

    def set_limit(self, limit):
        self.get_trYYT.limit = limit
        self.get_YYTfactor.limit = limit

    def _get_trYYT(self, Y):
        return np.einsum("ij,ij->", Y, Y)
        # faster than, but same as:
        # return np.sum(np.square(Y))

        # has to be overridden, as Cacher objects cannot be pickled.
        return self.limit

        self.limit = state
        self.get_trYYT = Cacher(self._get_trYYT, self.limit)
        self.get_YYTfactor = Cacher(self._get_YYTfactor, self.limit)

    def _get_YYTfactor(self, Y):
        """
        find a matrix L which satisfies LLT = YYT.

        Note that L may have fewer columns than Y.
        """
        N, D = Y.shape
        if (N>=D):
            return Y.view(np.ndarray)
        else:
            return jitchol(tdot(Y))

    def get_VVTfactor(self, Y, prec):
        return Y * prec # TODO chache this, and make it effective

    def inference(self, kern, X, Z, likelihood, Y, Y_metadata=None, Lm=None, dL_dKmm=None):

        _, output_dim = Y.shape
        uncertain_inputs = isinstance(X, VariationalPosterior)

        #see whether we've got a different noise variance for each datum
        beta = 1./np.fmax(likelihood.gaussian_variance(Y_metadata), 1e-6)
        # VVT_factor is a matrix such that tdot(VVT_factor) = VVT...this is for efficiency!
        #self.YYTfactor = self.get_YYTfactor(Y)
        #VVT_factor = self.get_VVTfactor(self.YYTfactor, beta)
        het_noise = beta.size > 1
        if beta.ndim == 1:
            beta = beta[:, None]
        VVT_factor = beta*Y
        #VVT_factor = beta*Y
        trYYT = self.get_trYYT(Y)

        # do the inference:
        num_inducing = Z.shape[0]
        num_data = Y.shape[0]
        # kernel computations, using BGPLVM notation

        Kmm = kern.K(Z).copy()
        diag.add(Kmm, self.const_jitter)
        if Lm is None:
            Lm = jitchol(Kmm)

        # The rather complex computations of A, and the psi stats
        if uncertain_inputs:
            psi0 = kern.psi0(Z, X)
            psi1 = kern.psi1(Z, X)
            if het_noise:
                psi2_beta = np.sum([kern.psi2(Z,X[i:i+1,:]) * beta_i for i,beta_i in enumerate(beta)],0)
            else:
                psi2_beta = kern.psi2(Z,X) * beta
            LmInv = dtrtri(Lm)
            A = LmInv.dot(psi2_beta.dot(LmInv.T))
        else:
            psi0 = kern.Kdiag(X)
            psi1 = kern.K(X, Z)
                tmp = psi1 * (np.sqrt(beta))
            else:
            tmp, _ = dtrtrs(Lm, tmp.T, lower=1)
            A = tdot(tmp) #print A.sum()

        # factor B
        B = np.eye(num_inducing) + A
        LB = jitchol(B)
        psi1Vf = np.dot(psi1.T, VVT_factor)
        # back substutue C into psi1Vf
        tmp, _ = dtrtrs(Lm, psi1Vf, lower=1, trans=0)
        _LBi_Lmi_psi1Vf, _ = dtrtrs(LB, tmp, lower=1, trans=0)
        tmp, _ = dtrtrs(LB, _LBi_Lmi_psi1Vf, lower=1, trans=1)
        Cpsi1Vf, _ = dtrtrs(Lm, tmp, lower=1, trans=1)

        # data fit and derivative of L w.r.t. Kmm
        delit = tdot(_LBi_Lmi_psi1Vf)
        data_fit = np.trace(delit)
        DBi_plus_BiPBi = backsub_both_sides(LB, output_dim * np.eye(num_inducing) + delit)
        if dL_dKmm is None:
            delit = -0.5 * DBi_plus_BiPBi
            delit += -0.5 * B * output_dim
            delit += output_dim * np.eye(num_inducing)
            # Compute dL_dKmm
            dL_dKmm = backsub_both_sides(Lm, delit)

        # derivatives of L w.r.t. psi
        dL_dpsi0, dL_dpsi1, dL_dpsi2 = _compute_dL_dpsi(num_inducing, num_data, output_dim, beta, Lm,
            VVT_factor, Cpsi1Vf, DBi_plus_BiPBi,
            psi1, het_noise, uncertain_inputs)

        # log marginal likelihood
        log_marginal = _compute_log_marginal_likelihood(likelihood, num_data, output_dim, beta, het_noise,
            psi0, A, LB, trYYT, data_fit, Y)

        #noise derivatives
        dL_dR = _compute_dL_dR(likelihood,
            het_noise, uncertain_inputs, LB,
            _LBi_Lmi_psi1Vf, DBi_plus_BiPBi, Lm, A,
            psi0, psi1, beta,
            data_fit, num_data, output_dim, trYYT, Y, VVT_factor)

        dL_dthetaL = likelihood.exact_inference_gradients(dL_dR,Y_metadata)

        #put the gradients in the right places
            grad_dict = {'dL_dKmm': dL_dKmm,
                         'dL_dpsi0':dL_dpsi0,
                         'dL_dpsi1':dL_dpsi1,
                         'dL_dpsi2':dL_dpsi2,
                         'dL_dthetaL':dL_dthetaL}
        else:
                         'dL_dKdiag':dL_dpsi0,
                         'dL_dKnm':dL_dpsi1,

        #get sufficient things for posterior prediction
        #TODO: do we really want to do this in  the loop?
        if VVT_factor.shape[1] == Y.shape[1]:
            woodbury_vector = Cpsi1Vf # == Cpsi1V
        else:
            print 'foobar'
            import ipdb; ipdb.set_trace()
            psi1V = np.dot(Y.T*beta, psi1).T
            tmp, _ = dtrtrs(Lm, psi1V, lower=1, trans=0)
            tmp, _ = dpotrs(LB, tmp, lower=1)
            woodbury_vector, _ = dtrtrs(Lm, tmp, lower=1, trans=1)
        Bi, _ = dpotri(LB, lower=1)
        symmetrify(Bi)
        Bi = -dpotri(LB, lower=1)[0]
        diag.add(Bi, 1)

        woodbury_inv = backsub_both_sides(Lm, Bi)

        #construct a posterior object
        post = Posterior(woodbury_inv=woodbury_inv, woodbury_vector=woodbury_vector, K=Kmm, mean=None, cov=None, K_chol=Lm)
        return post, log_marginal, grad_dict

def _compute_dL_dpsi(num_inducing, num_data, output_dim, beta, Lm, VVT_factor, Cpsi1Vf, DBi_plus_BiPBi, psi1, het_noise, uncertain_inputs):
    dL_dpsi0 = -0.5 * output_dim * (beta* np.ones([num_data, 1])).flatten()
    dL_dpsi1 = np.dot(VVT_factor, Cpsi1Vf.T)
    dL_dpsi2_beta = 0.5 * backsub_both_sides(Lm, output_dim * np.eye(num_inducing) - DBi_plus_BiPBi)
    if het_noise:
            dL_dpsi2 = beta[:, None] * dL_dpsi2_beta[None, :, :]
        else:
            dL_dpsi1 += 2.*np.dot(dL_dpsi2_beta, (psi1 * beta).T).T
            dL_dpsi2 = None
    else:
        dL_dpsi2 = beta * dL_dpsi2_beta
        if not uncertain_inputs:
            # subsume back into psi1 (==Kmn)
            dL_dpsi1 += 2.*np.dot(psi1, dL_dpsi2)
    return dL_dpsi0, dL_dpsi1, dL_dpsi2

def _compute_dL_dR(likelihood, het_noise, uncertain_inputs, LB, _LBi_Lmi_psi1Vf, DBi_plus_BiPBi, Lm, A, psi0, psi1, beta, data_fit, num_data, output_dim, trYYT, Y, VVT_factr=None):
    # the partial derivative vector for the likelihood
    if likelihood.size == 0:
        # save computation here.
        dL_dR = None
    elif het_noise:
            raise NotImplementedError, "heteroscedatic derivates with uncertain inputs not implemented"
        else:
            #from ...util.linalg import chol_inv
            #LBi = chol_inv(LB)
            LBi, _ = dtrtrs(LB,np.eye(LB.shape[0]))

            Lmi_psi1, nil = dtrtrs(Lm, psi1.T, lower=1, trans=0)
            _LBi_Lmi_psi1, _ = dtrtrs(LB, Lmi_psi1, lower=1, trans=0)
            dL_dR = -0.5 * beta + 0.5 * VVT_factr**2
            dL_dR += 0.5 * output_dim * (psi0 - np.sum(Lmi_psi1**2,0))[:,None] * beta**2

            dL_dR += 0.5*np.sum(mdot(LBi.T,LBi,Lmi_psi1)*Lmi_psi1,0)[:,None]*beta**2

            dL_dR += -np.dot(_LBi_Lmi_psi1Vf.T,_LBi_Lmi_psi1).T * Y * beta**2
            dL_dR += 0.5*np.dot(_LBi_Lmi_psi1Vf.T,_LBi_Lmi_psi1).T**2 * beta**2
    else:
        # likelihood is not heteroscedatic
        dL_dR = -0.5 * num_data * output_dim * beta + 0.5 * trYYT * beta ** 2
        dL_dR += 0.5 * output_dim * (psi0.sum() * beta ** 2 - np.trace(A) * beta)
        dL_dR += beta * (0.5 * np.sum(A * DBi_plus_BiPBi) - data_fit)
    return dL_dR

def _compute_log_marginal_likelihood(likelihood, num_data, output_dim, beta, het_noise, psi0, A, LB, trYYT, data_fit, Y):
    #compute log marginal likelihood
        lik_1 = -0.5 * num_data * output_dim * np.log(2. * np.pi) + 0.5 * output_dim * np.sum(np.log(beta)) - 0.5 * np.sum(beta.ravel() * np.square(Y).sum(axis=-1))
        lik_2 = -0.5 * output_dim * (np.sum(beta.flatten() * psi0) - np.trace(A))
    else:
        lik_1 = -0.5 * num_data * output_dim * (np.log(2. * np.pi) - np.log(beta)) - 0.5 * beta * trYYT
        lik_2 = -0.5 * output_dim * (np.sum(beta * psi0) - np.trace(A))
    lik_3 = -output_dim * (np.sum(np.log(np.diag(LB))))
    lik_4 = 0.5 * data_fit
    log_marginal = lik_1 + lik_2 + lik_3 + lik_4
    return log_marginal

'''
Created on 24 Jul 2013

'''

class AxisEventController(object):
    def __init__(self, ax):
        self.ax = ax
        self.activate()
    def deactivate(self):
        for cb_class in self.ax.callbacks.callbacks.values():
            for cb_num in cb_class.keys():
                self.ax.callbacks.disconnect(cb_num)
    def activate(self):
        self.ax.callbacks.connect('xlim_changed', self.xlim_changed)
        self.ax.callbacks.connect('ylim_changed', self.ylim_changed)
    def xlim_changed(self, ax):
        pass
    def ylim_changed(self, ax):
        pass

class AxisChangedController(AxisEventController):
    '''
    Buffered control of axis limit changes
    '''
    _changing = False

    def __init__(self, ax, update_lim=None):
        '''
        Constructor
        '''
        super(AxisChangedController, self).__init__(ax)
        self._lim_ratio_threshold = update_lim or .95
        self._x_lim = self.ax.get_xlim()
        self._y_lim = self.ax.get_ylim()

    def update(self, ax):
        pass

        super(AxisChangedController, self).xlim_changed(ax)
        if not self._changing and self.lim_changed(ax.get_xlim(), self._x_lim):
            self._changing = True
            self._x_lim = ax.get_xlim()
            self.update(ax)
            self._changing = False

        super(AxisChangedController, self).ylim_changed(ax)
        if not self._changing and self.lim_changed(ax.get_ylim(), self._y_lim):
            self._y_lim = ax.get_ylim()

    def extent(self, lim):
        return numpy.subtract(*lim)

    def lim_changed(self, axlim, savedlim):
        axextent = self.extent(axlim)
        extent = self.extent(savedlim)
        lim_changed = ((axextent / extent) < self._lim_ratio_threshold ** 2
                       or (extent / axextent) < self._lim_ratio_threshold ** 2
                       or ((1 - (self.extent((axlim[0], savedlim[0])) / self.extent((savedlim[0], axlim[1]))))
                           < self._lim_ratio_threshold)
                       or ((1 - (self.extent((savedlim[0], axlim[0])) / self.extent((axlim[0], savedlim[1]))))
                       )
        return lim_changed

    def _buffer_lim(self, lim):
        # buffer_size = 1 - self._lim_ratio_threshold
        # extent = self.extent(lim)
        return lim

class BufferedAxisChangedController(AxisChangedController):
    def __init__(self, ax, plot_function, plot_limits, resolution=50, update_lim=None, **kwargs):
        """
        Buffered axis changed controller. Controls the buffer and handles update events for when the axes changed.
        
        Updated plotting will be after first reload (first time will be within plot limits, after that the limits will be buffered)
        
        :param plot_function:
            function to use for creating image for plotting (return ndarray-like)
            plot_function gets called with (2D!) Xtest grid if replotting required
        :type plot_function: function
        :param plot_limits:
            beginning plot limits [xmin, ymin, xmax, ymax]

        :param kwargs: additional kwargs are for pyplot.imshow(**kwargs)
        """
        super(BufferedAxisChangedController, self).__init__(ax, update_lim=update_lim)
        self.plot_function = plot_function
        xmin, ymin, xmax, ymax = plot_limits#self._x_lim # self._compute_buffered(*self._x_lim)
        # imshow acts on the limits of the plot, this is why we need to override the limits here, to make sure the right plot limits are used:
        self._x_lim = xmin, xmax
        self._y_lim = ymin, ymax
        self.resolution = resolution
        self._not_init = False
        self.view = self._init_view(self.ax, self.recompute_X(buffered=False), xmin, xmax, ymin, ymax, **kwargs)
        self._not_init = True

        super(BufferedAxisChangedController, self).update(ax)
        if self._not_init:
            xmin, xmax = self._compute_buffered(*self._x_lim)
            ymin, ymax = self._compute_buffered(*self._y_lim)
            self.update_view(self.view, self.recompute_X(), xmin, xmax, ymin, ymax)

    def _init_view(self, ax, X, xmin, xmax, ymin, ymax):
        raise NotImplementedError('return view for this controller')

    def update_view(self, view, X, xmin, xmax, ymin, ymax):
        raise NotImplementedError('update view given in here')

    def get_grid(self, buffered=True):
        if buffered: comp = self._compute_buffered
        else: comp = lambda a,b: (a,b)
        xmin, xmax = comp(*self._x_lim)
        ymin, ymax = comp(*self._y_lim)
        x, y = numpy.mgrid[xmin:xmax:1j * self.resolution, ymin:ymax:1j * self.resolution]
        return numpy.hstack((x.flatten()[:, None], y.flatten()[:, None]))

    def recompute_X(self, buffered=True):
        X = self.plot_function(self.get_grid(buffered))
        if isinstance(X, (tuple, list)):
            for x in X:
                x.shape = [self.resolution, self.resolution]
                x[:, :] = x.T[::-1, :]
            return X
        return X.reshape(self.resolution, self.resolution).T[::-1, :]

    def _compute_buffered(self, mi, ma):
        buffersize = self._buffersize()
        size = ma - mi
        return mi - (buffersize * size), ma + (buffersize * size)

    def _buffersize(self):
        try:
            buffersize = 1. - self._lim_ratio_threshold
        except:
            buffersize = .4
        return buffersize

from ..inference.latent_function_inference.var_dtc_parallel import VarDTC_minibatch

class BayesianGPLVMMiniBatch(SparseGPMiniBatch):
    """
    Bayesian Gaussian Process Latent Variable Model

    :param Y: observed data (np.ndarray) or GPy.likelihood
    :type Y: np.ndarray| GPy.likelihood instance
    :param init: initialisation method for the latent space
    :type init: 'PCA'|'random'

    """
    def __init__(self, Y, input_dim, X=None, X_variance=None, init='PCA', num_inducing=10,
                 Z=None, kernel=None, inference_method=None, likelihood=None,
                 name='bayesian gplvm', normalizer=None,
                 missing_data=False, stochastic=False, batchsize=1):
            from ..util.initialization import initialize_latent
            self.logger.info("initializing latent space X with method {}".format(init))
            X, fracs = initialize_latent(init, input_dim, Y)
        else:
            fracs = np.ones(input_dim)

        self.init = init

            self.logger.info("initializing latent space variance ~ uniform(0,.1)")
            X_variance = np.random.uniform(0,.1,X.shape)

            self.logger.info("initializing inducing inputs")
            Z = np.random.permutation(X.copy())[:num_inducing]
        assert Z.shape[1] == X.shape[1]

            self.logger.info("initializing kernel RBF")
            kernel = kern.RBF(input_dim, lengthscale=1./fracs, ARD=True) #+ kern.Bias(input_dim) + kern.White(input_dim)

        if likelihood is None:
            likelihood = Gaussian()

        X = NormalPosterior(X, X_variance)

        self.kl_factr = 1.

            from ..inference.latent_function_inference.var_dtc import VarDTC
            self.logger.debug("creating inference_method var_dtc")
            inference_method = VarDTC(limit=1 if not missing_data else Y.shape[1])

        if kernel.useGPU and isinstance(inference_method, VarDTC_GPU):
            kernel.psicomp.GPU_direct = True

        super(BayesianGPLVMMiniBatch,self).__init__(X, Y, Z, kernel, likelihood=likelihood,
                                           name=name, inference_method=inference_method,
                                           normalizer=normalizer,
                                           missing_data=missing_data, stochastic=stochastic,
                                           batchsize=batchsize)
        self.X = X
        self.link_parameter(self.X, 0)

    def set_X_gradients(self, X, X_grad):
        """Set the gradients of the posterior distribution of X in its specific form."""
        X.mean.gradient, X.variance.gradient = X_grad

    def get_X_gradients(self, X):
        """Get the gradients of the posterior distribution of X in its specific form."""
        return X.mean.gradient, X.variance.gradient

    def _inner_parameters_changed(self, kern, X, Z, likelihood, Y, Y_metadata, Lm=None, dL_dKmm=None, subset_indices=None):
        posterior, log_marginal_likelihood, grad_dict, current_values, value_indices = super(BayesianGPLVMMiniBatch, self)._inner_parameters_changed(kern, X, Z, likelihood, Y, Y_metadata, Lm=Lm, dL_dKmm=dL_dKmm, subset_indices=subset_indices)

        current_values['meangrad'], current_values['vargrad'] = self.kern.gradients_qX_expectations(
                                            variational_posterior=X,
                                            Z=Z, dL_dpsi0=grad_dict['dL_dpsi0'],
                                            dL_dpsi1=grad_dict['dL_dpsi1'],
                                            dL_dpsi2=grad_dict['dL_dpsi2'])

        kl_fctr = self.kl_factr
        if self.missing_data:
            d = self.output_dim
            log_marginal_likelihood -= kl_fctr*self.variational_prior.KL_divergence(X)/d
        else:
            log_marginal_likelihood -= kl_fctr*self.variational_prior.KL_divergence(X)

        # Subsetting Variational Posterior objects, makes the gradients
        # empty. We need them to be 0 though:
        X.mean.gradient[:] = 0
        X.variance.gradient[:] = 0

        self.variational_prior.update_gradients_KL(X)
            current_values['meangrad'] += kl_fctr*X.mean.gradient/d
            current_values['vargrad'] += kl_fctr*X.variance.gradient/d
        else:
            current_values['meangrad'] += kl_fctr*X.mean.gradient
            current_values['vargrad'] += kl_fctr*X.variance.gradient

        if subset_indices is not None:
            value_indices['meangrad'] = subset_indices['samples']
            value_indices['vargrad'] = subset_indices['samples']
        return posterior, log_marginal_likelihood, grad_dict, current_values, value_indices

    def _outer_values_update(self, full_values):
        """
        Here you put the values, which were collected before in the right places.
        E.g. set the gradients of parameters, etc.
        """
        super(BayesianGPLVMMiniBatch, self)._outer_values_update(full_values)
        self.X.mean.gradient = full_values['meangrad']
        self.X.variance.gradient = full_values['vargrad']

    def _outer_init_full_values(self):
        return dict(meangrad=np.zeros(self.X.mean.shape),
                    vargrad=np.zeros(self.X.variance.shape))

        super(BayesianGPLVMMiniBatch,self).parameters_changed()
        if isinstance(self.inference_method, VarDTC_minibatch):
            return

        #super(BayesianGPLVM, self).parameters_changed()
        #self._log_marginal_likelihood -= self.variational_prior.KL_divergence(self.X)

        #self.X.mean.gradient, self.X.variance.gradient = self.kern.gradients_qX_expectations(variational_posterior=self.X, Z=self.Z, dL_dpsi0=self.grad_dict['dL_dpsi0'], dL_dpsi1=self.grad_dict['dL_dpsi1'], dL_dpsi2=self.grad_dict['dL_dpsi2'])

        # This is testing code -------------------------
#         i = np.random.randint(self.X.shape[0])
#         X_ = self.X.mean
#         which = np.sqrt(((X_ - X_[i:i+1])**2).sum(1)).argsort()>(max(0, self.X.shape[0]-51))
#         _, _, grad_dict = self.inference_method.inference(self.kern, self.X[which], self.Z, self.likelihood, self.Y[which], self.Y_metadata)
#         grad = self.kern.gradients_qX_expectations(variational_posterior=self.X[which], Z=self.Z, dL_dpsi0=grad_dict['dL_dpsi0'], dL_dpsi1=grad_dict['dL_dpsi1'], dL_dpsi2=grad_dict['dL_dpsi2'])
#
#         self.X.mean.gradient[:] = 0
#         self.X.variance.gradient[:] = 0
#         self.X.mean.gradient[which] = grad[0]
#         self.X.variance.gradient[which] = grad[1]

        # update for the KL divergence
#         self.variational_prior.update_gradients_KL(self.X, which)
        # -----------------------------------------------

        #self.variational_prior.update_gradients_KL(self.X)

        import sys

        return dim_reduction_plots.plot_latent(self, labels, which_indices,
                resolution, ax, marker, s,
                fignum, plot_inducing, legend,
                plot_limits, aspect, updates, predict_kwargs, imshow_kwargs)

    def do_test_latents(self, Y):
        """
        Compute the latent representation for a set of new points Y

        Notes:
        This will only work with a univariate Gaussian likelihood (for now)
        """
        N_test = Y.shape[0]
        input_dim = self.Z.shape[1]

        means = np.zeros((N_test, input_dim))
        covars = np.zeros((N_test, input_dim))

        dpsi0 = -0.5 * self.input_dim / self.likelihood.variance
        dpsi2 = self.grad_dict['dL_dpsi2'][0][None, :, :] # TODO: this may change if we ignore het. likelihoods
        V = Y/self.likelihood.variance

        #compute CPsi1V
        #if self.Cpsi1V is None:
        #    psi1V = np.dot(self.psi1.T, self.likelihood.V)
        #    tmp, _ = linalg.dtrtrs(self._Lm, np.asfortranarray(psi1V), lower=1, trans=0)
        #    tmp, _ = linalg.dpotrs(self.LB, tmp, lower=1)
        #    self.Cpsi1V, _ = linalg.dtrtrs(self._Lm, tmp, lower=1, trans=1)

        dpsi1 = np.dot(self.posterior.woodbury_vector, V.T)

        #start = np.zeros(self.input_dim * 2)

        from scipy.optimize import minimize

        for n, dpsi1_n in enumerate(dpsi1.T[:, :, None]):
            args = (input_dim, self.kern.copy(), self.Z, dpsi0, dpsi1_n.T, dpsi2)
            res = minimize(latent_cost_and_grad, jac=True, x0=np.hstack((means[n], covars[n])), args=args, method='BFGS')
            xopt = res.x
            mu, log_S = xopt.reshape(2, 1, -1)
            means[n] = mu[0].copy()
            covars[n] = np.exp(log_S[0]).copy()

        X = NormalPosterior(means, covars)

        return X

    def dmu_dX(self, Xnew):
        """
        Calculate the gradient of the prediction at Xnew w.r.t Xnew.
        """
        dmu_dX = np.zeros_like(Xnew)
        for i in range(self.Z.shape[0]):
            dmu_dX += self.kern.gradients_X(self.grad_dict['dL_dpsi1'][i:i + 1, :], Xnew, self.Z[i:i + 1, :])
        return dmu_dX

    def dmu_dXnew(self, Xnew):
        """
        Individual gradient of prediction at Xnew w.r.t. each sample in Xnew
        """
        gradients_X = np.zeros((Xnew.shape[0], self.num_inducing))
        ones = np.ones((1, 1))
            gradients_X[:, i] = self.kern.gradients_X(ones, Xnew, self.Z[i:i + 1, :]).sum(-1)
        return np.dot(gradients_X, self.grad_dict['dL_dpsi1'])

    def plot_steepest_gradient_map(self, *args, ** kwargs):
        """
        See GPy.plotting.matplot_dep.dim_reduction_plots.plot_steepest_gradient_map
        """
        import sys

        return dim_reduction_plots.plot_steepest_gradient_map(self,*args,**kwargs)

def latent_cost_and_grad(mu_S, input_dim, kern, Z, dL_dpsi0, dL_dpsi1, dL_dpsi2):
    """
    objective function for fitting the latent variables for test points
    (negative log-likelihood: should be minimised!)
    """
    mu = mu_S[:input_dim][None]
    log_S = mu_S[input_dim:][None]
    S = np.exp(log_S)

    X = NormalPosterior(mu, S)

    psi0 = kern.psi0(Z, X)
    psi1 = kern.psi1(Z, X)
    psi2 = kern.psi2(Z, X)

    lik = dL_dpsi0 * psi0.sum() + np.einsum('ij,kj->...', dL_dpsi1, psi1) + np.einsum('ijk,lkj->...', dL_dpsi2, psi2) - 0.5 * np.sum(np.square(mu) + S) + 0.5 * np.sum(log_S)

    dLdmu, dLdS = kern.gradients_qX_expectations(dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, X)
    dmu = dLdmu - mu
    # dS = S0 + S1 + S2 -0.5 + .5/S
    dlnS = S * (dLdS - 0.5) + .5

    return -lik, -np.hstack((dmu.flatten(), dlnS.flatten()))

try:
    import Tango
except:
    pass
from base_plots import x_frame1D, x_frame2D

def plot_mapping(self, plot_limits=None, which_data='all', which_parts='all', resolution=None, levels=20, samples=0, fignum=None, ax=None, fixed_inputs=[], linecol=Tango.colorsHex['darkBlue']):
    """
    Plots the mapping associated with the model.
      - In one dimension, the function is plotted.
      - In two dimsensions, a contour-plot shows the function
      - In higher dimensions, we've not implemented this yet !TODO!

    Can plot only part of the data and part of the posterior functions
    using which_data and which_functions

    :param plot_limits: The limits of the plot. If 1D [xmin,xmax], if 2D [[xmin,ymin],[xmax,ymax]]. Defaluts to data limits
    :type plot_limits: np.array
    :param which_data: which if the training data to plot (default all)
    :type which_data: 'all' or a slice object to slice self.X, self.Y
    :param which_parts: which of the kernel functions to plot (additively)
    :type which_parts: 'all', or list of bools
    :param resolution: the number of intervals to sample the GP on. Defaults to 200 in 1D and 50 (a 50x50 grid) in 2D
    :type resolution: int
    :param levels: number of levels to plot in a contour plot.
    :type levels: int
    :param samples: the number of a posteriori samples to plot
    :type samples: int
    :param fignum: figure to plot on.
    :type fignum: figure number
    :param ax: axes to plot on.
    :type ax: axes handle
    :param fixed_inputs: a list of tuple [(i,v), (i,v)...], specifying that input index i should be set to value v.
    :type fixed_inputs: a list of tuples
    :param linecol: color of line to plot.
    :type linecol:
    :param levels: for 2D plotting, the number of contour levels to use is ax is None, create a new figure

    """
    # TODO include samples
    if which_data == 'all':
        which_data = slice(None)

        fig = pb.figure(num=fignum)
        ax = fig.add_subplot(111)

    plotdims = self.input_dim - len(fixed_inputs)

    if plotdims == 1:

        Xu = self.X * self._Xscale + self._Xoffset # NOTE self.X are the normalized values now

        fixed_dims = np.array([i for i,v in fixed_inputs])
        freedim = np.setdiff1d(np.arange(self.input_dim),fixed_dims)

        Xnew, xmin, xmax = x_frame1D(Xu[:,freedim], plot_limits=plot_limits)
        Xgrid = np.empty((Xnew.shape[0],self.input_dim))
        Xgrid[:,freedim] = Xnew
        for i,v in fixed_inputs:
            Xgrid[:,i] = v

        f = self.predict(Xgrid, which_parts=which_parts)
        for d in range(y.shape[1]):
            ax.plot(Xnew, f[:, d], edgecol=linecol)

    elif self.X.shape[1] == 2:
        resolution = resolution or 50
        Xnew, _, _, xmin, xmax = x_frame2D(self.X, plot_limits, resolution)
        x, y = np.linspace(xmin[0], xmax[0], resolution), np.linspace(xmin[1], xmax[1], resolution)
        f = self.predict(Xnew, which_parts=which_parts)
        m = m.reshape(resolution, resolution).T
        ax.contour(x, y, f, levels, vmin=m.min(), vmax=m.max(), cmap=pb.cm.jet) # @UndefinedVariable
        ax.set_xlim(xmin[0], xmax[0])
        ax.set_ylim(xmin[1], xmax[1])

    else:
        raise NotImplementedError, "Cannot define a frame with more than two input dimensions"

import GPy

class FITCtest(unittest.TestCase):
        ######################################
        # # 1 dimensional example

        N = 20
        # sample inputs and outputs
        self.X1D = np.random.uniform(-3., 3., (N, 1))
        self.Y1D = np.sin(self.X1D) + np.random.randn(N, 1) * 0.05

        # # 2 dimensional example

        self.X2D = np.random.uniform(-3., 3., (N, 2))
        self.Y2D = np.sin(self.X2D[:, 0:1]) * np.sin(self.X2D[:, 1:2]) + np.random.randn(N, 1) * 0.05

    def test_fitc_1d(self):
        m = GPy.models.SparseGPRegression(self.X1D, self.Y1D)
        m.inference_method=GPy.inference.latent_function_inference.FITC()
        self.assertTrue(m.checkgrad())

    def test_fitc_2d(self):
        m = GPy.models.SparseGPRegression(self.X2D, self.Y2D)

import numpy as _np

# default_seed = _np.random.seed(123344)

def bgplvm_test_model(optimize=False, verbose=1, plot=False, output_dim=200, nan=False):
    """
    model for testing purposes. Samples from a GP with rbf kernel and learns
    the samples with a new kernel. Normally not for optimization, just model cheking
    """
    import GPy

    num_inputs = 13
    num_inducing = 5
    if plot:
        output_dim = 1
        input_dim = 3
    else:
        input_dim = 2
        output_dim = output_dim

    # generate GPLVM-like data
    X = _np.random.rand(num_inputs, input_dim)
    lengthscales = _np.random.rand(input_dim)
    k = GPy.kern.RBF(input_dim, .5, lengthscales, ARD=True)
    K = k.K(X)
    Y = _np.random.multivariate_normal(_np.zeros(num_inputs), K, (output_dim,)).T

    # k = GPy.kern.RBF_inv(input_dim, .5, _np.ones(input_dim) * 2., ARD=True) + GPy.kern.bias(input_dim) + GPy.kern.white(input_dim)
    # k = GPy.kern.linear(input_dim)# + GPy.kern.bias(input_dim) + GPy.kern.white(input_dim, 0.00001)
    # k = GPy.kern.RBF(input_dim, ARD = False)  + GPy.kern.white(input_dim, 0.00001)
    # k = GPy.kern.RBF(input_dim, .5, _np.ones(input_dim) * 2., ARD=True) + GPy.kern.RBF(input_dim, .3, _np.ones(input_dim) * .2, ARD=True)
    # k = GPy.kern.RBF(input_dim, .5, 2., ARD=0) + GPy.kern.RBF(input_dim, .3, .2, ARD=0)
    # k = GPy.kern.RBF(input_dim, .5, _np.ones(input_dim) * 2., ARD=True) + GPy.kern.linear(input_dim, _np.ones(input_dim) * .2, ARD=True)

    p = .3

    m = GPy.models.BayesianGPLVM(Y, input_dim, kernel=k, num_inducing=num_inducing)

    if nan:
        m.inference_method = GPy.inference.latent_function_inference.var_dtc.VarDTCMissingData()
        m.Y[_np.random.binomial(1, p, size=(Y.shape)).astype(bool)] = _np.nan
        m.parameters_changed()

    #===========================================================================
    # randomly obstruct data with percentage p
    # m2 = GPy.models.BayesianGPLVMWithMissingData(Y_obstruct, input_dim, kernel=k, num_inducing=num_inducing)
    # m.lengthscales = lengthscales

    if plot:
        import matplotlib.pyplot as pb
        m.plot()
        pb.title('PCA initialisation')
        # m2.plot()
        # pb.title('PCA initialisation')

    if optimize:
        m.optimize('scg', messages=verbose)
        # m2.optimize('scg', messages=verbose)
        if plot:
            m.plot()
            pb.title('After optimisation')
            # m2.plot()
            # pb.title('After optimisation')

    return m

def gplvm_oil_100(optimize=True, verbose=1, plot=True):
    import GPy
    import pods
    data = pods.datasets.oil_100()
    Y = data['X']
    # create simple GP model
    kernel = GPy.kern.RBF(6, ARD=True) + GPy.kern.Bias(6)
    m = GPy.models.GPLVM(Y, 6, kernel=kernel)
    m.data_labels = data['Y'].argmax(axis=1)
    if optimize: m.optimize('scg', messages=verbose)
    if plot: m.plot_latent(labels=m.data_labels)
    return m

def sparse_gplvm_oil(optimize=True, verbose=0, plot=True, N=100, Q=6, num_inducing=15, max_iters=50):
    import GPy

    _np.random.seed(0)
    data = pods.datasets.oil()
    Y = data['X'][:N]
    Y = Y - Y.mean(0)
    Y /= Y.std(0)
    # Create the model
    kernel = GPy.kern.RBF(Q, ARD=True) + GPy.kern.Bias(Q)
    m = GPy.models.SparseGPLVM(Y, Q, kernel=kernel, num_inducing=num_inducing)
    m.data_labels = data['Y'][:N].argmax(axis=1)

    if optimize: m.optimize('scg', messages=verbose, max_iters=max_iters)
    if plot:
        m.plot_latent(labels=m.data_labels)
        m.kern.plot_ARD()
    return m

def swiss_roll(optimize=True, verbose=1, plot=True, N=1000, num_inducing=25, Q=4, sigma=.2):
    import GPy
    from pods.datasets import swiss_roll_generated
    from GPy.models import BayesianGPLVM

    data = swiss_roll_generated(num_samples=N, sigma=sigma)
    Y = data['Y']
    Y -= Y.mean()
    Y /= Y.std()

    t = data['t']
    c = data['colors']

    try:
        from sklearn.manifold.isomap import Isomap
        iso = Isomap().fit(Y)
        X = iso.embedding_
        if Q > 2:
            X = _np.hstack((X, _np.random.randn(N, Q - 2)))
    except ImportError:
        X = _np.random.randn(N, Q)

    if plot:
        from mpl_toolkits.mplot3d import Axes3D  # @UnusedImport
        fig = plt.figure("Swiss Roll Data")
        ax = fig.add_subplot(121, projection='3d')
        ax.scatter(*Y.T, c=c)
        ax.set_title("Swiss Roll")

        ax = fig.add_subplot(122)
        ax.scatter(*X.T[:2], c=c)
        ax.set_title("BGPLVM init")

    var = .5
    S = (var * _np.ones_like(X) + _np.clip(_np.random.randn(N, Q) * var ** 2,
                                         - (1 - var),
                                         (1 - var))) + .001
    Z = _np.random.permutation(X)[:num_inducing]

    kernel = GPy.kern.RBF(Q, ARD=True) + GPy.kern.Bias(Q, _np.exp(-2)) + GPy.kern.White(Q, _np.exp(-2))

    m = BayesianGPLVM(Y, Q, X=X, X_variance=S, num_inducing=num_inducing, Z=Z, kernel=kernel)
    m.data_colors = c
    m.data_t = t

        m.optimize('bfgs', messages=verbose, max_iters=2e3)

    if plot:
        fig = plt.figure('fitted')
        s = m.input_sensitivity().argsort()[::-1][:2]
        ax.scatter(*m.X.mean.T[s], c=c)

    return m

def bgplvm_oil(optimize=True, verbose=1, plot=True, N=200, Q=7, num_inducing=40, max_iters=1000, **k):
    import GPy
    from matplotlib import pyplot as plt
    import numpy as np
    try:
        import pods
        data = pods.datasets.oil()
        data = GPy.util.datasets.oil()

    kernel = GPy.kern.RBF(Q, 1., 1. / _np.random.uniform(0, 1, (Q,)), ARD=True)  # + GPy.kern.Bias(Q, _np.exp(-2))
    m = GPy.models.BayesianGPLVM(Y, Q, kernel=kernel, num_inducing=num_inducing, **k)

        m.optimize('bfgs', messages=verbose, max_iters=max_iters, gtol=.05)

    if plot:
        fig, (latent_axes, sense_axes) = plt.subplots(1, 2)
        m.plot_latent(ax=latent_axes, labels=m.data_labels)
        data_show = GPy.plotting.matplot_dep.visualize.vector_show((m.Y[0, :]))
        lvm_visualizer = GPy.plotting.matplot_dep.visualize.lvm_dimselect(m.X.mean.values[0:1, :],  # @UnusedVariable
            m, data_show, latent_axes=latent_axes, sense_axes=sense_axes, labels=m.data_labels)
        raw_input('Press enter to finish')
        plt.close(fig)
    return m

def ssgplvm_oil(optimize=True, verbose=1, plot=True, N=200, Q=7, num_inducing=40, max_iters=1000, **k):
    import GPy

    m = GPy.models.SSGPLVM(Y, Q, kernel=kernel, num_inducing=num_inducing, **k)

    if plot:
    return m

def _simulate_matern(D1, D2, D3, N, num_inducing, plot_sim=False):
    Q_signal = 4
    import GPy
    np.random.seed(3000)

    k = GPy.kern.Matern32(Q_signal, 1., lengthscale=(np.random.uniform(1, 6, Q_signal)), ARD=1)
    for i in range(Q_signal):
        k += GPy.kern.PeriodicExponential(1, variance=1., active_dims=[i], period=3., lower=-2, upper=6)
    t = np.c_[[np.linspace(-1, 5, N) for _ in range(Q_signal)]].T
    K = k.K(t)
    s2, s1, s3, sS = np.random.multivariate_normal(np.zeros(K.shape[0]), K, size=(4))[:, :, None]

    Y1, Y2, Y3, S1, S2, S3 = _generate_high_dimensional_output(D1, D2, D3, s1, s2, s3, sS)

    slist = [sS, s1, s2, s3]
    slist_names = ["sS", "s1", "s2", "s3"]
    Ylist = [Y1, Y2, Y3]

    if plot_sim:
        import matplotlib.cm as cm
        import itertools
        fig = plt.figure("MRD Simulation Data", figsize=(8, 6))
        fig.clf()
        ax = fig.add_subplot(2, 1, 1)
        labls = slist_names
        for S, lab in itertools.izip(slist, labls):
            ax.plot(S, label=lab)
        ax.legend()
        for i, Y in enumerate(Ylist):
            ax = fig.add_subplot(2, len(Ylist), len(Ylist) + 1 + i)
            ax.imshow(Y, aspect='auto', cmap=cm.gray)  # @UndefinedVariable
            ax.set_title("Y{}".format(i + 1))
        plt.tight_layout()

    return slist, [S1, S2, S3], Ylist

def _simulate_sincos(D1, D2, D3, N, num_inducing, plot_sim=False):
    _np.random.seed(1234)

    x = _np.linspace(0, 4 * _np.pi, N)[:, None]
    s1 = _np.vectorize(lambda x: _np.sin(x))
    s2 = _np.vectorize(lambda x: _np.cos(x) ** 2)
    s3 = _np.vectorize(lambda x:-_np.exp(-_np.cos(2 * x)))
    sS = _np.vectorize(lambda x: _np.cos(x))

    s1 = s1(x)
    s2 = s2(x)
    s3 = s3(x)
    sS = sS(x)

    s1 -= s1.mean(); s1 /= s1.std(0)
    s2 -= s2.mean(); s2 /= s2.std(0)
    s3 -= s3.mean(); s3 /= s3.std(0)
    sS -= sS.mean(); sS /= sS.std(0)

        fig.clf()

def _generate_high_dimensional_output(D1, D2, D3, s1, s2, s3, sS):
    S1 = _np.hstack([s1, sS])
    S2 = _np.hstack([s2, s3, sS])
    S3 = _np.hstack([s3, sS])
    Y1 = S1.dot(_np.random.randn(S1.shape[1], D1))
    Y2 = S2.dot(_np.random.randn(S2.shape[1], D2))
    Y3 = S3.dot(_np.random.randn(S3.shape[1], D3))
    Y1 += .3 * _np.random.randn(*Y1.shape)
    Y2 += .2 * _np.random.randn(*Y2.shape)
    Y3 += .25 * _np.random.randn(*Y3.shape)
    Y1 -= Y1.mean(0)
    Y2 -= Y2.mean(0)
    Y3 -= Y3.mean(0)
    Y1 /= Y1.std(0)
    Y2 /= Y2.std(0)
    Y3 /= Y3.std(0)
    return Y1, Y2, Y3, S1, S2, S3

def bgplvm_simulation(optimize=True, verbose=1,
                      plot=True, plot_sim=False,
                      max_iters=2e4,
                      ):
    from GPy import kern

    D1, D2, D3, N, num_inducing, Q = 13, 5, 8, 45, 3, 9
    _, _, Ylist = _simulate_matern(D1, D2, D3, N, num_inducing, plot_sim)
    Y = Ylist[0]
    k = kern.Linear(Q, ARD=True)  # + kern.white(Q, _np.exp(-2)) # + kern.bias(Q)
    # k = kern.RBF(Q, ARD=True, lengthscale=10.)
    m = BayesianGPLVM(Y, Q, init="PCA", num_inducing=num_inducing, kernel=k)
    m.X.variance[:] = _np.random.uniform(0, .01, m.X.shape)
    m.likelihood.variance = .1

        print "Optimizing model:"
        m.optimize('bfgs', messages=verbose, max_iters=max_iters,
                   gtol=.05)
    if plot:
        m.X.plot("BGPLVM Latent Space 1D")
        m.kern.plot_ARD('BGPLVM Simulation ARD Parameters')
    return m

def ssgplvm_simulation(optimize=True, verbose=1,
                      max_iters=2e4, useGPU=False
                      ):
    from GPy.models import SSGPLVM

    m = SSGPLVM(Y, Q, init="pca", num_inducing=num_inducing, kernel=k)

        m.optimize('scg', messages=verbose, max_iters=max_iters,
                   gtol=.05)
    if plot:
        m.X.plot("SSGPLVM Latent Space 1D")
        m.kern.plot_ARD('SSGPLVM Simulation ARD Parameters')
    return m

def bgplvm_simulation_missing_data(optimize=True, verbose=1,
                      max_iters=2e4, percent_missing=.1,
                      ):
    from GPy.models.bayesian_gplvm_minibatch import BayesianGPLVMMiniBatch

    D1, D2, D3, N, num_inducing, Q = 13, 5, 8, 400, 3, 4

    inan = _np.random.binomial(1, percent_missing, size=Y.shape).astype(bool)  # 80% missing data
    Ymissing = Y.copy()
    Ymissing[inan] = _np.nan

    m = BayesianGPLVMMiniBatch(Ymissing, Q, init="random", num_inducing=num_inducing,
                      kernel=k, missing_data=True)

    m.Yreal = Y

                   gtol=.05)
    if plot:
    return m

def mrd_simulation(optimize=True, verbose=True, plot=True, plot_sim=True, **kw):
    from GPy.models import MRD

    D1, D2, D3, N, num_inducing, Q = 60, 20, 36, 60, 6, 5

    # Ylist = [Ylist[0]]
    k = kern.Linear(Q, ARD=True)
    m = MRD(Ylist, input_dim=Q, num_inducing=num_inducing, kernel=k, initx="PCA_concat", initz='permute', **kw)

    m['.*noise'] = [Y.var() / 40. for Y in Ylist]

        print "Optimizing Model:"
        m.optimize(messages=verbose, max_iters=8e3)
    if plot:
        m.X.plot("MRD Latent Space 1D")
        m.plot_scales("MRD Scales")
    return m

def mrd_simulation_missing_data(optimize=True, verbose=True, plot=True, plot_sim=True, **kw):

    inanlist = []

    for Y in Ylist:
        inan = _np.random.binomial(1, .6, size=Y.shape).astype(bool)
        inanlist.append(inan)
        Y[inan] = _np.nan

    m = MRD(Ylist, input_dim=Q, num_inducing=num_inducing,
            kernel=k, inference_method=None,
            initx="random", initz='permute', **kw)

        m.optimize('bfgs', messages=verbose, max_iters=8e3, gtol=.1)
    if plot:
    return m

def brendan_faces(optimize=True, verbose=True, plot=True):
    import GPy

    data = pods.datasets.brendan_faces()
    Q = 2
    Yn = Y - Y.mean()
    Yn /= Yn.std()

    m = GPy.models.BayesianGPLVM(Yn, Q, num_inducing=20)

    # optimize

    if optimize: m.optimize('bfgs', messages=verbose, max_iters=1000)

    if plot:
        ax = m.plot_latent(which_indices=(0, 1))
        y = m.Y[0, :]
        data_show = GPy.plotting.matplot_dep.visualize.image_show(y[None, :], dimensions=(20, 28), transpose=True, order='F', invert=False, scale=False)
        lvm = GPy.plotting.matplot_dep.visualize.lvm(m.X.mean[0, :].copy(), m, data_show, ax)

    return m

def olivetti_faces(optimize=True, verbose=True, plot=True):
    import GPy

    data = pods.datasets.olivetti_faces()
    Q = 2

    if plot:
        data_show = GPy.plotting.matplot_dep.visualize.image_show(y[None, :], dimensions=(112, 92), transpose=False, invert=False, scale=False)

    return m

def stick_play(range=None, frame_rate=15, optimize=False, verbose=True, plot=True):
    import GPy

    data = pods.datasets.osu_run1()
    # optimize
    if range == None:
        Y = data['Y'].copy()
    else:
        Y = data['Y'][range[0]:range[1], :].copy()
    if plot:
        y = Y[0, :]
        data_show = GPy.plotting.matplot_dep.visualize.stick_show(y[None, :], connect=data['connect'])
        GPy.plotting.matplot_dep.visualize.data_play(Y, data_show, frame_rate)
    return Y

def stick(kernel=None, optimize=True, verbose=True, plot=True):
    import GPy

    # optimize
    m = GPy.models.GPLVM(data['Y'], 2, kernel=kernel)
    if optimize: m.optimize('bfgs', messages=verbose, max_f_eval=10000)
    if plot:
        plt.clf
        ax = m.plot_latent()
        lvm_visualizer = GPy.plotting.matplot_dep.visualize.lvm(m.X[:1, :].copy(), m, data_show, latent_axes=ax)
        lvm_visualizer.close()
        data_show.close()
    return m

def bcgplvm_linear_stick(kernel=None, optimize=True, verbose=True, plot=True):
    import GPy

    # optimize
    mapping = GPy.mappings.Linear(data['Y'].shape[1], 2)
    m = GPy.models.BCGPLVM(data['Y'], 2, kernel=kernel, mapping=mapping)
    if optimize: m.optimize(messages=verbose, max_f_eval=10000)
    if plot and GPy.plotting.matplot_dep.visualize.visual_available:
        plt.clf
        y = m.likelihood.Y[0, :]
        GPy.plotting.matplot_dep.visualize.lvm(m.X[0, :].copy(), m, data_show, ax)

    return m

def bcgplvm_stick(kernel=None, optimize=True, verbose=True, plot=True):
    import GPy

    # optimize
    back_kernel = GPy.kern.RBF(data['Y'].shape[1], lengthscale=5.)
    mapping = GPy.mappings.Kernel(X=data['Y'], output_dim=2, kernel=back_kernel)
        plt.clf
        # raw_input('Press enter to finish')

    return m

def robot_wireless(optimize=True, verbose=True, plot=True):
    import GPy

    data = pods.datasets.robot_wireless()
    # optimize
    m = GPy.models.BayesianGPLVM(data['Y'], 4, num_inducing=25)
    if plot:
        m.plot_latent()

    return m

def stick_bgplvm(model=None, optimize=True, verbose=True, plot=True):
    import GPy

    Q = 6
    kernel = GPy.kern.RBF(Q, lengthscale=np.repeat(.5, Q), ARD=True)
    m = BayesianGPLVM(data['Y'], Q, init="PCA", num_inducing=20, kernel=kernel)

    m.data = data
    m.likelihood.variance = 0.001

    # optimize
    try:
        if optimize: m.optimize('bfgs', messages=verbose, max_iters=5e3, bfgs_factor=10)
    except KeyboardInterrupt:
        print "Keyboard interrupt, continuing to plot and return"

    if plot:
        plt.sca(latent_axes)
        m.plot_latent(ax=latent_axes)
        y = m.Y[:1, :].copy()
        data_show = GPy.plotting.matplot_dep.visualize.stick_show(y, connect=data['connect'])
        dim_select = GPy.plotting.matplot_dep.visualize.lvm_dimselect(m.X.mean[:1, :].copy(), m, data_show, latent_axes=latent_axes, sense_axes=sense_axes)
        fig.canvas.draw()
        fig.canvas.show()

    return m

def cmu_mocap(subject='35', motion=['01'], in_place=True, optimize=True, verbose=True, plot=True):
    import GPy

    data = pods.datasets.cmu_mocap(subject, motion)
    if in_place:
        # Make figure move in place.
        data['Y'][:, 0:3] = 0.0
    Y_mean = Y.mean(0)
    Y_std = Y.std(0)
    m = GPy.models.GPLVM((Y - Y_mean) / Y_std, 2)

    if plot:
        data_show = GPy.plotting.matplot_dep.visualize.skeleton_show(y[None, :], data['skel'])
        lvm_visualizer = GPy.plotting.matplot_dep.visualize.lvm(m.X[0].copy(), m, data_show, latent_axes=ax)

    return m

def ssgplvm_simulation_linear():
    import GPy
    N, D, Q = 1000, 20, 5
    pi = 0.2

    def sample_X(Q, pi):
        x = np.empty(Q)
        dies = np.random.rand(Q)
        for q in xrange(Q):
            if dies[q] < pi:
                x[q] = np.random.randn()
            else:
                x[q] = 0.
        return x

    Y = np.empty((N, D))
    X = np.empty((N, Q))
    # Generate data from random sampled weight matrices
    for n in xrange(N):
        X[n] = sample_X(Q, pi)
        w = np.random.randn(D, Q)
        Y[n] = np.dot(w, X[n])

from latent_space_visualizations.controllers.imshow_controller import ImshowController,ImAnnotateController
from .base_plots import x_frame2D
try:
    from matplotlib.cm import get_cmap
except:
    pass

def most_significant_input_dimensions(model, which_indices):
    """
    Determine which dimensions should be plotted
    """
    if which_indices is None:
        if model.input_dim == 1:
            input_1 = 0
            input_2 = None
        if model.input_dim == 2:
            input_1, input_2 = 0, 1
        else:
            try:
                input_1, input_2 = np.argsort(model.input_sensitivity())[::-1][:2]
            except:
                raise ValueError, "cannot automatically determine which dimensions to plot, please pass 'which_indices'"
    else:
        input_1, input_2 = which_indices
    return input_1, input_2

def plot_latent(model, labels=None, which_indices=None,
                fignum=None, plot_inducing=False, legend=True,
    """
    :param labels: a np.array of size model.num_data containing labels for the points (can be number, strings, etc)
    :param resolution: the resolution of the grid on which to evaluate the predictive variance
    """
    else:
        fig = ax.figure
    Tango.reset()

    if labels is None:
        labels = np.ones(model.num_data)

    input_1, input_2 = most_significant_input_dimensions(model, which_indices)

    #fethch the data points X that we'd like to plot
    X = model.X
    if isinstance(X, VariationalPosterior):
        X = X.mean
    else:
        X = X

    if X.shape[0] > 1000:
        print "Warning: subsampling X, as it has more samples then 1000. X.shape={!s}".format(X.shape)
        subsample = np.random.choice(X.shape[0], size=1000, replace=False)
        X = X[subsample]
        labels = labels[subsample]
        #=======================================================================
        #     <<<WORK IN PROGRESS>>>
        #     <<<DO NOT DELETE>>>
        #     plt.close('all')
        #     fig, ax = plt.subplots(1,1)
        #     from GPy.plotting.matplot_dep.dim_reduction_plots import most_significant_input_dimensions
        #     import matplotlib.patches as mpatches
        #     i1, i2 = most_significant_input_dimensions(m, None)
        #     xmin, xmax = 100, -100
        #     ymin, ymax = 100, -100
        #     legend_handles = []
        #
        #     X = m.X.mean[:, [i1, i2]]
        #     X = m.X.variance[:, [i1, i2]]
        #
        #     xmin = X[:,0].min(); xmax = X[:,0].max()
        #     ymin = X[:,1].min(); ymax = X[:,1].max()
        #     range_ = [[xmin, xmax], [ymin, ymax]]
        #     ul = np.unique(labels)
        #
        #     for i, l in enumerate(ul):
        #         #cdict = dict(red  =[(0., colors[i][0], colors[i][0]), (1., colors[i][0], colors[i][0])],
        #         #             green=[(0., colors[i][0], colors[i][1]), (1., colors[i][1], colors[i][1])],
        #         #             blue =[(0., colors[i][0], colors[i][2]), (1., colors[i][2], colors[i][2])],
        #         #             alpha=[(0., 0., .0), (.5, .5, .5), (1., .5, .5)])
        #         #cmap = LinearSegmentedColormap('{}'.format(l), cdict)
        #         cmap = LinearSegmentedColormap.from_list('cmap_{}'.format(str(l)), [colors[i], colors[i]], 255)
        #         cmap._init()
        #         #alphas = .5*(1+scipy.special.erf(np.linspace(-2,2, cmap.N+3)))#np.log(np.linspace(np.exp(0), np.exp(1.), cmap.N+3))
        #         alphas = (scipy.special.erf(np.linspace(0,2.4, cmap.N+3)))#np.log(np.linspace(np.exp(0), np.exp(1.), cmap.N+3))
        #         cmap._lut[:, -1] = alphas
        #         print l
        #         x, y = X[labels==l].T
        #
        #         heatmap, xedges, yedges = np.histogram2d(x, y, bins=300, range=range_)
        #         #heatmap, xedges, yedges = np.histogram2d(x, y, bins=100)
        #
        #         im = ax.imshow(heatmap, extent=[xedges[0], xedges[-1], yedges[0], yedges[-1]], cmap=cmap, aspect='auto', interpolation='nearest', label=str(l))
        #         legend_handles.append(mpatches.Patch(color=colors[i], label=l))
        #     ax.set_xlim(xmin, xmax)
        #     ax.set_ylim(ymin, ymax)
        #     plt.legend(legend_handles, [l.get_label() for l in legend_handles])
        #     plt.draw()
        #     plt.show()

    # create a function which computes the shading of latent space according to the output variance
    def plot_function(x):
        Xtest_full = np.zeros((x.shape[0], model.X.shape[1]))
        Xtest_full[:, [input_1, input_2]] = x
        _, var = model.predict(Xtest_full, **predict_kwargs)
        var = var[:, :1]
        return np.log(var)

    #Create an IMshow controller that can re-plot the latent space shading at a good resolution
    if plot_limits is None:
        xmin, ymin = X[:, [input_1, input_2]].min(0)
        xmax, ymax = X[:, [input_1, input_2]].max(0)
        x_r, y_r = xmax-xmin, ymax-ymin
        xmin -= .1*x_r
        xmax += .1*x_r
        ymin -= .1*y_r
        ymax += .1*y_r
    else:
        try:
            xmin, xmax, ymin, ymax = plot_limits
        except (TypeError, ValueError) as e:
            raise e.__class__, "Wrong plot limits: {} given -> need (xmin, xmax, ymin, ymax)".format(plot_limits)
    view = ImshowController(ax, plot_function,
                            (xmin, ymin, xmax, ymax),
                            resolution, aspect=aspect, interpolation='bilinear',
                            cmap=pb.cm.binary, **imshow_kwargs)

    # make sure labels are in order of input:
    labels = np.asarray(labels)
    ulabels = []
    for lab in labels:
        if not lab in ulabels:
            ulabels.append(lab)

    marker = itertools.cycle(list(marker))

    for i, ul in enumerate(ulabels):
        if type(ul) is np.string_:
            this_label = ul
        elif type(ul) is np.int64:
            this_label = 'class %i' % ul
        else:
            this_label = unicode(ul)
        m = marker.next()

        index = np.nonzero(labels == ul)[0]
            x = X[index, input_1]
            y = np.zeros(index.size)
        else:
            y = X[index, input_2]
        ax.scatter(x, y, marker=m, s=s, c=Tango.nextMedium(), label=this_label, linewidth=.2, edgecolor='k', alpha=.9)

    ax.set_xlabel('latent dimension %i' % input_1)
    ax.set_ylabel('latent dimension %i' % input_2)

    if not np.all(labels == 1.) and legend:
        ax.legend(loc=0, numpoints=1)

    ax.grid(b=False) # remove the grid if present, it doesn't look good
    ax.set_aspect('auto') # set a nice aspect ratio

    if plot_inducing:
        Z = model.Z
        ax.scatter(Z[:, input_1], Z[:, input_2], c='w', s=14, marker="^", edgecolor='k', linewidth=.3, alpha=.6)

    ax.set_xlim((xmin, xmax))
    ax.set_ylim((ymin, ymax))

    try:
        fig.tight_layout()
    except Exception as e:
        print "Could not invoke tight layout: {}".format(e)
        pass

    if updates:
        try:
            ax.figure.canvas.show()
        except Exception as e:
            print "Could not invoke show: {}".format(e)
        raw_input('Enter to continue')
        view.deactivate()
    return ax

def plot_magnification(model, labels=None, which_indices=None,
                resolution=60, ax=None, marker='o', s=40,
                aspect='auto', updates=False):
    """
    """

    # first, plot the output variance as a function of the latent space
    Xtest, xx, yy, xmin, xmax = x_frame2D(model.X[:, [input_1, input_2]], resolution=resolution)
    Xtest_full = np.zeros((Xtest.shape[0], model.X.shape[1]))

        mf=model.magnification(Xtest_full)
        return mf

                            tuple(model.X.min(0)[:, [input_1, input_2]]) + tuple(model.X.max(0)[:, [input_1, input_2]]),
                            cmap=pb.cm.gray)

        else:
            this_label = 'class %i' % i

            x = model.X[index, input_1]
        else:
            y = model.X[index, input_2]
        ax.scatter(x, y, marker=m, s=s, color=Tango.nextMedium(), label=this_label)

    ax.set_xlim(xmin[0], xmax[0])
    ax.set_ylim(xmin[1], xmax[1])

        ax.plot(model.Z[:, input_1], model.Z[:, input_2], '^w')

    pb.title('Magnification Factor')
    return ax

def plot_steepest_gradient_map(model, fignum=None, ax=None, which_indices=None, labels=None, data_labels=None, data_marker='o', data_s=40, resolution=20, aspect='auto', updates=False, ** kwargs):

    input_1, input_2 = significant_dims = most_significant_input_dimensions(model, which_indices)

    X = np.zeros((resolution ** 2, model.input_dim))
    indices = np.r_[:X.shape[0]]
        labels = range(model.output_dim)

        X[:, significant_dims] = x
        dmu_dX = model.dmu_dXnew(X)
        argmax = np.argmax(dmu_dX, 1)
        return dmu_dX[indices, argmax], np.array(labels)[argmax]

    if data_labels is None:
        data_labels = np.ones(model.num_data)
    for lab in data_labels:
    marker = itertools.cycle(list(data_marker))
        else:
        index = np.nonzero(data_labels == ul)[0]
        x = model.X[index, input_1]
        y = model.X[index, input_2]
        ax.scatter(x, y, marker=m, s=data_s, color=Tango.nextMedium(), label=this_label)

    controller = ImAnnotateController(ax,
                                  plot_function,
                                  tuple(model.X.min(0)[:, significant_dims]) + tuple(model.X.max(0)[:, significant_dims]),
                                  resolution=resolution,
                                  aspect=aspect,
                                  cmap=get_cmap('jet'),
                                  **kwargs)
    ax.legend()
    ax.figure.tight_layout()
        pb.show()
        clear = raw_input('Enter to continue')
        if clear.lower() in 'yes' or clear == '':
            controller.deactivate()
    return controller.view

# Copyright (c) 2012, James Hensman and Ricardo Andrade

from scipy import weave
from ...util.config import config # for assesing whether to use weave

class Coregionalize(Kern):
    """
    Covariance function for intrinsic/linear coregionalization models

    This covariance has the form:
    .. math::
       \mathbf{B} = \mathbf{W}\mathbf{W}^\top + \text{diag}(kappa)

    An intrinsic/linear coregionalization covariance function of the form:
    .. math::

       k_2(x, y)=\mathbf{B} k(x, y)

    it is obtained as the tensor product between a covariance function
    k(x, y) and B.

    :param output_dim: number of outputs to coregionalize
    :param rank: number of columns of the W matrix (this parameter is ignored if parameter W is not None)
    :type rank: int
    :param W: a low rank matrix that determines the correlations between the different outputs, together with kappa it forms the coregionalization matrix B
    :type W: numpy array of dimensionality (num_outpus, W_columns)
    :param kappa: a vector which allows the outputs to behave independently
    :type kappa: numpy array of dimensionality  (output_dim, )

    .. note: see coregionalization examples in GPy.examples.regression for some usage.
    """
    def __init__(self, input_dim, output_dim, rank=1, W=None, kappa=None, active_dims=None, name='coregion'):
        super(Coregionalize, self).__init__(input_dim, active_dims, name=name)
        self.output_dim = output_dim
        self.rank = rank
        if self.rank>output_dim:
            print("Warning: Unusual choice of rank, it should normally be less than the output_dim.")
        if W is None:
            W = 0.5*np.random.randn(self.output_dim, self.rank)/np.sqrt(self.rank)
        else:
            assert W.shape==(self.output_dim, self.rank)
        self.W = Param('W', W)
        if kappa is None:
            kappa = 0.5*np.ones(self.output_dim)
        else:
            assert kappa.shape==(self.output_dim, )
        self.kappa = Param('kappa', kappa, Logexp())
        self.link_parameters(self.W, self.kappa)

        self.B = np.dot(self.W, self.W.T) + np.diag(self.kappa)

        if config.getboolean('weave', 'working'):
            try:
                return self._K_weave(X, X2)
            except:
                print "\n Weave compilation failed. Falling back to (slower) numpy implementation\n"
                config.set('weave', 'working', 'False')
                return self._K_numpy(X, X2)
        else:
            return self._K_numpy(X, X2)

    def _K_numpy(self, X, X2=None):
        index = np.asarray(X, dtype=np.int)
            return self.B[index,index.T]
        else:
            index2 = np.asarray(X2, dtype=np.int)
            return self.B[index,index2.T]

    def _K_weave(self, X, X2=None):
        """compute the kernel function using scipy.weave"""

            target = np.empty((X.shape[0], X.shape[0]), dtype=np.float64)
            code="""
            for(int i=0;i<N; i++){
              target[i+i*N] = B[index[i]+output_dim*index[i]];
              for(int j=0; j<i; j++){
                  target[j+i*N] = B[index[i]+output_dim*index[j]];
                  target[i+j*N] = target[j+i*N];
                }
              }
            """
            N, B, output_dim = index.size, self.B, self.output_dim
            weave.inline(code, ['target', 'index', 'N', 'B', 'output_dim'])
        else:
            target = np.empty((X.shape[0], X2.shape[0]), dtype=np.float64)
            code="""
            for(int i=0;i<num_inducing; i++){
              for(int j=0; j<N; j++){
                  target[i+j*num_inducing] = B[output_dim*index[j]+index2[i]];
                }
              }
            """
            N, num_inducing, B, output_dim = index.size, index2.size, self.B, self.output_dim
            weave.inline(code, ['target', 'index', 'index2', 'N', 'num_inducing', 'B', 'output_dim'])

        return np.diag(self.B)[np.asarray(X, dtype=np.int).flatten()]

            index2 = index
        else:

        #attempt to use weave for a nasty double indexing loop: fall back to numpy
            try:
                dL_dK_small = self._gradient_reduce_weave(dL_dK, index, index2)
            except:
        else:
            dL_dK_small = self._gradient_reduce_weave(dL_dK, index, index2)

        dkappa = np.diag(dL_dK_small)
        dL_dK_small += dL_dK_small.T
        dW = (self.W[:, None, :]*dL_dK_small[:, :, None]).sum(0)

        self.W.gradient = dW
        self.kappa.gradient = dkappa

    def _gradient_reduce_weave(self, dL_dK, index, index2):
        dL_dK_small = np.zeros_like(self.B)
        code="""
        for(int i=0; i<num_inducing; i++){
          for(int j=0; j<N; j++){
            dL_dK_small[index[j] + output_dim*index2[i]] += dL_dK[i+j*num_inducing];
          }
        }
        """
        N, num_inducing, output_dim = index.size, index2.size, self.output_dim
        weave.inline(code, ['N', 'num_inducing', 'output_dim', 'dL_dK', 'dL_dK_small', 'index', 'index2'])
        return dL_dK_small

    def _gradient_reduce_numpy(self, dL_dK, index, index2):
        index, index2 = index[:,0], index2[:,0]
        for i in range(k.output_dim):
            tmp1 = dL_dK[index==i]
            for j in range(k.output_dim):
                dL_dK_small[j,i] = tmp1[:,index2==j].sum()

        index = np.asarray(X, dtype=np.int).flatten()
        dL_dKdiag_small = np.array([dL_dKdiag[index==i].sum() for i in xrange(self.output_dim)])
        self.W.gradient = 2.*self.W*dL_dKdiag_small[:, None]
        self.kappa.gradient = dL_dKdiag_small

        return np.zeros(X.shape)

from ...util.caching import Cache_this

class TruncLinear(Kern):
    """
    Truncated Linear kernel

    .. math::

       k(x,y) = \sum_{i=1}^input_dim \sigma^2_i \max(0, x_iy_i - \simga_q)

    :param variances: the vector of variances :math:`\sigma^2_i`
    :type variances: array or list of the appropriate size (or float if there
                     is only one variance parameter)
    :param ARD: Auto Relevance Determination. If False, the kernel has only one
                variance parameter \sigma^2, otherwise there is one variance
                parameter per dimension.
    :type ARD: Boolean
    :rtype: kernel object

    """

    def __init__(self, input_dim, variances=None, delta=None, ARD=False, active_dims=None, name='linear'):
        super(TruncLinear, self).__init__(input_dim, active_dims, name)
        self.ARD = ARD
        if not ARD:
            if variances is not None:
                variances = np.asarray(variances)
                delta = np.asarray(delta)
                assert variances.size == 1, "Only one variance needed for non-ARD kernel"
            else:
                variances = np.ones(1)
                delta = np.zeros(1)
        else:
                assert variances.size == self.input_dim, "bad number of variances, need one ARD variance per input_dim"
            else:
                variances = np.ones(self.input_dim)
                delta = np.zeros(self.input_dim)

        self.variances = Param('variances', variances, Logexp())
        self.delta = Param('delta', delta)
        self.add_parameter(self.variances)
        self.add_parameter(self.delta)
            
    @Cache_this(limit=2)
        XX = self.variances*self._product(X, X2)
        return XX.sum(axis=-1)

    def _product(self, X, X2=None):
            X2 = X
        XX = np.einsum('nq,mq->nmq',X-self.delta,X2-self.delta)
        XX[XX<0] = 0
        return XX

        return (self.variances*np.square(X-self.delta)).sum(axis=-1)

        dK_dvar = self._product(X, X2)
            X2=X
        dK_ddelta = self.variances*(2*self.delta-X[:,None,:]-X2[None,:,:])*(dK_dvar>0)
        if self.ARD:
            self.variances.gradient[:] = np.einsum('nmq,nm->q',dK_dvar,dL_dK)
            self.delta.gradient[:] = np.einsum('nmq,nm->q',dK_ddelta,dL_dK)
        else:
            self.variances.gradient[:] = np.einsum('nmq,nm->',dK_dvar,dL_dK)
            self.delta.gradient[:] = np.einsum('nmq,nm->',dK_ddelta,dL_dK)

            self.variances.gradient[:] = np.einsum('nq,n->q',np.square(X-self.delta),dL_dKdiag)
            self.delta.gradient[:] = np.einsum('nq,n->q',2*self.variances*(self.delta-X),dL_dKdiag)
        else:
            self.variances.gradient[:] = np.einsum('nq,n->',np.square(X-self.delta),dL_dKdiag)
            self.delta.gradient[:] = np.einsum('nq,n->',2*self.variances*(self.delta-X),dL_dKdiag)

        XX = self._product(X, X2)
            Xp = (self.variances*(X-self.delta))*(XX>0)
        else:
            Xp = (self.variances*(X2-self.delta))*(XX>0)
            return np.einsum('nmq,nm->nq',Xp,dL_dK)+np.einsum('mnq,nm->mq',Xp,dL_dK)
        else:
            return np.einsum('nmq,nm->nq',Xp,dL_dK)

        return 2.*self.variances*dL_dKdiag[:,None]*(X-self.delta)

    def input_sensitivity(self):
        return np.ones(self.input_dim) * self.variances

class TruncLinear_inf(Kern):
    """

    .. math::

    """

    def __init__(self, input_dim, interval, variances=None, ARD=False, active_dims=None, name='linear'):
        super(TruncLinear_inf, self).__init__(input_dim, active_dims, name)
        self.interval = interval
        if not ARD:
            else:
        else:
            else:

#     @Cache_this(limit=2)
        tmp = self._product(X, X2)
        return (self.variances*tmp).sum(axis=-1)

            X2 = X
        X_X2 = X[:,None,:]-X2[None,:,:]
        tmp = np.abs(X_X2**3)/6+np.einsum('nq,mq->nmq',X,X2)*(self.interval[1]-self.interval[0]) \
              -(X[:,None,:]+X2[None,:,:])*(self.interval[1]*self.interval[1]-self.interval[0]*self.interval[0])/2+(self.interval[1]**3-self.interval[0]**3)/3.
        return tmp

        tmp = np.square(X)*(self.interval[1]-self.interval[0]) \
              -X*(self.interval[1]*self.interval[1]-self.interval[0]*self.interval[0])+(self.interval[1]**3-self.interval[0]**3)/3

        else:

            self.variances.gradient[:] = np.einsum('nq,n->q',tmp,dL_dKdiag)
        else:
            self.variances.gradient[:] = np.einsum('nq,n->',tmp,dL_dKdiag)

        else:
        else:

'''
Created on 11 Mar 2014

'''
from ...core.parameterization.parameterized import ParametersChangedMeta

def put_clean(dct, name, func):
    if name in dct:
        dct['_clean_{}'.format(name)] = dct[name]
        dct[name] = func(dct[name])

class KernCallsViaSlicerMeta(ParametersChangedMeta):
    def __new__(cls, name, bases, dct):
        put_clean(dct, 'K', _slice_K)
        put_clean(dct, 'Kdiag', _slice_Kdiag)
        put_clean(dct, 'update_gradients_full', _slice_update_gradients_full)
        put_clean(dct, 'update_gradients_diag', _slice_update_gradients_diag)
        put_clean(dct, 'gradients_X', _slice_gradients_X)
        put_clean(dct, 'gradients_X_diag', _slice_gradients_X_diag)

        put_clean(dct, 'psi0', _slice_psi)
        put_clean(dct, 'psi1', _slice_psi)
        put_clean(dct, 'psi2', _slice_psi)
        put_clean(dct, 'update_gradients_expectations', _slice_update_gradients_expectations)
        put_clean(dct, 'gradients_Z_expectations', _slice_gradients_Z_expectations)
        put_clean(dct, 'gradients_qX_expectations', _slice_gradients_qX_expectations)
        return super(KernCallsViaSlicerMeta, cls).__new__(cls, name, bases, dct)

class _Slice_wrap(object):
    def __init__(self, k, X, X2=None):
        self.k = k
        self.shape = X.shape
        assert X.ndim == 2, "only matrices are allowed as inputs to kernels for now, given X.shape={!s}".format(X.shape)
        if X2 is not None:
            assert X2.ndim == 2, "only matrices are allowed as inputs to kernels for now, given X2.shape={!s}".format(X2.shape)
        if (self.k.active_dims is not None) and (self.k._sliced_X == 0):
            self.k._check_active_dims(X)
            self.X = self.k._slice_X(X)
            self.X2 = self.k._slice_X(X2) if X2 is not None else X2
            self.ret = True
        else:
            self.k._check_input_dim(X)
            self.X = X
            self.X2 = X2
            self.ret = False
        self.k._sliced_X += 1
        return self
    def __exit__(self, *a):
        self.k._sliced_X -= 1
    def handle_return_array(self, return_val):
        if self.ret:
            ret = np.zeros(self.shape)
            ret[:, self.k.active_dims] = return_val
            return ret
        return return_val

def _slice_K(f):
    @wraps(f)
    def wrap(self, X, X2 = None, *a, **kw):
        with _Slice_wrap(self, X, X2) as s:
            ret = f(self, s.X, s.X2, *a, **kw)
        return ret
    return wrap

def _slice_Kdiag(f):
    @wraps(f)
    def wrap(self, X, *a, **kw):
        with _Slice_wrap(self, X, None) as s:
            ret = f(self, s.X, *a, **kw)
        return ret

def _slice_update_gradients_full(f):
    @wraps(f)
    def wrap(self, dL_dK, X, X2=None):
            ret = f(self, dL_dK, s.X, s.X2)
        return ret

def _slice_update_gradients_diag(f):
    @wraps(f)
    def wrap(self, dL_dKdiag, X):
            ret = f(self, dL_dKdiag, s.X)
        return ret

def _slice_gradients_X(f):
    @wraps(f)
            ret = s.handle_return_array(f(self, dL_dK, s.X, s.X2))
        return ret

def _slice_gradients_X_diag(f):
    @wraps(f)
            ret = s.handle_return_array(f(self, dL_dKdiag, s.X))
        return ret

def _slice_psi(f):
    @wraps(f)
    def wrap(self, Z, variational_posterior):
        with _Slice_wrap(self, Z, variational_posterior) as s:
            ret = f(self, s.X, s.X2)
        return ret

def _slice_update_gradients_expectations(f):
    @wraps(f)
    def wrap(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
            ret = f(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, s.X, s.X2)
        return ret

def _slice_gradients_Z_expectations(f):
    @wraps(f)
            ret = s.handle_return_array(f(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, s.X, s.X2))
        return ret

def _slice_gradients_qX_expectations(f):
    @wraps(f)
        with _Slice_wrap(self, variational_posterior, Z) as s:
            ret = list(f(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, s.X2, s.X))
            r2 = ret[:2]
            ret[0] = s.handle_return_array(r2[0])
            ret[1] = s.handle_return_array(r2[1])
            del r2
        return ret

warnings.filterwarnings("ignore", category=DeprecationWarning)

import core
from core.parameterization import transformations, priors
constraints = transformations
import models
import mappings
import inference
import util
import examples
import likelihoods
import testing
from numpy.testing import Tester
import kern
import plotting

# Direct imports for convenience:
from core import Model
from core.parameterization import Param, Parameterized, ObsAr

#@nottest
try:
    #Get rid of nose dependency by only ignoring if you have nose installed
    from nose.tools import nottest
    @nottest
    def tests():
        Tester(testing).test(verbose=10)
except:

def load(file_path):
    """
    Load a previously pickled model, using `m.pickle('path/to/file.pickle)'

    :param file_name: path/to/file.pickle
    """
    import cPickle as pickle
    try:
        with open(file_path, 'rb') as f:
            m = pickle.load(f)
    except:
        import pickle as pickle
    return m

# Copyright (c) 2012, 2013 Ricardo Andrade

def std_norm_pdf(x):
    """Standard Gaussian density function"""
    return 1./np.sqrt(2.*np.pi)*np.exp(-.5*x**2)

def std_norm_cdf(x):
    """
    Cumulative standard Gaussian distribution
    Based on Abramowitz, M. and Stegun, I. (1970)
    """
    x_shape = np.asarray(x).shape

    if len(x_shape) == 0 or x_shape[0] == 1:
        sign = np.sign(x)
        x *= sign
        x /= np.sqrt(2.)
        t = 1.0/(1.0 +  0.3275911*x)
        erf = 1. - np.exp(-x**2)*t*(0.254829592 + t*(-0.284496736 + t*(1.421413741 + t*(-1.453152027 + t*(1.061405429)))))
        cdf_x = 0.5*(1.0 + sign*erf)
        return cdf_x
    else:
        x = np.atleast_1d(x).copy()
        cdf_x = np.zeros_like(x)
        sign = np.ones_like(x)
        neg_x_ind = x<0
        sign[neg_x_ind] = -1.0
        x[neg_x_ind] = -x[neg_x_ind]
        cdf_x = cdf_x.reshape(x_shape)
    return cdf_x

def std_norm_cdf_weave(x):
    """

    A weave implementation of std_norm_cdf, which is faster. this is unused,
    because of the difficulties of a weave dependency. (see github issue #94)

    """
    #Generalize for many x
    x = np.asarray(x).copy()
    cdf_x = np.zeros_like(x)
    N = x.size
    support_code = "#include <math.h>"
    code = """

    double sign, t, erf;
    for (int i=0; i<N; i++){
        sign = 1.0;
        if (x[i] < 0.0){
            sign = -1.0;
            x[i] = -x[i];
        }
        x[i] = x[i]/sqrt(2.0);

        t = 1.0/(1.0 +  0.3275911*x[i]);

        erf = 1. - exp(-x[i]*x[i])*t*(0.254829592 + t*(-0.284496736 + t*(1.421413741 + t*(-1.453152027 + t*(1.061405429)))));

        //return_val = 0.5*(1.0 + sign*erf);
        cdf_x[i] = 0.5*(1.0 + sign*erf);
    }
    """
    weave.inline(code, arg_names=['x', 'cdf_x', 'N'], support_code=support_code)

def inv_std_norm_cdf(x):
    """
    Inverse cumulative standard Gaussian distribution
    Based on Winitzki, S. (2008)
    """
    z = 2*x -1
    ln1z2 = np.log(1-z**2)
    a = 8*(np.pi -3)/(3*np.pi*(4-np.pi))
    b = 2/(np.pi * a) + ln1z2/2
    inv_erf = np.sign(z) * np.sqrt( np.sqrt(b**2 - ln1z2/a) - b )
    return np.sqrt(2) * inv_erf

# Copyright (c) 2012-2014 GPy Authors

from likelihood import Likelihood

class Exponential(Likelihood):
    """
    Expoential likelihood
    Y is expected to take values in {0,1,2,...}
    -----
    $$
    L(x) = \exp(\lambda) * \lambda**Y_i / Y_i!
    $$
    """
    def __init__(self,gp_link=None):
        if gp_link is None:
            gp_link = link_functions.Log()
        super(Exponential, self).__init__(gp_link, 'ExpLikelihood')

    def pdf_link(self, link_f, y, Y_metadata=None):
        """
        Likelihood function given link(f)

        .. math::
            p(y_{i}|\\lambda(f_{i})) = \\lambda(f_{i})\\exp (-y\\lambda(f_{i}))

        :param link_f: latent variables link(f)
        :type link_f: Nx1 array
        :param Y_metadata: Y_metadata which is not used in exponential distribution
        """
        assert np.atleast_1d(link_f).shape == np.atleast_1d(y).shape
        log_objective = link_f*np.exp(-y*link_f)
        return np.exp(np.sum(np.log(log_objective)))

    def logpdf_link(self, link_f, y, Y_metadata=None):
        """
        Log Likelihood Function given link(f)

        .. math::
            \\ln p(y_{i}|\lambda(f_{i})) = \\ln \\lambda(f_{i}) - y_{i}\\lambda(f_{i})

        :param link_f: latent variables (link(f))

        """
        log_objective = np.log(link_f) - y*link_f
        return np.sum(log_objective)

    def dlogpdf_dlink(self, link_f, y, Y_metadata=None):
        """
        Gradient of the log likelihood function at y, given link(f) w.r.t link(f)

        .. math::
            \\frac{d \\ln p(y_{i}|\lambda(f_{i}))}{d\\lambda(f)} = \\frac{1}{\\lambda(f)} - y_{i}

        :param link_f: latent variables (f)
        :returns: gradient of likelihood evaluated at points
        :rtype: Nx1 array

        """
        grad = 1./link_f - y
        #grad = y/(link_f**2) - 1./link_f
        return grad

    def d2logpdf_dlink2(self, link_f, y, Y_metadata=None):
        """
        Hessian at y, given link(f), w.r.t link(f)
        i.e. second derivative logpdf at y given link(f_i) and link(f_j)  w.r.t link(f_i) and link(f_j)
        The hessian will be 0 unless i == j

        .. math::
            \\frac{d^{2} \\ln p(y_{i}|\lambda(f_{i}))}{d^{2}\\lambda(f)} = -\\frac{1}{\\lambda(f_{i})^{2}}

        :returns: Diagonal of hessian matrix (second derivative of likelihood evaluated at points f)

        .. Note::
            Will return diagonal of hessian, since every where else it is 0, as the likelihood factorizes over cases
            (the distribution for y_i depends only on link(f_i) not on link(f_(j!=i))
        """
        hess = -1./(link_f**2)
        #hess = -2*y/(link_f**3) + 1/(link_f**2)
        return hess

    def d3logpdf_dlink3(self, link_f, y, Y_metadata=None):
        """
        Third order derivative log-likelihood function at y given link(f) w.r.t link(f)

        .. math::
            \\frac{d^{3} \\ln p(y_{i}|\lambda(f_{i}))}{d^{3}\\lambda(f)} = \\frac{2}{\\lambda(f_{i})^{3}}

        :returns: third derivative of likelihood evaluated at points f
        """
        d3lik_dlink3 = 2./(link_f**3)
        #d3lik_dlink3 = 6*y/(link_f**4) - 2./(link_f**3)
        return d3lik_dlink3

    def samples(self, gp):
        """

        """
        orig_shape = gp.shape
        gp = gp.flatten()
        Ysim = np.random.exponential(1.0/self.gp_link.transf(gp))
        return Ysim.reshape(orig_shape)

from scipy import stats, special
from gaussian import Gaussian
from ..core.parameterization import Param
from ..core.parameterization.transformations import Logexp

class MixedNoise(Likelihood):
    def __init__(self, likelihoods_list, name='mixed_noise'):
        #NOTE at the moment this likelihood only works for using a list of gaussians
        super(Likelihood, self).__init__(name=name)

        self.link_parameters(*likelihoods_list)
        self.likelihoods_list = likelihoods_list

    def gaussian_variance(self, Y_metadata):
        assert all([isinstance(l, Gaussian) for l in self.likelihoods_list])
        ind = Y_metadata['output_index'].flatten()
        variance = np.zeros(ind.size)
        for lik, j in zip(self.likelihoods_list, range(len(self.likelihoods_list))):
            variance[ind==j] = lik.variance
        return variance

    def betaY(self,Y,Y_metadata):
        #TODO not here.
        return Y/self.gaussian_variance(Y_metadata=Y_metadata)[:,None]

    def update_gradients(self, gradients):
        self.gradient = gradients

    def exact_inference_gradients(self, dL_dKdiag, Y_metadata):
        return np.array([dL_dKdiag[ind==i].sum() for i in range(len(self.likelihoods_list))])

        _variance = np.array([self.likelihoods_list[j].variance for j in ind ])
        if full_cov:
            var += np.eye(var.shape[0])*_variance
        else:
            var += _variance
        return mu, var

    def predictive_variance(self, mu, sigma, Y_metadata):
        _variance = self.gaussian_variance(Y_metadata)
        return _variance + sigma**2

    def predictive_quantiles(self, mu, var, quantiles, Y_metadata):
        outputs = np.unique(ind)
        Q = np.zeros( (mu.size,len(quantiles)) )
        for j in outputs:
            q = self.likelihoods_list[j].predictive_quantiles(mu[ind==j,:],
                var[ind==j,:],quantiles,Y_metadata=None)
            Q[ind==j,:] = np.hstack(q)
        return [q[:,None] for q in Q.T]

    def samples(self, gp, Y_metadata):
        """

        """
        N1, N2 = gp.shape
        Ysim = np.zeros((N1,N2))
        for j in np.unique(ind):
            flt = ind==j
            gp_filtered = gp[flt,:]
            n1 = gp_filtered.shape[0]
            lik = self.likelihoods_list[j]
            _ysim = np.array([np.random.normal(lik.gp_link.transf(gpj), scale=np.sqrt(lik.variance), size=1) for gpj in gp_filtered.flatten()])
            Ysim[flt,:] = _ysim.reshape(n1,N2)
        return Ysim

try:
except:
    pass

def univariate_plot(prior):
    rvs = prior.rvs(1000)
    pb.hist(rvs, 100, normed=True)
    xmin, xmax = pb.xlim()
    xx = np.linspace(xmin, xmax, 1000)
    pb.plot(xx, prior.pdf(xx), 'r', linewidth=2)

def plot(prior):

    if prior.input_dim == 2:
        rvs = prior.rvs(200)
        pb.plot(rvs[:, 0], rvs[:, 1], 'kx', mew=1.5)
        xmin, xmax = pb.xlim()
        ymin, ymax = pb.ylim()
        xx, yy = np.mgrid[xmin:xmax:100j, ymin:ymax:100j]
        xflat = np.vstack((xx.flatten(), yy.flatten())).T
        zz = prior.pdf(xflat).reshape(100, 100)
        pb.contour(xx, yy, zz, linewidths=2)

    else:

from ..core import GP, Param
from .. import util

class GPLVM(GP):
    """
    Gaussian Process Latent Variable Model

    """
    def __init__(self, Y, input_dim, init='PCA', X=None, kernel=None, name="gplvm"):

        """
        :param Y: observed data
        :type Y: np.ndarray
        :param input_dim: latent dimensionality
        :type input_dim: int
        :param init: initialisation method for the latent space
        :type init: 'PCA'|'random'
        """
        else:
            kernel = kern.RBF(input_dim, lengthscale=fracs, ARD=input_dim > 1) + kern.Bias(input_dim, np.exp(-2))

        likelihood = Gaussian()

        super(GPLVM, self).__init__(X, Y, kernel, likelihood, name='GPLVM')
        self.X = Param('latent_mean', X)
        self.link_parameter(self.X, index=0)

        super(GPLVM, self).parameters_changed()
        self.X.gradient = self.kern.gradients_X(self.grad_dict['dL_dK'], self.X, None)

    def jacobian(self,X):
        J = np.zeros((X.shape[0],X.shape[1],self.output_dim))
        for i in range(self.output_dim):
            J[:,:,i] = self.kern.gradients_X(self.posterior.woodbury_vector[:,i:i+1], X, self.X)
        return J

    def magnification(self,X):
        target=np.zeros(X.shape[0])
        #J = np.zeros((X.shape[0],X.shape[1],self.output_dim))
        J = self.jacobian(X)
        for i in range(X.shape[0]):
            target[i]=np.sqrt(np.linalg.det(np.dot(J[i,:,:],np.transpose(J[i,:,:]))))

    def plot(self):
        assert self.likelihood.Y.shape[1] == 2
        pb.scatter(self.likelihood.Y[:, 0], self.likelihood.Y[:, 1], 40, self.X[:, 0].copy(), linewidth=0, cmap=pb.cm.jet)  # @UndefinedVariable
        Xnew = np.linspace(self.X.min(), self.X.max(), 200)[:, None]
        mu, _ = self.predict(Xnew)
        import pylab as pb
        pb.plot(mu[:, 0], mu[:, 1], 'k', linewidth=1.5)

                fignum=None, legend=True,
                aspect='auto', updates=False, **kwargs):
        import sys

                fignum, False, legend,
                plot_limits, aspect, updates, **kwargs)

    def plot_magnification(self, *args, **kwargs):
        return util.plot_latent.plot_magnification(self, *args, **kwargs)

from ...util.linalg import pdinv, dpotrs, tdot

class ExactGaussianInference(LatentFunctionInference):
    """
    An object for inference when the likelihood is Gaussian.

    """
        pass#self._YYTfactor_cache = caching.cache()

    def get_YYTfactor(self, Y):
        """
        find a matrix L which satisfies LL^T = YY^T.

        Note that L may have fewer columns than Y, else L=Y.
        """
        if (N>D):
            return Y
        else:
            #if Y in self.cache, return self.Cache[Y], else store Y in cache and return L.
            #print "WARNING: N>D of Y, we need caching of L, such that L*L^T = Y, returning Y still!"
            return Y

    def inference(self, kern, X, likelihood, Y, Y_metadata=None):
        """
        Returns a Posterior class containing essential quantities of the posterior
        """
        YYT_factor = self.get_YYTfactor(Y)

        Ky = K.copy()
        diag.add(Ky, likelihood.gaussian_variance(Y_metadata))
        Wi, LW, LWi, W_logdet = pdinv(Ky)

        alpha, _ = dpotrs(LW, YYT_factor, lower=1)

        log_marginal =  0.5*(-Y.size * log_2_pi - Y.shape[1] * W_logdet - np.sum(alpha * YYT_factor))

        dL_dK = 0.5 * (tdot(alpha) - Y.shape[1] * Wi)

        dL_dthetaL = likelihood.exact_inference_gradients(np.diag(dL_dK),Y_metadata)

        return Posterior(woodbury_chol=LW, woodbury_vector=alpha, K=K), log_marginal, {'dL_dK':dL_dK, 'dL_dthetaL':dL_dthetaL}

from ...util.linalg import mdot
from ...util.decorators import silence_errors
from ...core.parameterization.param import Param

class Periodic(Kern):
    def __init__(self, input_dim, variance, lengthscale, period, n_freq, lower, upper, active_dims, name):
        """
        :param variance: the variance of the Matern kernel
        :type variance: float
        :param lengthscale: the lengthscale of the Matern kernel
        :type lengthscale: np.ndarray of size (input_dim,)
        :param period: the period
        :type period: float
        :param n_freq: the number of frequencies considered for the periodic subspace
        :type n_freq: int
        :rtype: kernel object
        """

        assert input_dim==1, "Periodic kernels are only defined for input_dim=1"
        super(Periodic, self).__init__(input_dim, active_dims, name)
        self.lower,self.upper = lower, upper
        self.n_freq = n_freq
        self.n_basis = 2*n_freq
        self.variance = Param('variance', np.float64(variance), Logexp())
        self.lengthscale = Param('lengthscale', np.float64(lengthscale), Logexp())
        self.period = Param('period', np.float64(period), Logexp())
        self.link_parameters(self.variance, self.lengthscale, self.period)

    def _cos(self, alpha, omega, phase):
        def f(x):
            return alpha*np.cos(omega*x + phase)
        return f

    @silence_errors
    def _cos_factorization(self, alpha, omega, phase):
        r1 = np.sum(alpha*np.cos(phase),axis=1)[:,None]
        r2 = np.sum(alpha*np.sin(phase),axis=1)[:,None]
        r =  np.sqrt(r1**2 + r2**2)
        psi = np.where(r1 != 0, (np.arctan(r2/r1) + (r1<0.)*np.pi),np.arcsin(r2))
        return r,omega[:,0:1], psi

    def _int_computation(self,r1,omega1,phi1,r2,omega2,phi2):
        Gint1 = 1./(omega1+omega2.T)*( np.sin((omega1+omega2.T)*self.upper+phi1+phi2.T) - np.sin((omega1+omega2.T)*self.lower+phi1+phi2.T)) + 1./(omega1-omega2.T)*( np.sin((omega1-omega2.T)*self.upper+phi1-phi2.T) - np.sin((omega1-omega2.T)*self.lower+phi1-phi2.T) )
        Gint2 = 1./(omega1+omega2.T)*( np.sin((omega1+omega2.T)*self.upper+phi1+phi2.T) - np.sin((omega1+omega2.T)*self.lower+phi1+phi2.T)) +  np.cos(phi1-phi2.T)*(self.upper-self.lower)
        Gint = np.dot(r1,r2.T)/2 * np.where(np.isnan(Gint1),Gint2,Gint1)
        return Gint

        FX = self._cos(self.basis_alpha[None,:],self.basis_omega[None,:],self.basis_phi[None,:])(X)
            FX2 = FX
        else:
            FX2 = self._cos(self.basis_alpha[None,:],self.basis_omega[None,:],self.basis_phi[None,:])(X2)
        return mdot(FX,self.Gi,FX2.T)

        return np.diag(self.K(X))

class PeriodicExponential(Periodic):
    """
    Kernel of the periodic subspace (up to a given frequency) of a exponential
    (Matern 1/2) RKHS.

    Only defined for input_dim=1.
    """

    def __init__(self, input_dim=1, variance=1., lengthscale=1., period=2.*np.pi, n_freq=10, lower=0., upper=4*np.pi, active_dims=None, name='periodic_exponential'):
        super(PeriodicExponential, self).__init__(input_dim, variance, lengthscale, period, n_freq, lower, upper, active_dims, name)

        self.a = [1./self.lengthscale, 1.]
        self.b = [1]

        self.basis_alpha = np.ones((self.n_basis,))
        self.basis_omega = (2*np.pi*np.arange(1,self.n_freq+1)/self.period).repeat(2)
        self.basis_phi =   np.zeros(self.n_freq * 2)
        self.basis_phi[::2] = -np.pi/2

        self.G = self.Gram_matrix()
        self.Gi = np.linalg.inv(self.G)

    def Gram_matrix(self):
        La = np.column_stack((self.a[0]*np.ones((self.n_basis,1)),self.a[1]*self.basis_omega))
        Lo = np.column_stack((self.basis_omega,self.basis_omega))
        Lp = np.column_stack((self.basis_phi,self.basis_phi+np.pi/2))
        r,omega,phi =  self._cos_factorization(La,Lo,Lp)
        Gint = self._int_computation( r,omega,phi, r,omega,phi)
        Flower = np.array(self._cos(self.basis_alpha,self.basis_omega,self.basis_phi)(self.lower))[:,None]
        return(self.lengthscale/(2*self.variance) * Gint + 1./self.variance*np.dot(Flower,Flower.T))

        """derivative of the covariance matrix with respect to the parameters (shape is N x num_inducing x num_params)"""
        if X2 is None: X2 = X
        FX  = self._cos(self.basis_alpha[None,:],self.basis_omega[None,:],self.basis_phi[None,:])(X)
        FX2 = self._cos(self.basis_alpha[None,:],self.basis_omega[None,:],self.basis_phi[None,:])(X2)

        #dK_dvar
        dK_dvar = 1./self.variance*mdot(FX,self.Gi,FX2.T)

        #dK_dlen
        da_dlen = [-1./self.lengthscale**2,0.]
        dLa_dlen =  np.column_stack((da_dlen[0]*np.ones((self.n_basis,1)),da_dlen[1]*self.basis_omega))
        r1,omega1,phi1 = self._cos_factorization(dLa_dlen,Lo,Lp)
        dGint_dlen = self._int_computation(r1,omega1,phi1, r,omega,phi)
        dGint_dlen = dGint_dlen + dGint_dlen.T
        dG_dlen = 1./2*Gint + self.lengthscale/2*dGint_dlen
        dK_dlen = -mdot(FX,self.Gi,dG_dlen/self.variance,self.Gi,FX2.T)

        #dK_dper
        dFX_dper  = self._cos(-self.basis_alpha[None,:]*self.basis_omega[None,:]/self.period*X ,self.basis_omega[None,:],self.basis_phi[None,:]+np.pi/2)(X)
        dFX2_dper = self._cos(-self.basis_alpha[None,:]*self.basis_omega[None,:]/self.period*X2,self.basis_omega[None,:],self.basis_phi[None,:]+np.pi/2)(X2)

        dLa_dper = np.column_stack((-self.a[0]*self.basis_omega/self.period, -self.a[1]*self.basis_omega**2/self.period))
        dLp_dper = np.column_stack((self.basis_phi+np.pi/2,self.basis_phi+np.pi))
        r1,omega1,phi1 =  self._cos_factorization(dLa_dper,Lo,dLp_dper)

        IPPprim1 =  self.upper*(1./(omega+omega1.T)*np.cos((omega+omega1.T)*self.upper+phi+phi1.T-np.pi/2)  +  1./(omega-omega1.T)*np.cos((omega-omega1.T)*self.upper+phi-phi1.T-np.pi/2))
        IPPprim1 -= self.lower*(1./(omega+omega1.T)*np.cos((omega+omega1.T)*self.lower+phi+phi1.T-np.pi/2)  +  1./(omega-omega1.T)*np.cos((omega-omega1.T)*self.lower+phi-phi1.T-np.pi/2))
        # SIMPLIFY!!!       IPPprim1 = (self.upper - self.lower)*np.cos((omega+omega1.T)*self.upper+phi+phi1.T-np.pi/2)  +  1./(omega-omega1.T)*np.cos((omega-omega1.T)*self.upper+phi-phi1.T-np.pi/2))
        IPPprim2 =  self.upper*(1./(omega+omega1.T)*np.cos((omega+omega1.T)*self.upper+phi+phi1.T-np.pi/2)  + self.upper*np.cos(phi-phi1.T))
        IPPprim2 -= self.lower*(1./(omega+omega1.T)*np.cos((omega+omega1.T)*self.lower+phi+phi1.T-np.pi/2)  + self.lower*np.cos(phi-phi1.T))
        IPPprim = np.where(np.logical_or(np.isnan(IPPprim1), np.isinf(IPPprim1)), IPPprim2, IPPprim1)

        IPPint1 =  1./(omega+omega1.T)**2*np.cos((omega+omega1.T)*self.upper+phi+phi1.T-np.pi)  +  1./(omega-omega1.T)**2*np.cos((omega-omega1.T)*self.upper+phi-phi1.T-np.pi)
        IPPint1 -= 1./(omega+omega1.T)**2*np.cos((omega+omega1.T)*self.lower+phi+phi1.T-np.pi)  +  1./(omega-omega1.T)**2*np.cos((omega-omega1.T)*self.lower+phi-phi1.T-np.pi)
        IPPint2 =  1./(omega+omega1.T)**2*np.cos((omega+omega1.T)*self.upper+phi+phi1.T-np.pi)  + 1./2*self.upper**2*np.cos(phi-phi1.T)
        IPPint2 -= 1./(omega+omega1.T)**2*np.cos((omega+omega1.T)*self.lower+phi+phi1.T-np.pi)  + 1./2*self.lower**2*np.cos(phi-phi1.T)
        #IPPint2[0,0] = (self.upper**2 - self.lower**2)*np.cos(phi[0,0])*np.cos(phi1[0,0])
        IPPint = np.where(np.isnan(IPPint1),IPPint2,IPPint1)

        dLa_dper2 = np.column_stack((-self.a[1]*self.basis_omega/self.period))
        dLp_dper2 = np.column_stack((self.basis_phi+np.pi/2))
        r2,omega2,phi2 = dLa_dper2.T,Lo[:,0:1],dLp_dper2.T

        dGint_dper = np.dot(r,r1.T)/2 * (IPPprim - IPPint) + self._int_computation(r2,omega2,phi2, r,omega,phi)
        dGint_dper = dGint_dper + dGint_dper.T

        dFlower_dper  = np.array(self._cos(-self.lower*self.basis_alpha*self.basis_omega/self.period,self.basis_omega,self.basis_phi+np.pi/2)(self.lower))[:,None]

        dG_dper = 1./self.variance*(self.lengthscale/2*dGint_dper + self.b[0]*(np.dot(dFlower_dper,Flower.T)+np.dot(Flower,dFlower_dper.T)))

        dK_dper = mdot(dFX_dper,self.Gi,FX2.T) - mdot(FX,self.Gi,dG_dper,self.Gi,FX2.T) + mdot(FX,self.Gi,dFX2_dper.T)

        self.variance.gradient = np.sum(dK_dvar*dL_dK)
        self.lengthscale.gradient = np.sum(dK_dlen*dL_dK)
        self.period.gradient = np.sum(dK_dper*dL_dK)

class PeriodicMatern32(Periodic):
    """
    Kernel of the periodic subspace (up to a given frequency) of a Matern 3/2 RKHS. Only defined for input_dim=1.

    :param variance: the variance of the Matern kernel
    :param lengthscale: the lengthscale of the Matern kernel
    :type lengthscale: np.ndarray of size (input_dim,)
    :param period: the period
    :type period: float
    :param n_freq: the number of frequencies considered for the periodic subspace
    :type n_freq: int

    """

    def __init__(self, input_dim=1, variance=1., lengthscale=1., period=2.*np.pi, n_freq=10, lower=0., upper=4*np.pi, active_dims=None, name='periodic_Matern32'):
        super(PeriodicMatern32, self).__init__(input_dim, variance, lengthscale, period, n_freq, lower, upper, active_dims, name)
        self.a = [3./self.lengthscale**2, 2*np.sqrt(3)/self.lengthscale, 1.]
        self.b = [1,self.lengthscale**2/3]

        La = np.column_stack((self.a[0]*np.ones((self.n_basis,1)),self.a[1]*self.basis_omega,self.a[2]*self.basis_omega**2))
        Lo = np.column_stack((self.basis_omega,self.basis_omega,self.basis_omega))
        Lp = np.column_stack((self.basis_phi,self.basis_phi+np.pi/2,self.basis_phi+np.pi))

        F1lower = np.array(self._cos(self.basis_alpha*self.basis_omega,self.basis_omega,self.basis_phi+np.pi/2)(self.lower))[:,None]
        return(self.lengthscale**3/(12*np.sqrt(3)*self.variance) * Gint + 1./self.variance*np.dot(Flower,Flower.T) + self.lengthscale**2/(3.*self.variance)*np.dot(F1lower,F1lower.T))

    def update_gradients_full(self,dL_dK,X,X2):
        """derivative of the covariance matrix with respect to the parameters (shape is num_data x num_inducing x num_params)"""

        #dK_dvar

        #dK_dlen
        da_dlen = [-6/self.lengthscale**3,-2*np.sqrt(3)/self.lengthscale**2,0.]
        db_dlen = [0.,2*self.lengthscale/3.]
        dLa_dlen =  np.column_stack((da_dlen[0]*np.ones((self.n_basis,1)),da_dlen[1]*self.basis_omega,da_dlen[2]*self.basis_omega**2))
        dG_dlen = self.lengthscale**2/(4*np.sqrt(3))*Gint + self.lengthscale**3/(12*np.sqrt(3))*dGint_dlen + db_dlen[0]*np.dot(Flower,Flower.T) + db_dlen[1]*np.dot(F1lower,F1lower.T)

        #dK_dper

        dLa_dper = np.column_stack((-self.a[0]*self.basis_omega/self.period, -self.a[1]*self.basis_omega**2/self.period, -self.a[2]*self.basis_omega**3/self.period))
        dLp_dper = np.column_stack((self.basis_phi+np.pi/2,self.basis_phi+np.pi,self.basis_phi+np.pi*3/2))

        IPPprim = np.where(np.isnan(IPPprim1),IPPprim2,IPPprim1)

        dLa_dper2 = np.column_stack((-self.a[1]*self.basis_omega/self.period, -2*self.a[2]*self.basis_omega**2/self.period))
        dLp_dper2 = np.column_stack((self.basis_phi+np.pi/2,self.basis_phi+np.pi))
        r2,omega2,phi2 =  self._cos_factorization(dLa_dper2,Lo[:,0:2],dLp_dper2)

        dGint_dper = np.dot(r,r1.T)/2 * (IPPprim - IPPint) +  self._int_computation(r2,omega2,phi2, r,omega,phi)

        dF1lower_dper = np.array(self._cos(-self.lower*self.basis_alpha*self.basis_omega**2/self.period,self.basis_omega,self.basis_phi+np.pi)(self.lower)+self._cos(-self.basis_alpha*self.basis_omega/self.period,self.basis_omega,self.basis_phi+np.pi/2)(self.lower))[:,None]

        dG_dper = 1./self.variance*(self.lengthscale**3/(12*np.sqrt(3))*dGint_dper + self.b[0]*(np.dot(dFlower_dper,Flower.T)+np.dot(Flower,dFlower_dper.T)) + self.b[1]*(np.dot(dF1lower_dper,F1lower.T)+np.dot(F1lower,dF1lower_dper.T)))

class PeriodicMatern52(Periodic):
    """
    Kernel of the periodic subspace (up to a given frequency) of a Matern 5/2 RKHS. Only defined for input_dim=1.

    """

    def __init__(self, input_dim=1, variance=1., lengthscale=1., period=2.*np.pi, n_freq=10, lower=0., upper=4*np.pi, active_dims=None, name='periodic_Matern52'):
        super(PeriodicMatern52, self).__init__(input_dim, variance, lengthscale, period, n_freq, lower, upper, active_dims, name)

        self.a = [5*np.sqrt(5)/self.lengthscale**3, 15./self.lengthscale**2,3*np.sqrt(5)/self.lengthscale, 1.]
        self.b  = [9./8, 9*self.lengthscale**4/200., 3*self.lengthscale**2/5., 3*self.lengthscale**2/(5*8.), 3*self.lengthscale**2/(5*8.)]

        self.basis_alpha = np.ones((2*self.n_freq,))

        La = np.column_stack((self.a[0]*np.ones((self.n_basis,1)), self.a[1]*self.basis_omega, self.a[2]*self.basis_omega**2, self.a[3]*self.basis_omega**3))
        Lo = np.column_stack((self.basis_omega, self.basis_omega, self.basis_omega, self.basis_omega))
        Lp = np.column_stack((self.basis_phi, self.basis_phi+np.pi/2, self.basis_phi+np.pi, self.basis_phi+np.pi*3/2))

        F2lower = np.array(self._cos(self.basis_alpha*self.basis_omega**2,self.basis_omega,self.basis_phi+np.pi)(self.lower))[:,None]
        lower_terms = self.b[0]*np.dot(Flower,Flower.T) + self.b[1]*np.dot(F2lower,F2lower.T) + self.b[2]*np.dot(F1lower,F1lower.T) + self.b[3]*np.dot(F2lower,Flower.T) + self.b[4]*np.dot(Flower,F2lower.T)
        return(3*self.lengthscale**5/(400*np.sqrt(5)*self.variance) * Gint + 1./self.variance*lower_terms)

        #dK_dvar

        #dK_dlen
        da_dlen = [-3*self.a[0]/self.lengthscale, -2*self.a[1]/self.lengthscale, -self.a[2]/self.lengthscale, 0.]
        db_dlen = [0., 4*self.b[1]/self.lengthscale, 2*self.b[2]/self.lengthscale, 2*self.b[3]/self.lengthscale, 2*self.b[4]/self.lengthscale]
        dLa_dlen =  np.column_stack((da_dlen[0]*np.ones((self.n_basis,1)), da_dlen[1]*self.basis_omega, da_dlen[2]*self.basis_omega**2, da_dlen[3]*self.basis_omega**3))
        dlower_terms_dlen = db_dlen[0]*np.dot(Flower,Flower.T) + db_dlen[1]*np.dot(F2lower,F2lower.T) + db_dlen[2]*np.dot(F1lower,F1lower.T) + db_dlen[3]*np.dot(F2lower,Flower.T) + db_dlen[4]*np.dot(Flower,F2lower.T)
        dG_dlen = 15*self.lengthscale**4/(400*np.sqrt(5))*Gint + 3*self.lengthscale**5/(400*np.sqrt(5))*dGint_dlen + dlower_terms_dlen

        #dK_dper

        dLa_dper = np.column_stack((-self.a[0]*self.basis_omega/self.period, -self.a[1]*self.basis_omega**2/self.period, -self.a[2]*self.basis_omega**3/self.period, -self.a[3]*self.basis_omega**4/self.period))
        dLp_dper = np.column_stack((self.basis_phi+np.pi/2,self.basis_phi+np.pi,self.basis_phi+np.pi*3/2,self.basis_phi))

        dLa_dper2 = np.column_stack((-self.a[1]*self.basis_omega/self.period, -2*self.a[2]*self.basis_omega**2/self.period, -3*self.a[3]*self.basis_omega**3/self.period))
        dLp_dper2 = np.column_stack((self.basis_phi+np.pi/2, self.basis_phi+np.pi, self.basis_phi+np.pi*3/2))

        dF2lower_dper = np.array(self._cos(-self.lower*self.basis_alpha*self.basis_omega**3/self.period,self.basis_omega,self.basis_phi+np.pi*3/2)(self.lower) + self._cos(-2*self.basis_alpha*self.basis_omega**2/self.period,self.basis_omega,self.basis_phi+np.pi)(self.lower))[:,None]

        dlower_terms_dper  = self.b[0] * (np.dot(dFlower_dper,Flower.T) + np.dot(Flower.T,dFlower_dper))
        dlower_terms_dper += self.b[1] * (np.dot(dF2lower_dper,F2lower.T) + np.dot(F2lower,dF2lower_dper.T)) - 4*self.b[1]/self.period*np.dot(F2lower,F2lower.T)
        dlower_terms_dper += self.b[2] * (np.dot(dF1lower_dper,F1lower.T) + np.dot(F1lower,dF1lower_dper.T)) - 2*self.b[2]/self.period*np.dot(F1lower,F1lower.T)
        dlower_terms_dper += self.b[3] * (np.dot(dF2lower_dper,Flower.T) + np.dot(F2lower,dFlower_dper.T)) - 2*self.b[3]/self.period*np.dot(F2lower,Flower.T)
        dlower_terms_dper += self.b[4] * (np.dot(dFlower_dper,F2lower.T) + np.dot(Flower,dF2lower_dper.T)) - 2*self.b[4]/self.period*np.dot(Flower,F2lower.T)

        dG_dper = 1./self.variance*(3*self.lengthscale**5/(400*np.sqrt(5))*dGint_dper + 0.5*dlower_terms_dper)

import os
import math
from GPy.util import datasets as dat
import urllib2

class vertex:
    def __init__(self, name, id, parents=[], children=[], meta = {}):
        self.name = name
        self.id = id
        self.parents = parents
        self.children = children
        self.meta = meta

        return self.name + '(' + str(self.id) + ').'
        
class tree:
        self.vertices = []
        self.vertices.append(vertex(name='root', id=0))

        index = self.find_root()
        return self.branch_str(index)

    def branch_str(self, index, indent=''):
        out = indent + str(self.vertices[index]) + '\n'
        for child in self.vertices[index].children:
            out+=self.branch_str(child, indent+'  ')
        return out

    def find_children(self):
        """Take a tree and set the children according to the parents.

        Takes a tree structure which lists the parents of each vertex
        and computes the children for each vertex and places them in."""
        for i in range(len(self.vertices)):
            self.vertices[i].children = []
            for parent in self.vertices[i].parents:
                if i not in self.vertices[parent].children:
                    self.vertices[parent].children.append(i) 

    def find_parents(self):
        """Take a tree and set the parents according to the children

        Takes a tree structure which lists the children of each vertex
        and computes the parents for each vertex and places them in."""
            self.vertices[i].parents = []
            for child in self.vertices[i].children:
                if i not in self.vertices[child].parents:
                    self.vertices[child].parents.append(i) 
                    
    def find_root(self):
        """Finds the index of the root node of the tree."""
        self.find_parents()
        index = 0
        while len(self.vertices[index].parents)>0:
            index = self.vertices[index].parents[0]
        return index
            
    def get_index_by_id(self, id):
        """Give the index associated with a given vertex id."""
            if self.vertices[i].id == id:
                return i
        raise ValueError('Reverse look up of id failed.')

    def get_index_by_name(self, name):
        """Give the index associated with a given vertex name."""
            if self.vertices[i].name == name:
                return i
        raise ValueError('Reverse look up of name failed.')

    def order_vertices(self):
        """Order vertices in the graph such that parents always have a lower index than children."""
        
        ordered = False
        while ordered == False:
            for i in range(len(self.vertices)):
                ordered = True
                for parent in self.vertices[i].parents:
                    if parent>i:
                        ordered = False
                        self.swap_vertices(i, parent)

    def swap_vertices(self, i, j):
        """
        Swap two vertices in the tree structure array.
        swap_vertex swaps the location of two vertices in a tree structure array. 

        :param tree: the tree for which two vertices are to be swapped.
        :param i: the index of the first vertex to be swapped.
        :param j: the index of the second vertex to be swapped.
        :rval tree: the tree structure with the two vertex locations swapped.

        """
        store_vertex_i = self.vertices[i]
        store_vertex_j = self.vertices[j]
        self.vertices[j] = store_vertex_i
        self.vertices[i] = store_vertex_j
        for k in range(len(self.vertices)):
            for swap_list in [self.vertices[k].children, self.vertices[k].parents]:
                if i in swap_list:
                    swap_list[swap_list.index(i)] = -1
                if j in swap_list:
                    swap_list[swap_list.index(j)] = i
                if -1 in swap_list:
                    swap_list[swap_list.index(-1)] = j

def rotation_matrix(xangle, yangle, zangle, order='zxy', degrees=False):

    """

    Compute the rotation matrix for an angle in each direction.
    This is a helper function for computing the rotation matrix for a given set of angles in a given order.

    :param xangle: rotation for x-axis.
    :param yangle: rotation for y-axis.
    :param zangle: rotation for z-axis.
    :param order: the order for the rotations.

     """
    if degrees:
        xangle = math.radians(xangle)
        yangle = math.radians(yangle)
        zangle = math.radians(zangle)

    # Here we assume we rotate z, then x then y.
    c1 = math.cos(xangle) # The x angle
    c2 = math.cos(yangle) # The y angle
    c3 = math.cos(zangle) # the z angle
    s1 = math.sin(xangle)
    s2 = math.sin(yangle)
    s3 = math.sin(zangle)

    # see http://en.wikipedia.org/wiki/Rotation_matrix for
    # additional info.

    if order=='zxy':
        rot_mat = np.array([[c2*c3-s1*s2*s3, c2*s3+s1*s2*c3, -s2*c1],[-c1*s3, c1*c3, s1],[s2*c3+c2*s1*s3, s2*s3-c2*s1*c3, c2*c1]])
    else:
        rot_mat = np.eye(3)
        for i in range(len(order)):
            if order[i]=='x':
                rot_mat = np.dot(np.array([[1, 0, 0], [0,  c1, s1], [0, -s1, c1]]),rot_mat)
            elif order[i] == 'y':
                rot_mat = np.dot(np.array([[c2, 0, -s2], [0, 1, 0], [s2, 0, c2]]),rot_mat)
            elif order[i] == 'z':
                rot_mat = np.dot(np.array([[c3, s3, 0], [-s3, c3, 0], [0, 0, 1]]),rot_mat)

    return rot_mat

# Motion capture data routines.
class skeleton(tree):
        tree.__init__(self)

    def connection_matrix(self):
        connection = np.zeros((len(self.vertices), len(self.vertices)), dtype=bool)
            for j in range(len(self.vertices[i].children)):
                connection[i, self.vertices[i].children[j]] = True
        return connection

    def to_xyz(self, channels):
        raise NotImplementedError, "this needs to be implemented to use the skeleton class"

    def finalize(self):
        """After loading in a skeleton ensure parents are correct, vertex orders are correct and rotation matrices are correct."""

        self.order_vertices()
        self.set_rotation_matrices()

    def smooth_angle_channels(self, channels):
        """Remove discontinuities in angle channels so that they don't cause artifacts in algorithms that rely on the smoothness of the functions."""
        for vertex in self.vertices:
            for col in vertex.meta['rot_ind']:
                if col:
                    for k in range(1, channels.shape[0]):
                        diff=channels[k, col]-channels[k-1, col]
                    if abs(diff+360.)<abs(diff):
                        channels[k:, col]=channels[k:, col]+360.
                    elif abs(diff-360.)<abs(diff):
                        channels[k:, col]=channels[k:, col]-360.

# class bvh_skeleton(skeleton):
#     def __init__(self):
#         skeleton.__init__(self)

#     def to_xyz(self, channels):
        
class acclaim_skeleton(skeleton):
    def __init__(self, file_name=None):
        skeleton.__init__(self)
        self.documentation = []
        self.angle = 'deg'
        self.length = 1.0
        self.mass = 1.0
        self.type = 'acclaim'
        self.vertices[0] = vertex(name='root', id=0,
                             parents = [0], children=[],
                             meta = {'orientation': [], 
                                     'axis': [0., 0., 0.], 
                                     'axis_order': [], 
                                     'C': np.eye(3), 
                                     'Cinv': np.eye(3), 
                                     'channels': [], 
                                     'bodymass': [], 
                                     'confmass': [], 
                                     'order': [], 
                                     'rot_ind': [], 
                                     'pos_ind': [], 
                                     'limits': [],
                                     'xyz': np.array([0., 0., 0.]),
                                     'rot': np.eye(3)})

        if file_name:
            self.load_skel(file_name)

        rot_val = list(self.vertices[0].meta['orientation'])
        for i in range(len(self.vertices[0].meta['rot_ind'])):
            rind = self.vertices[0].meta['rot_ind'][i]
            if rind != -1:
                rot_val[i] += channels[rind]

        self.vertices[0].meta['rot'] = rotation_matrix(rot_val[0],
                                                       rot_val[1],
                                                       rot_val[2],
                                                       self.vertices[0].meta['axis_order'],
                                                       degrees=True)
        # vertex based store of the xyz location
        self.vertices[0].meta['xyz'] = list(self.vertices[0].meta['offset'])

        for i in range(len(self.vertices[0].meta['pos_ind'])):
            pind = self.vertices[0].meta['pos_ind'][i]
            if pind != -1:
                self.vertices[0].meta['xyz'][i] += channels[pind]

        for i in range(len(self.vertices[0].children)):
            ind = self.vertices[0].children[i]
            self.get_child_xyz(ind, channels)

        xyz = []
            xyz.append(vertex.meta['xyz'])
        return np.array(xyz)

    def get_child_xyz(self, ind, channels):

        parent = self.vertices[ind].parents[0]
        children = self.vertices[ind].children
        rot_val = np.zeros(3)
        for j in range(len(self.vertices[ind].meta['rot_ind'])):
            rind = self.vertices[ind].meta['rot_ind'][j]
                rot_val[j] = channels[rind]
            else:
                rot_val[j] = 0
        tdof = rotation_matrix(rot_val[0], rot_val[1], rot_val[2],
                               self.vertices[ind].meta['order'],
                               degrees=True)

        torient = rotation_matrix(self.vertices[ind].meta['axis'][0],
                                  self.vertices[ind].meta['axis'][1],
                                  self.vertices[ind].meta['axis'][2],
                                  self.vertices[ind].meta['axis_order'],
                                  degrees=True)

        torient_inv = rotation_matrix(-self.vertices[ind].meta['axis'][0],
                                      -self.vertices[ind].meta['axis'][1],
                                      -self.vertices[ind].meta['axis'][2],
                                      self.vertices[ind].meta['axis_order'][::-1],
                                      degrees=True)

        self.vertices[ind].meta['rot'] = np.dot(np.dot(np.dot(torient_inv,tdof),torient),self.vertices[parent].meta['rot'])

        self.vertices[ind].meta['xyz'] = self.vertices[parent].meta['xyz'] + np.dot(self.vertices[ind].meta['offset'],self.vertices[ind].meta['rot'])

        for i in range(len(children)):
            cind = children[i]
            self.get_child_xyz(cind, channels)

    def load_channels(self, file_name):

        fid=open(file_name, 'r')
        channels = self.read_channels(fid)
        fid.close()
        return channels

    def load_skel(self, file_name):

        """
        Loads an ASF file into a skeleton structure.

        :param file_name: The file name to load in.

         """         

        fid = open(file_name, 'r')
        self.read_skel(fid)
        self.name = file_name

    def read_bonedata(self, fid):
        """Read bone data from an acclaim skeleton file stream."""

        bone_count = 0
        lin = self.read_line(fid)
        while lin[0]!=':':
            parts = lin.split()
            if parts[0] == 'begin':
                bone_count += 1
                self.vertices.append(vertex(name = '', id=np.NaN,
                                       meta={'name': [],
                                             'id': [], 
                                             'offset': [], 
                                             'orientation': [], 
                                             'axis': [0., 0., 0.], 
                                             'axis_order': [], 
                                             'C': np.eye(3), 
                                             'Cinv': np.eye(3), 
                                             'channels': [], 
                                             'bodymass': [], 
                                             'confmass': [], 
                                             'order': [], 
                                             'rot_ind': [], 
                                             'pos_ind': [], 
                                             'limits': [],
                                             'xyz': np.array([0., 0., 0.]),
                                             'rot': np.eye(3)}))
                lin = self.read_line(fid)

            elif parts[0]=='id':
                self.vertices[bone_count].id = int(parts[1])

                self.vertices[bone_count].children = []

            elif parts[0]=='name':
                self.vertices[bone_count].name = parts[1]

            elif parts[0]=='direction':
                direction = np.array([float(parts[1]), float(parts[2]), float(parts[3])])

            elif parts[0]=='length':
                lgth =  float(parts[1])

            elif parts[0]=='axis':
                self.vertices[bone_count].meta['axis'] = np.array([float(parts[1]),
                                                         float(parts[2]),
                                                         float(parts[3])])
                # order is reversed compared to bvh
                self.vertices[bone_count].meta['axis_order'] =  parts[-1][::-1].lower()

            elif parts[0]=='dof':
                order = []
                for i in range(1, len(parts)):
                    if parts[i]== 'rx':
                        chan = 'Xrotation'
                        order.append('x')
                    elif parts[i] =='ry':
                        chan = 'Yrotation'
                        order.append('y')
                    elif parts[i] == 'rz':
                        chan = 'Zrotation'
                        order.append('z')
                    elif parts[i] == 'tx':
                        chan = 'Xposition'
                    elif parts[i] == 'ty':
                        chan = 'Yposition'
                    elif parts[i] == 'tz':
                        chan = 'Zposition'
                    elif parts[i] == 'l':
                        chan = 'length'
                    self.vertices[bone_count].meta['channels'].append(chan)
                    # order is reversed compared to bvh
                self.vertices[bone_count].meta['order'] = order[::-1]

            elif parts[0]=='limits':
                self.vertices[bone_count].meta['limits'] = [[float(parts[1][1:]),  float(parts[2][:-1])]]

                while lin !='end':
                    parts = lin.split()

                    self.vertices[bone_count].meta['limits'].append([float(parts[0][1:]), float(parts[1][:-1])])
                    lin = self.read_line(fid)
                self.vertices[bone_count].meta['limits'] = np.array(self.vertices[bone_count].meta['limits'])

            elif parts[0]=='end':
                self.vertices[bone_count].meta['offset'] = direction*lgth

        return lin

    def read_channels(self, fid):
        """Read channels from an acclaim file."""
        bones = [[] for i in self.vertices]
        num_channels = 0
            num_channels = num_channels + len(vertex.meta['channels'])

        while lin != ':DEGREES':
            lin = self.read_line(fid)
            if lin == '':
                raise ValueError('Could not find :DEGREES in ' + fid.name)

        counter = 0
        while lin:
            if len(parts)==1:
                frame_no = int(parts[0])
                if frame_no:
                    counter += 1
                    if counter != frame_no:
                        raise ValueError('Unexpected frame number.')
                else:
                    raise ValueError('Single bone name  ...')
            else:
                ind = self.get_index_by_name(parts[0])
                bones[ind].append(np.array([float(channel) for channel in parts[1:]]))

        num_frames = counter

        channels = np.zeros((num_frames, num_channels))

        end_val = 0
            vertex = self.vertices[i]
            if len(vertex.meta['channels'])>0:                
                start_val = end_val
                end_val = end_val + len(vertex.meta['channels'])
                for j in range(num_frames):
                    channels[j, start_val:end_val] = bones[i][j]
            self.resolve_indices(i, start_val)

        self.smooth_angle_channels(channels)

    def read_documentation(self, fid):
        """Read documentation from an acclaim skeleton file stream."""

        while lin[0] != ':':
            self.documentation.append(lin)
        return lin

    def read_hierarchy(self, fid):
        """Read hierarchy information from acclaim skeleton file stream."""

        while lin != 'end':
            if lin != 'begin':
                    self.vertices[ind].children.append(self.get_index_by_name(parts[i]))
        return lin

    def read_line(self, fid):
        """Read a line from a file string and check it isn't either empty or commented before returning."""
        lin = '#'
        while lin[0] == '#':
            lin = fid.readline().strip()
                return lin
        return lin

    def read_root(self, fid):
        """Read the root node from an acclaim skeleton file stream."""
        lin = self.read_line(fid)                    
            if parts[0]=='order':
                order = []
                    if parts[i].lower()=='rx':
                    elif parts[i].lower()=='ry':
                    elif parts[i].lower()=='rz':
                    elif parts[i].lower()=='tx':
                    elif parts[i].lower()=='ty':
                    elif parts[i].lower()=='tz':
                    elif parts[i].lower()=='l':
                    self.vertices[0].meta['channels'].append(chan)
                self.vertices[0].meta['order'] = order[::-1]

                self.vertices[0].meta['axis_order'] = parts[1][::-1].lower()
            elif parts[0]=='position':
                self.vertices[0].meta['offset'] = [float(parts[1]),
                                       float(parts[2]),
                                       float(parts[3])]
            elif parts[0]=='orientation':
                self.vertices[0].meta['orientation'] =  [float(parts[1]),
                                             float(parts[2]),
                                             float(parts[3])]
        return lin
    
    def read_skel(self, fid):
        """Loads an acclaim skeleton format from a file stream."""
        while lin:
            if lin[0]==':':
                if lin[1:]== 'name':
                    self.name = lin
                elif lin[1:]=='units':
                    lin = self.read_units(fid)
                elif lin[1:]=='documentation':
                    lin = self.read_documentation(fid)
                elif lin[1:]=='root':
                    lin = self.read_root(fid)
                elif lin[1:]=='bonedata':
                    lin = self.read_bonedata(fid)
                elif lin[1:]=='hierarchy':
                    lin = self.read_hierarchy(fid)
                elif lin[1:8]=='version':
                    continue
                else: 
                    if not lin:
                        self.finalize()
                        return
            else:
                raise ValueError('Unrecognised file format')
            self.finalize()
            
    def read_units(self, fid):
        """Read units from an acclaim skeleton file stream."""
        lin = self.read_line(fid)                   
            if parts[0]=='mass':
                self.mass = float(parts[1])
                self.length = float(parts[1])
            elif parts[0]=='angle':
                self.angle = parts[1]
        return lin

    def resolve_indices(self, index, start_val):
        """Get indices for the skeleton from the channels when loading in channel data."""

        channels = self.vertices[index].meta['channels']
        base_channel = start_val 
        rot_ind = -np.ones(3, dtype=int)
        pos_ind = -np.ones(3, dtype=int)
        for i in range(len(channels)):
            if channels[i]== 'Xrotation':
                rot_ind[0] = base_channel + i
            elif channels[i]=='Yrotation':
                rot_ind[1] = base_channel + i
            elif channels[i]=='Zrotation':
                rot_ind[2] = base_channel + i
            elif channels[i]=='Xposition':
                pos_ind[0] = base_channel + i
            elif channels[i]=='Yposition':
                pos_ind[1] = base_channel + i
            elif channels[i]=='Zposition':
                pos_ind[2] = base_channel + i
        self.vertices[index].meta['rot_ind'] = list(rot_ind)
        self.vertices[index].meta['pos_ind'] = list(pos_ind)

    def set_rotation_matrices(self):
        """Set the meta information at each vertex to contain the correct matrices C and Cinv as prescribed by the rotations and rotation orders."""
            self.vertices[i].meta['C'] = rotation_matrix(self.vertices[i].meta['axis'][0], 
                                                         self.vertices[i].meta['axis'][1], 
                                                         self.vertices[i].meta['axis'][2], 
                                                         self.vertices[i].meta['axis_order'],
                                                         degrees=True)
            # Todo: invert this by applying angle operations in reverse order
            self.vertices[i].meta['Cinv'] = np.linalg.inv(self.vertices[i].meta['C'])
            
# Utilities for loading in x,y,z data.
def load_text_data(dataset, directory, centre=True):
    """Load in a data set of marker points from the Ohio State University C3D motion capture files (http://accad.osu.edu/research/mocap/mocap_data.htm)."""

    points, point_names = parse_text(os.path.join(directory, dataset + '.txt'))[0:2]
    # Remove markers where there is a NaN
    present_index = [i for i in range(points[0].shape[1]) if not (np.any(np.isnan(points[0][:, i])) or np.any(np.isnan(points[0][:, i])) or np.any(np.isnan(points[0][:, i])))]

    point_names = point_names[present_index]
    for i in range(3):
        points[i] = points[i][:, present_index]
        if centre:
            points[i] = (points[i].T - points[i].mean(axis=1)).T 

    # Concatanate the X, Y and Z markers together
    Y = np.concatenate((points[0], points[1], points[2]), axis=1)
    Y = Y/400.
    connect = read_connections(os.path.join(directory, 'connections.txt'), point_names)
    return Y, connect

def parse_text(file_name):
    """Parse data from Ohio State University text mocap files (http://accad.osu.edu/research/mocap/mocap_data.htm)."""

    # Read the header
    fid = open(file_name, 'r')
    point_names = np.array(fid.readline().split())[2:-1:3]
    for i in range(len(point_names)):
        point_names[i] = point_names[i][0:-2]

    # Read the matrix data
    S = np.loadtxt(file_name, skiprows=1)
    field = np.uint(S[:, 0])
    times = S[:, 1]
    S = S[:, 2:]

    # Set the -9999.99 markers to be not present
    S[S==-9999.99] = np.NaN

    # Store x, y and z in different arrays
    points = []
    points.append(S[:, 0:-1:3])
    points.append(S[:, 1:-1:3])
    points.append(S[:, 2:-1:3])

    return points, point_names, times

def read_connections(file_name, point_names):
    """Read a file detailing which markers should be connected to which for motion capture data."""

    connections = []
    line=fid.readline()
    while(line):
        connections.append(np.array(line.split(',')))
        connections[-1][0] = connections[-1][0].strip()
        connections[-1][1] = connections[-1][1].strip()
        line = fid.readline()
    connect = np.zeros((len(point_names), len(point_names)),dtype=bool)
        for j in range(len(point_names)):
            for k in range(len(connections)):
                if connections[k][0] == point_names[i] and connections[k][1] == point_names[j]:
                    
                    connect[i,j]=True
                    connect[j,i]=True
                    break
    
    return connect

skel = acclaim_skeleton()

'''
Created on 13 Mar 2014

'''
#import cPickle as pickle
import pickle
from GPy.core.parameterization.index_operations import ParameterIndexOperations,\
    ParameterIndexOperationsView
import tempfile
from GPy.core.parameterization.priors import Gaussian
from GPy.kern._src.rbf import RBF
from GPy.kern._src.linear import Linear
from GPy.kern._src.static import Bias, White
from GPy.examples.dimensionality_reduction import mrd_simulation
from GPy.core.parameterization.variational import NormalPosterior
from GPy.models.gp_regression import GPRegression

def toy_model():
    X = np.linspace(0,1,50)[:, None]
    Y = np.sin(X)
    m = GPRegression(X=X, Y=Y)
    return m

class ListDictTestCase(unittest.TestCase):
    def assertListDictEquals(self, d1, d2, msg=None):
        for k,v in d1.iteritems():
            self.assertListEqual(list(v), list(d2[k]), msg)
    def assertArrayListEquals(self, l1, l2):
        for a1, a2 in itertools.izip(l1,l2):
            np.testing.assert_array_equal(a1, a2)

class Test(ListDictTestCase):
    def test_parameter_index_operations(self):
        pio = ParameterIndexOperations(dict(test1=np.array([4,3,1,6,4]), test2=np.r_[2:130]))
        piov = ParameterIndexOperationsView(pio, 20, 250)
        self.assertListDictEquals(dict(piov.items()), dict(piov.copy().iteritems()))
        self.assertListDictEquals(dict(pio.iteritems()), dict(pio.copy().items()))

        self.assertArrayListEquals(pio.copy().indices(), pio.indices())
        self.assertArrayListEquals(piov.copy().indices(), piov.indices())

        with tempfile.TemporaryFile('w+b') as f:
            pickle.dump(pio, f)
            f.seek(0)
            pio2 = pickle.load(f)
            self.assertListDictEquals(pio._properties, pio2._properties)

            pickle.dump(piov, f)
            f.seek(0)
            self.assertListDictEquals(dict(piov.items()), dict(pio2.iteritems()))

    def test_param(self):
        param = Param('test', np.arange(4*2).reshape(4,2))
        param[0].constrain_positive()
        param[1].fix()
        param[2].set_prior(Gaussian(0,1))
        pcopy = param.copy()
        self.assertListEqual(param.tolist(), pcopy.tolist())
        self.assertListEqual(str(param).split('\n'), str(pcopy).split('\n'))
        self.assertIsNot(param, pcopy)
            pickle.dump(param, f)
            f.seek(0)
            pcopy = pickle.load(f)
        self.assertSequenceEqual(str(param), str(pcopy))

    def test_observable_array(self):
        obs = ObsAr(np.arange(4*2).reshape(4,2))
        pcopy = obs.copy()
        self.assertListEqual(obs.tolist(), pcopy.tolist())
            pickle.dump(obs, f)
            f.seek(0)
        self.assertSequenceEqual(str(obs), str(pcopy))

    def test_parameterized(self):
        par = RBF(1, active_dims=[1]) + Linear(2, active_dims=[0,2]) + Bias(3) + White(3)
        par.gradient = 10
        par.randomize()
        pcopy = par.copy()
        self.assertIsInstance(pcopy.constraints, ParameterIndexOperations)
        self.assertIsInstance(pcopy.rbf.constraints, ParameterIndexOperationsView)
        self.assertIs(pcopy.constraints, pcopy.rbf.constraints._param_index_ops)
        self.assertIs(pcopy.constraints, pcopy.rbf.lengthscale.constraints._param_index_ops)
        self.assertIs(pcopy.constraints, pcopy.linear.constraints._param_index_ops)
        self.assertListEqual(par.param_array.tolist(), pcopy.param_array.tolist())
        pcopy.gradient = 10 # gradient does not get copied anymore
        self.assertListEqual(par.gradient_full.tolist(), pcopy.gradient_full.tolist())
        self.assertSequenceEqual(str(par), str(pcopy))
        self.assertIsNot(par.param_array, pcopy.param_array)
        self.assertIsNot(par.gradient_full, pcopy.gradient_full)
            par.pickle(f)
            f.seek(0)
        pcopy.gradient = 10
        np.testing.assert_allclose(par.linear.gradient_full, pcopy.linear.gradient_full)
        np.testing.assert_allclose(pcopy.linear.gradient_full, 10)

    def test_model(self):
        par = toy_model()
        np.testing.assert_allclose(par.gradient_full, pcopy.gradient_full)
        self.assertTrue(pcopy.checkgrad())
        self.assert_(np.any(pcopy.gradient!=0.0))
            f.seek(0)
        self.assert_(pcopy.checkgrad())

    def test_modelrecreation(self):
        pcopy = GPRegression(par.X.copy(), par.Y.copy(), kernel=par.kern.copy())
        np.testing.assert_allclose(par.param_array, pcopy.param_array)
        pcopy.optimize('bfgs')
        par.optimize('bfgs')
        np.testing.assert_allclose(pcopy.param_array, par.param_array, atol=1e-6)
            f.seek(0)
        np.testing.assert_allclose(par.gradient_full, pcopy.gradient_full, atol=1e-6)

    def test_posterior(self):
        X = np.random.randn(3,5)
        Xv = np.random.rand(*X.shape)
        par = NormalPosterior(X,Xv)
            f.seek(0)
        np.testing.assert_allclose(pcopy.mean.gradient_full, 10)

    def test_model_concat(self):
        par = mrd_simulation(optimize=0, plot=0, plot_sim=0)
        self.assertTrue(par.checkgrad())
            f.seek(0)

    def _callback(self, what, which):
        what.count += 1

    @unittest.skip
    def test_add_observer(self):
        par.name = "original"
        par.count = 0
        par.add_observer(self, self._callback, 1)
        self.assertNotIn(par.observers[0], pcopy.observers)
        pcopy.name = "copy"
        self.assertTrue(pcopy.kern.checkgrad())
        import ipdb;ipdb.set_trace()
        self.assertIn(par.observers[0], pcopy.observers)
        self.assertEqual(par.count, 3)
        self.assertEqual(pcopy.count, 6) # 3 of each call to checkgrad

    #import sys;sys.argv = ['', 'Test.test_parameter_index_operations']

# Copyright (c) 2012 - 2014 the GPy Austhors (see AUTHORS.txt)

class SparseGPCoregionalizedRegression(SparseGP):
    """
    Sparse Gaussian Process model for heteroscedastic multioutput regression

    This is a thin wrapper around the SparseGP class, with a set of sensible defaults

    :param X_list: list of input observations corresponding to each output
    :type X_list: list of numpy arrays
    :param Y_list: list of observed values related to the different noise models
    :type Y_list: list of numpy arrays
    :param Z_list: list of inducing inputs (optional)
    :type Z_list: empty list | list of numpy arrays
    :param kernel: a GPy kernel, defaults to RBF ** Coregionalized
    :type kernel: None | GPy.kernel defaults
    :likelihoods_list: a list of likelihoods, defaults to list of Gaussian likelihoods
    :type likelihoods_list: None | a list GPy.likelihoods
    :param num_inducing: number of inducing inputs, defaults to 10 per output (ignored if Z_list is not empty)
    :type num_inducing: integer | list of integers

    :param name: model name
    :type name: string
    :param W_rank: number tuples of the corregionalization parameters 'W' (see coregionalize kernel documentation)
    :type W_rank: integer
    :param kernel_name: name of the kernel
    :type kernel_name: string
    """

    def __init__(self, X_list, Y_list, Z_list=[], kernel=None, likelihoods_list=None, num_inducing=10, X_variance=None, name='SGPCR',W_rank=1,kernel_name='coreg'):

        #Input and Output
        X,Y,self.output_index = util.multioutput.build_XY(X_list,Y_list)
        Ny = len(Y_list)

        #Kernel
            kernel = util.multioutput.ICM(input_dim=X.shape[1]-1, num_outputs=Ny, kernel=kern.RBF(X.shape[1]-1), W_rank=1,name=kernel_name)

        #Likelihood
        likelihood = util.multioutput.build_likelihood(Y_list,self.output_index,likelihoods_list)

        #Inducing inputs list
        if len(Z_list):
            assert len(Z_list) == Ny, 'Number of outputs do not match length of inducing inputs list.'
        else:
            if isinstance(num_inducing,np.int):
                num_inducing = [num_inducing] * Ny
            num_inducing = np.asarray(num_inducing)
            assert num_inducing.size == Ny, 'Number of outputs do not match length of inducing inputs list.'
            for ni,Xi in zip(num_inducing,X_list):
                i = np.random.permutation(Xi.shape[0])[:ni]
                Z_list.append(Xi[i].copy())

        Z, _, Iz = util.multioutput.build_XY(Z_list)

        super(SparseGPCoregionalizedRegression, self).__init__(X, Y, Z, kernel, likelihood, inference_method=VarDTC(), Y_metadata={'output_index':self.output_index})
        self['.*inducing'][:,-1].fix()

"""
The package for the psi statistics computation
"""

try:
    from scipy import weave
     
    def _psicomputations(variance, lengthscale, Z, variational_posterior):
        """
        Z - MxQ
        mu - NxQ
        S - NxQ
        gamma - NxQ
        """
        # here are the "statistics" for psi0, psi1 and psi2
        # Produced intermediate results:
        # _psi1                NxM
        mu = variational_posterior.mean
        S = variational_posterior.variance
         
        N,M,Q = mu.shape[0],Z.shape[0],mu.shape[1]
        l2 = np.square(lengthscale)
        log_denom1 = np.log(S/l2+1)
        log_denom2 = np.log(2*S/l2+1)
        log_gamma,log_gamma1 = variational_posterior.gamma_log_prob()
        variance = float(variance)
        psi0 = np.empty(N)
        psi0[:] = variance
        psi1 = np.empty((N,M))
        psi2n = np.empty((N,M,M))
         
        from ....util.misc import param_to_array
        S = param_to_array(S)
        mu = param_to_array(mu)
        Z = param_to_array(Z)
         
        support_code = """
        #include <math.h>
        """
        code = """
        for(int n=0; n<N; n++) {
            for(int m1=0;m1<M;m1++) {
                double log_psi1=0;
                for(int m2=0;m2<=m1;m2++) {
                    double log_psi2_n=0;
                    for(int q=0;q<Q;q++) {
                        double Snq = S(n,q);
                        double lq = l2(q);
                        double Zm1q = Z(m1,q);
                        double Zm2q = Z(m2,q);
                         
                        if(m2==0) {
                            // Compute Psi_1
                            double muZ = mu(n,q)-Z(m1,q);
                             
                            double psi1_exp1 = log_gamma(n,q) - (muZ*muZ/(Snq+lq) +log_denom1(n,q))/2.;
                            double psi1_exp2 = log_gamma1(n,q) -Zm1q*Zm1q/(2.*lq);
                            log_psi1 += (psi1_exp1>psi1_exp2)?psi1_exp1+log1p(exp(psi1_exp2-psi1_exp1)):psi1_exp2+log1p(exp(psi1_exp1-psi1_exp2));
                        }
                        // Compute Psi_2
                        double muZhat = mu(n,q) - (Zm1q+Zm2q)/2.;
                        double Z2 = Zm1q*Zm1q+ Zm2q*Zm2q;
                        double dZ = Zm1q - Zm2q;
                         
                        double psi2_exp1 = dZ*dZ/(-4.*lq)-muZhat*muZhat/(2.*Snq+lq) - log_denom2(n,q)/2. + log_gamma(n,q);
                        double psi2_exp2 = log_gamma1(n,q) - Z2/(2.*lq);
                        log_psi2_n += (psi2_exp1>psi2_exp2)?psi2_exp1+log1p(exp(psi2_exp2-psi2_exp1)):psi2_exp2+log1p(exp(psi2_exp1-psi2_exp2));                    
                    }
                    double exp_psi2_n = exp(log_psi2_n);
                    psi2n(n,m1,m2) = variance*variance*exp_psi2_n;
                    if(m1!=m2) { psi2n(n,m2,m1) = variance*variance*exp_psi2_n;}
                }
                psi1(n,m1) = variance*exp(log_psi1);
            }
        }
        """
        weave.inline(code, support_code=support_code, arg_names=['psi1','psi2n','N','M','Q','variance','l2','Z','mu','S','log_denom1','log_denom2','log_gamma','log_gamma1'], type_converters=weave.converters.blitz)
     
        psi2 = psi2n.sum(axis=0)
        return psi0,psi1,psi2,psi2n
     
    from GPy.util.caching import Cacher
    psicomputations = Cacher(_psicomputations, limit=1)
     
    def psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior):
        ARD = (len(lengthscale)!=1)
         
        _,psi1,_,psi2n = psicomputations(variance, lengthscale, Z, variational_posterior)
     
        gamma, gamma1 = variational_posterior.gamma_probabilities()
     
        dvar = np.zeros(1)
        dmu = np.zeros((N,Q))
        dS = np.zeros((N,Q))
        dgamma = np.zeros((N,Q))
        dl = np.zeros(Q)
        dZ = np.zeros((M,Q))
        dvar += np.sum(dL_dpsi0)
         
        """
        code = """
                for(int m2=0;m2<M;m2++) {
                        double gnq = gamma(n,q);
                        double g1nq = gamma1(n,q);
                        double mu_nq = mu(n,q);
                         
                            // Compute Psi_1                        
                            double lpsi1 = psi1(n,m1)*dL_dpsi1(n,m1);
                            if(q==0) {dvar(0) += lpsi1/variance;}
                             
                            double Zmu = Zm1q - mu_nq;
                            double denom = Snq+lq;
                            double Zmu2_denom = Zmu*Zmu/denom;
                             
                            double exp1 = log_gamma(n,q)-(Zmu*Zmu/(Snq+lq)+log_denom1(n,q))/(2.);
                            double exp2 = log_gamma1(n,q)-Zm1q*Zm1q/(2.*lq);
                            double d_exp1,d_exp2;
                            if(exp1>exp2) {
                                d_exp1 = 1.;
                                d_exp2 = exp(exp2-exp1);
                            } else {
                                d_exp1 = exp(exp1-exp2);
                                d_exp2 = 1.;
                            }
                            double exp_sum = d_exp1+d_exp2;
                             
                            dmu(n,q) += lpsi1*Zmu*d_exp1/(denom*exp_sum);
                            dS(n,q) += lpsi1*(Zmu2_denom-1.)*d_exp1/(denom*exp_sum)/2.;
                            dgamma(n,q) += lpsi1*(d_exp1*g1nq-d_exp2*gnq)/exp_sum;
                            dl(q) += lpsi1*((Zmu2_denom+Snq/lq)/denom*d_exp1+Zm1q*Zm1q/(lq*lq)*d_exp2)/(2.*exp_sum);
                            dZ(m1,q) += lpsi1*(-Zmu/denom*d_exp1-Zm1q/lq*d_exp2)/exp_sum;
                        }
                        double lpsi2 = psi2n(n,m1,m2)*dL_dpsi2(m1,m2);
                        if(q==0) {dvar(0) += lpsi2*2/variance;}
                         
                        double dZm1m2 = Zm1q - Zm2q;
                        double Z2 = Zm1q*Zm1q+Zm2q*Zm2q;
                        double muZhat =  mu_nq - (Zm1q + Zm2q)/2.;
                        double denom = 2.*Snq+lq;
                        double muZhat2_denom = muZhat*muZhat/denom;
                         
                        double exp1 = dZm1m2*dZm1m2/(-4.*lq)-muZhat*muZhat/(2.*Snq+lq) - log_denom2(n,q)/2. + log_gamma(n,q);
                        double exp2 = log_gamma1(n,q) - Z2/(2.*lq);
                        double d_exp1,d_exp2;
                        if(exp1>exp2) {
                            d_exp1 = 1.;
                            d_exp2 = exp(exp2-exp1);
                        } else {
                            d_exp1 = exp(exp1-exp2);
                            d_exp2 = 1.;
                        }
                        double exp_sum = d_exp1+d_exp2;
                         
                        dmu(n,q) += -2.*lpsi2*muZhat/denom*d_exp1/exp_sum;
                        dS(n,q) += lpsi2*(2.*muZhat2_denom-1.)/denom*d_exp1/exp_sum;
                        dgamma(n,q) += lpsi2*(d_exp1*g1nq-d_exp2*gnq)/exp_sum;
                        dl(q) += lpsi2*(((Snq/lq+muZhat2_denom)/denom+dZm1m2*dZm1m2/(4.*lq*lq))*d_exp1+Z2/(2.*lq*lq)*d_exp2)/exp_sum;
                        dZ(m1,q) += 2.*lpsi2*((muZhat/denom-dZm1m2/(2*lq))*d_exp1-Zm1q/lq*d_exp2)/exp_sum;                   
                    }
                }
            }
        }
        """
        weave.inline(code, support_code=support_code, arg_names=['dL_dpsi1','dL_dpsi2','psi1','psi2n','N','M','Q','variance','l2','Z','mu','S','gamma','gamma1','log_denom1','log_denom2','log_gamma','log_gamma1','dvar','dl','dmu','dS','dgamma','dZ'], type_converters=weave.converters.blitz)
     
        dl *= 2.*lengthscale
        if not ARD:
            dl = dl.sum()
         
        return dvar, dl, dZ, dmu, dS, dgamma

except:

    def psicomputations(variance, lengthscale, Z, variational_posterior):
        """
        Z - MxQ
        mu - NxQ
        S - NxQ
        gamma - NxQ
        """
        # _psi1                NxM
        gamma = variational_posterior.binary_prob
         
        psi0 = np.empty(mu.shape[0])
        psi1 = _psi1computations(variance, lengthscale, Z, mu, S, gamma)
        psi2 = _psi2computations(variance, lengthscale, Z, mu, S, gamma)
        return psi0, psi1, psi2
    
    def _psi1computations(variance, lengthscale, Z, mu, S, gamma):
        """
        Z - MxQ
        mu - NxQ
        S - NxQ
        gamma - NxQ
        """
        # here are the "statistics" for psi1
        # _psi1                NxM
    
        lengthscale2 = np.square(lengthscale)
    
        # psi1
        _psi1_denom = S[:, None, :] / lengthscale2 + 1.  # Nx1xQ
        _psi1_denom_sqrt = np.sqrt(_psi1_denom) #Nx1xQ
        _psi1_dist = Z[None, :, :] - mu[:, None, :]  # NxMxQ
        _psi1_dist_sq = np.square(_psi1_dist) / (lengthscale2 * _psi1_denom) # NxMxQ
        _psi1_common = gamma[:,None,:] / (lengthscale2*_psi1_denom*_psi1_denom_sqrt) #Nx1xQ
        _psi1_exponent1 = np.log(gamma[:,None,:]) - (_psi1_dist_sq + np.log(_psi1_denom))/2. # NxMxQ
        _psi1_exponent2 = np.log(1.-gamma[:,None,:]) - (np.square(Z[None,:,:])/lengthscale2)/2. # NxMxQ
        _psi1_exponent_max = np.maximum(_psi1_exponent1,_psi1_exponent2)
        _psi1_exponent = _psi1_exponent_max+np.log(np.exp(_psi1_exponent1-_psi1_exponent_max) + np.exp(_psi1_exponent2-_psi1_exponent_max)) #NxMxQ
        _psi1_exp_sum = _psi1_exponent.sum(axis=-1) #NxM
        _psi1 = variance * np.exp(_psi1_exp_sum) # NxM
    
        return _psi1
    
    def _psi2computations(variance, lengthscale, Z, mu, S, gamma):
        """
        Z - MxQ
        mu - NxQ
        S - NxQ
        gamma - NxQ
        """
        # here are the "statistics" for psi2
        # _psi2                MxM
        
        _psi2_Zhat = 0.5 * (Z[:, None, :] + Z[None, :, :]) # M,M,Q
        _psi2_Zdist = 0.5 * (Z[:, None, :] - Z[None, :, :]) # M,M,Q
        _psi2_Zdist_sq = np.square(_psi2_Zdist / lengthscale) # M,M,Q
        _psi2_Z_sq_sum = (np.square(Z[:,None,:])+np.square(Z[None,:,:]))/lengthscale2 # MxMxQ
    
        # psi2
        _psi2_denom = 2.*S[:, None, None, :] / lengthscale2 + 1. # Nx1x1xQ
        _psi2_denom_sqrt = np.sqrt(_psi2_denom)
        _psi2_mudist = mu[:,None,None,:]-_psi2_Zhat #N,M,M,Q
        _psi2_mudist_sq = np.square(_psi2_mudist)/(lengthscale2*_psi2_denom)
        _psi2_common = gamma[:,None,None,:]/(lengthscale2 * _psi2_denom * _psi2_denom_sqrt) # Nx1x1xQ
        _psi2_exponent1 = -_psi2_Zdist_sq -_psi2_mudist_sq -0.5*np.log(_psi2_denom)+np.log(gamma[:,None,None,:]) #N,M,M,Q
        _psi2_exponent2 = np.log(1.-gamma[:,None,None,:]) - 0.5*(_psi2_Z_sq_sum) # NxMxMxQ
        _psi2_exponent_max = np.maximum(_psi2_exponent1, _psi2_exponent2)
        _psi2_exponent = _psi2_exponent_max+np.log(np.exp(_psi2_exponent1-_psi2_exponent_max) + np.exp(_psi2_exponent2-_psi2_exponent_max))
        _psi2_exp_sum = _psi2_exponent.sum(axis=-1) #NxM
        _psi2 = variance*variance * (np.exp(_psi2_exp_sum).sum(axis=0)) # MxM
    
        return _psi2
    
        dvar_psi1, dl_psi1, dZ_psi1, dmu_psi1, dS_psi1, dgamma_psi1 = _psi1compDer(dL_dpsi1, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
        dvar_psi2, dl_psi2, dZ_psi2, dmu_psi2, dS_psi2, dgamma_psi2 = _psi2compDer(dL_dpsi2, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
     
        dL_dvar = np.sum(dL_dpsi0) + dvar_psi1 + dvar_psi2
         
        dL_dlengscale = dl_psi1 + dl_psi2
        if not ARD:
            dL_dlengscale = dL_dlengscale.sum()
     
        dL_dgamma = dgamma_psi1 + dgamma_psi2
        dL_dmu = dmu_psi1 + dmu_psi2
        dL_dS = dS_psi1 + dS_psi2
        dL_dZ = dZ_psi1 + dZ_psi2
         
        return dL_dvar, dL_dlengscale, dL_dZ, dL_dmu, dL_dS, dL_dgamma
    
    def _psi1compDer(dL_dpsi1, variance, lengthscale, Z, mu, S, gamma):
        """
        dL_dpsi1 - NxM
        Z - MxQ
        mu - NxQ
        S - NxQ
        gamma - NxQ
        """
        # Produced intermediate results: dL_dparams w.r.t. psi1
        # _dL_dvariance     1
        # _dL_dlengthscale  Q
        # _dL_dZ            MxQ
        # _dL_dgamma        NxQ
        # _dL_dmu           NxQ
        # _dL_dS            NxQ
        
        # psi1
        _psi1_denom = S / lengthscale2 + 1.  # NxQ
        _psi1_denom_sqrt = np.sqrt(_psi1_denom) #NxQ
        _psi1_dist_sq = np.square(_psi1_dist) / (lengthscale2 * _psi1_denom[:,None,:]) # NxMxQ
        _psi1_common = gamma / (lengthscale2*_psi1_denom*_psi1_denom_sqrt) #NxQ
        _psi1_exponent1 = np.log(gamma[:,None,:]) -0.5 * (_psi1_dist_sq + np.log(_psi1_denom[:, None,:])) # NxMxQ
        _psi1_exponent2 = np.log(1.-gamma[:,None,:]) -0.5 * (np.square(Z[None,:,:])/lengthscale2) # NxMxQ
        _psi1_exp_dist_sq = np.exp(-0.5*_psi1_dist_sq) # NxMxQ
        _psi1_exp_Z = np.exp(-0.5*np.square(Z[None,:,:])/lengthscale2) # 1xMxQ
        _psi1_q = variance * np.exp(_psi1_exp_sum[:,:,None] - _psi1_exponent) # NxMxQ
        _dL_dvariance = np.einsum('nm,nm->',dL_dpsi1, _psi1)/variance # 1
        _dL_dgamma = np.einsum('nm,nmq,nmq->nq',dL_dpsi1, _psi1_q, (_psi1_exp_dist_sq/_psi1_denom_sqrt[:,None,:]-_psi1_exp_Z)) # NxQ
        _dL_dmu = np.einsum('nm, nmq, nmq, nmq, nq->nq',dL_dpsi1,_psi1_q,_psi1_exp_dist_sq,_psi1_dist,_psi1_common)  # NxQ
        _dL_dS = np.einsum('nm,nmq,nmq,nq,nmq->nq',dL_dpsi1,_psi1_q,_psi1_exp_dist_sq,_psi1_common,(_psi1_dist_sq-1.))/2.  # NxQ
        _dL_dZ = np.einsum('nm,nmq,nmq->mq',dL_dpsi1,_psi1_q, (- _psi1_common[:,None,:] * _psi1_dist * _psi1_exp_dist_sq - (1-gamma[:,None,:])/lengthscale2*Z[None,:,:]*_psi1_exp_Z))
        _dL_dlengthscale = lengthscale* np.einsum('nm,nmq,nmq->q',dL_dpsi1,_psi1_q,(_psi1_common[:,None,:]*(S[:,None,:]/lengthscale2+_psi1_dist_sq)*_psi1_exp_dist_sq + (1-gamma[:,None,:])*np.square(Z[None,:,:]/lengthscale2)*_psi1_exp_Z))
    
        return _dL_dvariance, _dL_dlengthscale, _dL_dZ, _dL_dmu, _dL_dS, _dL_dgamma 
    
    def _psi2compDer(dL_dpsi2, variance, lengthscale, Z, mu, S, gamma):
        """
        Z - MxQ
        mu - NxQ
        S - NxQ
        gamma - NxQ
        dL_dpsi2 - MxM
        """
        # Produced the derivatives w.r.t. psi2:
        # _dL_dvariance      1
        # _dL_dlengthscale   Q
        # _dL_dZ             MxQ
        # _dL_dgamma         NxQ
        # _dL_dmu            NxQ
        # _dL_dS             NxQ
        
        # psi2
        _psi2_denom = 2.*S / lengthscale2 + 1. # NxQ
        _psi2_mudist_sq = np.square(_psi2_mudist)/(lengthscale2*_psi2_denom[:,None,None,:])
        _psi2_common = gamma/(lengthscale2 * _psi2_denom * _psi2_denom_sqrt) # NxQ
        _psi2_exponent1 = -_psi2_Zdist_sq -_psi2_mudist_sq -0.5*np.log(_psi2_denom[:,None,None,:])+np.log(gamma[:,None,None,:]) #N,M,M,Q
        _psi2_q = variance*variance * np.exp(_psi2_exp_sum[:,:,:,None]-_psi2_exponent) # NxMxMxQ 
        _psi2_exp_dist_sq = np.exp(-_psi2_Zdist_sq -_psi2_mudist_sq) # NxMxMxQ
        _psi2_exp_Z = np.exp(-0.5*_psi2_Z_sq_sum) # MxMxQ
        _dL_dvariance = np.einsum('mo,mo->',dL_dpsi2,_psi2)*2./variance
        _dL_dgamma = np.einsum('mo,nmoq,nmoq->nq',dL_dpsi2,_psi2_q,(_psi2_exp_dist_sq/_psi2_denom_sqrt[:,None,None,:] - _psi2_exp_Z))
        _dL_dmu = -2.*np.einsum('mo,nmoq,nq,nmoq,nmoq->nq',dL_dpsi2,_psi2_q,_psi2_common,_psi2_mudist,_psi2_exp_dist_sq)
        _dL_dS = np.einsum('mo,nmoq,nq,nmoq,nmoq->nq',dL_dpsi2,_psi2_q, _psi2_common, (2.*_psi2_mudist_sq-1.), _psi2_exp_dist_sq)
        _dL_dZ = 2.*np.einsum('mo,nmoq,nmoq->mq',dL_dpsi2,_psi2_q,(_psi2_common[:,None,None,:]*(-_psi2_Zdist*_psi2_denom[:,None,None,:]+_psi2_mudist)*_psi2_exp_dist_sq - (1-gamma[:,None,None,:])*Z[:,None,:]/lengthscale2*_psi2_exp_Z))
        _dL_dlengthscale = 2.*lengthscale* np.einsum('mo,nmoq,nmoq->q',dL_dpsi2,_psi2_q,(_psi2_common[:,None,None,:]*(S[:,None,None,:]/lengthscale2+_psi2_Zdist_sq*_psi2_denom[:,None,None,:]+_psi2_mudist_sq)*_psi2_exp_dist_sq+(1-gamma[:,None,None,:])*_psi2_Z_sq_sum*0.5/lengthscale2*_psi2_exp_Z))
    
        return _dL_dvariance, _dL_dlengthscale, _dL_dZ, _dL_dmu, _dL_dS, _dL_dgamma

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# netpbmfile.py

# Copyright (c) 2011-2013, Christoph Gohlke
# Copyright (c) 2011-2013, The Regents of the University of California
# Produced at the Laboratory for Fluorescence Dynamics.
#
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
# * Neither the name of the copyright holders nor the names of any
#   contributors may be used to endorse or promote products derived
#   from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

"""Read and write image data from respectively to Netpbm files.

This implementation follows the Netpbm format specifications at
http://netpbm.sourceforge.net/doc/. No gamma correction is performed.

The following image formats are supported: PBM (bi-level), PGM (grayscale),
PPM (color), PAM (arbitrary), XV thumbnail (RGB332, read-only).

:Author:
  `Christoph Gohlke <http://www.lfd.uci.edu/~gohlke/>`_

:Organization:
  Laboratory for Fluorescence Dynamics, University of California, Irvine

:Version: 2013.01.18

Requirements
------------
* `CPython 2.7, 3.2 or 3.3 <http://www.python.org>`_
* `Numpy 1.7 <http://www.numpy.org>`_
* `Matplotlib 1.2 <http://www.matplotlib.org>`_  (optional for plotting)

Examples
--------
>>> im1 = numpy.array([[0, 1],[65534, 65535]], dtype=numpy.uint16)
>>> imsave('_tmp.pgm', im1)
>>> im2 = imread('_tmp.pgm')
>>> assert numpy.all(im1 == im2)

"""

from __future__ import division, print_function

import sys
import re
from copy import deepcopy

__version__ = '2013.01.18'
__docformat__ = 'restructuredtext en'
__all__ = ['imread', 'imsave', 'NetpbmFile']

def imread(filename, *args, **kwargs):
    """Return image data from Netpbm file as numpy array.

    `args` and `kwargs` are arguments to NetpbmFile.asarray().

    Examples
    --------
    >>> image = imread('_tmp.pgm')

    """
    try:
        netpbm = NetpbmFile(filename)
        image = netpbm.asarray()
    finally:
        netpbm.close()
    return image

def imsave(filename, data, maxval=None, pam=False):
    """Write image data to Netpbm file.

    Examples
    --------
    >>> image = numpy.array([[0, 1],[65534, 65535]], dtype=numpy.uint16)
    >>> imsave('_tmp.pgm', image)

    """
    try:
        netpbm = NetpbmFile(data, maxval=maxval)
        netpbm.write(filename, pam=pam)
    finally:

class NetpbmFile(object):
    """Read and write Netpbm PAM, PBM, PGM, PPM, files."""

    _types = {b'P1': b'BLACKANDWHITE', b'P2': b'GRAYSCALE', b'P3': b'RGB',
              b'P4': b'BLACKANDWHITE', b'P5': b'GRAYSCALE', b'P6': b'RGB',
              b'P7 332': b'RGB', b'P7': b'RGB_ALPHA'}

    def __init__(self, arg=None, **kwargs):
        """Initialize instance from filename, open file, or numpy array."""
        for attr in ('header', 'magicnum', 'width', 'height', 'maxval',
                     'depth', 'tupltypes', '_filename', '_fh', '_data'):
            setattr(self, attr, None)
        if arg is None:
            self._fromdata([], **kwargs)
        elif isinstance(arg, basestring):
            self._fh = open(arg, 'rb')
            self._filename = arg
            self._fromfile(self._fh, **kwargs)
        elif hasattr(arg, 'seek'):
            self._fromfile(arg, **kwargs)
            self._fh = arg
        else:
            self._fromdata(arg, **kwargs)

    def asarray(self, copy=True, cache=False, **kwargs):
        """Return image data from file as numpy array."""
        data = self._data
        if data is None:
            data = self._read_data(self._fh, **kwargs)
            if cache:
                self._data = data
            else:
                return data
        return deepcopy(data) if copy else data

    def write(self, arg, **kwargs):
        """Write instance to file."""
        if hasattr(arg, 'seek'):
            self._tofile(arg, **kwargs)
        else:
            with open(arg, 'wb') as fid:
                self._tofile(fid, **kwargs)

    def close(self):
        """Close open file. Future asarray calls might fail."""
        if self._filename and self._fh:
            self._fh.close()
            self._fh = None

    def __del__(self):
        self.close()

    def _fromfile(self, fh):
        """Initialize instance from open file."""
        fh.seek(0)
        data = fh.read(4096)
        if (len(data) < 7) or not (b'0' < data[1:2] < b'8'):
            raise ValueError("Not a Netpbm file:\n%s" % data[:32])
        try:
            self._read_pam_header(data)
            try:
                self._read_pnm_header(data)
            except Exception:
                raise ValueError("Not a Netpbm file:\n%s" % data[:32])

    def _read_pam_header(self, data):
        """Read PAM header and initialize instance."""
        regroups = re.search(
            b"(^P7[\n\r]+(?:(?:[\n\r]+)|(?:#.*)|"
            b"(HEIGHT\s+\d+)|(WIDTH\s+\d+)|(DEPTH\s+\d+)|(MAXVAL\s+\d+)|"
            b"(?:TUPLTYPE\s+\w+))*ENDHDR\n)", data).groups()
        self.header = regroups[0]
        self.magicnum = b'P7'
        for group in regroups[1:]:
            key, value = group.split()
            setattr(self, unicode(key).lower(), int(value))
        matches = re.findall(b"(TUPLTYPE\s+\w+)", self.header)
        self.tupltypes = [s.split(None, 1)[1] for s in matches]

    def _read_pnm_header(self, data):
        """Read PNM header and initialize instance."""
        bpm = data[1:2] in b"14"
        regroups = re.search(b"".join((
            b"(^(P[123456]|P7 332)\s+(?:#.*[\r\n])*",
            b"\s*(\d+)\s+(?:#.*[\r\n])*",
            b"\s*(\d+)\s+(?:#.*[\r\n])*" * (not bpm),
            b"\s*(\d+)\s(?:\s*#.*[\r\n]\s)*)")), data).groups() + (1, ) * bpm
        self.magicnum = regroups[1]
        self.width = int(regroups[2])
        self.height = int(regroups[3])
        self.maxval = int(regroups[4])
        self.depth = 3 if self.magicnum in b"P3P6P7 332" else 1
        self.tupltypes = [self._types[self.magicnum]]

    def _read_data(self, fh, byteorder='>'):
        """Return image data from open file as numpy array."""
        fh.seek(len(self.header))
        data = fh.read()
        dtype = 'u1' if self.maxval < 256 else byteorder + 'u2'
        depth = 1 if self.magicnum == b"P7 332" else self.depth
        shape = [-1, self.height, self.width, depth]
        size = numpy.prod(shape[1:])
        if self.magicnum in b"P1P2P3":
            data = numpy.array(data.split(None, size)[:size], dtype)
            data = data.reshape(shape)
        elif self.maxval == 1:
            shape[2] = int(math.ceil(self.width / 8))
            data = numpy.frombuffer(data, dtype).reshape(shape)
            data = numpy.unpackbits(data, axis=-2)[:, :, :self.width, :]
        else:
            data = numpy.frombuffer(data, dtype)
            data = data[:size * (data.size // size)].reshape(shape)
        if data.shape[0] < 2:
            data = data.reshape(data.shape[1:])
        if data.shape[-1] < 2:
            data = data.reshape(data.shape[:-1])
        if self.magicnum == b"P7 332":
            rgb332 = numpy.array(list(numpy.ndindex(8, 8, 4)), numpy.uint8)
            rgb332 *= [36, 36, 85]
            data = numpy.take(rgb332, data, axis=0)
        return data

    def _fromdata(self, data, maxval=None):
        """Initialize instance from numpy array."""
        data = numpy.array(data, ndmin=2, copy=True)
        if data.dtype.kind not in "uib":
            raise ValueError("not an integer type: %s" % data.dtype)
        if data.dtype.kind == 'i' and numpy.min(data) < 0:
            raise ValueError("data out of range: %i" % numpy.min(data))
        if maxval is None:
            maxval = numpy.max(data)
            maxval = 255 if maxval < 256 else 65535
        if maxval < 0 or maxval > 65535:
            raise ValueError("data out of range: %i" % maxval)
        data = data.astype('u1' if maxval < 256 else '>u2')
        self._data = data
        if data.ndim > 2 and data.shape[-1] in (3, 4):
            self.depth = data.shape[-1]
            self.width = data.shape[-2]
            self.height = data.shape[-3]
            self.magicnum = b'P7' if self.depth == 4 else b'P6'
        else:
            self.depth = 1
            self.width = data.shape[-1]
            self.height = data.shape[-2]
            self.magicnum = b'P5' if maxval > 1 else b'P4'
        self.maxval = maxval
        self.header = self._header()

    def _tofile(self, fh, pam=False):
        """Write Netbm file."""
        fh.write(self._header(pam))
        data = self.asarray(copy=False)
        if self.maxval == 1:
            data = numpy.packbits(data, axis=-1)
        data.tofile(fh)

    def _header(self, pam=False):
        """Return file header as byte string."""
        if pam or self.magicnum == b'P7':
            header = "\n".join((
                "P7",
                "HEIGHT %i" % self.height,
                "WIDTH %i" % self.width,
                "DEPTH %i" % self.depth,
                "MAXVAL %i" % self.maxval,
                "\n".join("TUPLTYPE %s" % unicode(i) for i in self.tupltypes),
                "ENDHDR\n"))
            header = "P4 %i %i\n" % (self.width, self.height)
        elif self.depth == 1:
            header = "P5 %i %i %i\n" % (self.width, self.height, self.maxval)
        else:
            header = "P6 %i %i %i\n" % (self.width, self.height, self.maxval)
        if sys.version_info[0] > 2:
            header = bytes(header, 'ascii')
        return header

        """Return information about instance."""
        return unicode(self.header)

if sys.version_info[0] > 2:
    basestring = str
    unicode = lambda x: str(x, 'ascii')

    # Show images specified on command line or all images in current directory
    from glob import glob
    from matplotlib import pyplot
    files = sys.argv[1:] if len(sys.argv) > 1 else glob('*.p*m')
    for fname in files:
        try:
            pam = NetpbmFile(fname)
            img = pam.asarray(copy=False)
            if False:
                pam.write('_tmp.pgm.out', pam=True)
                img2 = imread('_tmp.pgm.out')
                assert numpy.all(img == img2)
                imsave('_tmp.pgm.out', img)
            pam.close()
        except ValueError as e:
            print(fname, e)
            continue
        _shape = img.shape
        if img.ndim > 3 or (img.ndim > 2 and img.shape[-1] not in (3, 4)):
            img = img[0]
        cmap = 'gray' if pam.maxval > 1 else 'binary'
        pyplot.imshow(img, cmap, interpolation='nearest')
        pyplot.title("%s %s %s %s" % (fname, unicode(pam.magicnum),
                                      _shape, img.dtype))
        pyplot.show()

from ...util.linalg import tdot
from .psi_comp import PSICOMP_Linear

class Linear(Kern):
    """
    Linear kernel

    .. math::

       k(x,y) = \sum_{i=1}^input_dim \sigma^2_i x_iy_i

    """

    def __init__(self, input_dim, variances=None, ARD=False, active_dims=None, name='linear'):
        super(Linear, self).__init__(input_dim, active_dims, name)
        if not ARD:
            else:
        else:
            else:

        self.link_parameter(self.variances)
        self.psicomp = PSICOMP_Linear()

            if X2 is None:
                return tdot(X*np.sqrt(self.variances))
            else:
                rv = np.sqrt(self.variances)
                return np.dot(X*rv, (X2*rv).T)
        else:
            return self._dot_product(X, X2) * self.variances

    @Cache_this(limit=1, ignore_args=(0,))
            return tdot(X)
        else:

        return np.sum(self.variances * np.square(X), -1)

                #self.variances.gradient = np.array([np.sum(dL_dK * tdot(X[:, i:i + 1])) for i in range(self.input_dim)])
                self.variances.gradient = np.einsum('ij,iq,jq->q', dL_dK, X, X)
            else:
                #product = X[:, None, :] * X2[None, :, :]
                #self.variances.gradient = (dL_dK[:, :, None] * product).sum(0).sum(0)
                self.variances.gradient = np.einsum('ij,iq,jq->q', dL_dK, X, X2)
        else:
            self.variances.gradient = np.sum(self._dot_product(X, X2) * dL_dK)

        tmp = dL_dKdiag[:, None] * X ** 2
            self.variances.gradient = tmp.sum(0)
        else:
            self.variances.gradient = np.atleast_1d(tmp.sum())

            return np.einsum('jq,q,ij->iq', X, 2*self.variances, dL_dK)
        else:
            #return (((X2[None,:, :] * self.variances)) * dL_dK[:, :, None]).sum(1)
            return np.einsum('jq,q,ij->iq', X2, self.variances, dL_dK)

        return 2.*self.variances*dL_dKdiag[:,None]*X

    def input_sensitivity(self, summarize=True):

        return self.psicomp.psicomputations(self.variances, Z, variational_posterior)[0]

        return self.psicomp.psicomputations(self.variances, Z, variational_posterior)[1]

        return self.psicomp.psicomputations(self.variances, Z, variational_posterior)[2]

        dL_dvar = self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variances, Z, variational_posterior)[0]
            self.variances.gradient = dL_dvar
        else:
            self.variances.gradient = dL_dvar.sum()

        return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variances, Z, variational_posterior)[1]

        return self.psicomp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, self.variances, Z, variational_posterior)[2:]

class LinearFull(Kern):
    def __init__(self, input_dim, rank, W=None, kappa=None, active_dims=None, name='linear_full'):
        super(LinearFull, self).__init__(input_dim, active_dims, name)
            W = np.ones((input_dim, rank))
            kappa = np.ones(input_dim)
        assert W.shape == (input_dim, rank)
        assert kappa.shape == (input_dim,)

        P = np.dot(self.W, self.W.T) + np.diag(self.kappa)
        return np.einsum('ij,jk,lk->il', X, P, X if X2 is None else X2)

        self.kappa.gradient = np.einsum('ij,ik,kj->j', X, dL_dK, X if X2 is None else X2)
        self.W.gradient = np.einsum('ij,kl,ik,lm->jm', X, X if X2 is None else X2, dL_dK, self.W)
        self.W.gradient += np.einsum('ij,kl,ik,jm->lm', X, X if X2 is None else X2, dL_dK, self.W)

        return np.einsum('ij,jk,ik->i', X, P, X)

        self.kappa.gradient = np.einsum('ij,i->j', np.square(X), dL_dKdiag)
        self.W.gradient = 2.*np.einsum('ij,ik,jl,i->kl', X, X, self.W, dL_dKdiag)

            return 2.*np.einsum('ij,jk,kl->il', dL_dK, X, P)
        else:
            return np.einsum('ij,jk,kl->il', dL_dK, X2, P)

        return 2.*np.einsum('jk,i,ij->ik', P, dL_dKdiag, X)

# Copyright (c) 2014, GPy authors (see AUTHORS.txt).

from ...util.linalg import jitchol, backsub_both_sides, tdot, dtrtrs, dtrtri,pdinv

try:
    from mpi4py import MPI
except:
    pass

class VarDTC_minibatch(LatentFunctionInference):
    """

    """
    def __init__(self, batchsize=None, limit=1, mpi_comm=None):

        self.batchsize = batchsize

        # Cache functions

        self.midRes = {}
        self.batch_pos = 0 # the starting position of the current mini-batch
        self.Y_speedup = False # Replace Y with the cholesky factor of YY.T, but the computation of posterior object will be skipped.

        return self.batchsize, self.limit, self.Y_speedup

        self.batchsize, self.limit, self.Y_speedup = state
        self.mpi_comm = None
        self.batch_pos = 0

        return np.sum(np.square(Y))

        """

        """
        if (N>=D):
        else:

    def gatherPsiStat(self, kern, X, Z, Y, beta, uncertain_inputs):

        assert beta.size == 1

        if self.Y_speedup and not het_noise:
            Y =  self.get_YYTfactor(Y)

        batchsize = num_data if self.batchsize is None else self.batchsize

        psi2_full = np.zeros((num_inducing,num_inducing)) # MxM
        psi1Y_full = np.zeros((output_dim,num_inducing)) # DxM
        psi0_full = 0.
        YRY_full = 0.

        for n_start in xrange(0,num_data,batchsize):
            n_end = min(batchsize+n_start, num_data)
            if batchsize==num_data:
                Y_slice = Y
                X_slice = X
            else:
                Y_slice = Y[n_start:n_end]
                X_slice = X[n_start:n_end]

                b = beta[n_start]
                YRY_full += np.inner(Y_slice, Y_slice)*b
            else:
                b = beta

            if uncertain_inputs:
                psi0 = kern.psi0(Z, X_slice)
                psi1 = kern.psi1(Z, X_slice)
                psi2_full += kern.psi2(Z, X_slice)*b
            else:
                psi0 = kern.Kdiag(X_slice)
                psi1 = kern.K(X_slice, Z)
                psi2_full += np.dot(psi1.T,psi1)*b

            psi0_full += psi0.sum()*b
            psi1Y_full += np.dot(Y_slice.T,psi1)*b # DxM

        if not het_noise:
            YRY_full = trYYT*beta

            psi0_all = np.array(psi0_full)
            psi1Y_all = psi1Y_full.copy()
            psi2_all = psi2_full.copy()
            YRY_all = np.array(YRY_full)
            self.mpi_comm.Allreduce([psi0_full, MPI.DOUBLE], [psi0_all, MPI.DOUBLE])
            self.mpi_comm.Allreduce([psi1Y_full, MPI.DOUBLE], [psi1Y_all, MPI.DOUBLE])
            self.mpi_comm.Allreduce([psi2_full, MPI.DOUBLE], [psi2_all, MPI.DOUBLE])
            self.mpi_comm.Allreduce([YRY_full, MPI.DOUBLE], [YRY_all, MPI.DOUBLE])
            return psi0_all, psi1Y_all, psi2_all, YRY_all

        return psi0_full, psi1Y_full, psi2_full, YRY_full

    def inference_likelihood(self, kern, X, Z, likelihood, Y):
        """
        The first phase of inference:
        Compute: log-likelihood, dL_dKmm

        Cached intermediate results: Kmm, KmmInv,
        """

        input_dim = Z.shape[0]
            num_data_all = np.array(num_data,dtype=np.int32)
            self.mpi_comm.Allreduce([np.int32(num_data), MPI.INT], [num_data_all, MPI.INT])
            num_data = num_data_all

        if isinstance(X, VariationalPosterior):
            uncertain_inputs = True
        else:
            uncertain_inputs = False

        beta = 1./np.fmax(likelihood.variance, 1e-6)
        if het_noise:
            self.batchsize = 1

        psi0_full, psi1Y_full, psi2_full, YRY_full = self.gatherPsiStat(kern, X, Z, Y, beta, uncertain_inputs)

        #======================================================================
        # Compute Common Components

        Lm = jitchol(Kmm, maxtries=100)

        LmInvPsi2LmInvT = backsub_both_sides(Lm,psi2_full,transpose='right')
        Lambda = np.eye(Kmm.shape[0])+LmInvPsi2LmInvT
        LL = jitchol(Lambda, maxtries=100)
        logdet_L = 2.*np.sum(np.log(np.diag(LL)))
        b = dtrtrs(LL,dtrtrs(Lm,psi1Y_full.T)[0])[0]
        bbt = np.square(b).sum()
        v = dtrtrs(Lm,dtrtrs(LL,b,trans=1)[0],trans=1)[0]

        tmp  = -backsub_both_sides(LL, tdot(b)+output_dim*np.eye(input_dim), transpose='left')
        dL_dpsi2R = backsub_both_sides(Lm, tmp+output_dim*np.eye(input_dim), transpose='left')/2.

        # Cache intermediate results
        self.midRes['dL_dpsi2R'] = dL_dpsi2R
        self.midRes['v'] = v

        # Compute log-likelihood
            logL_R = -np.log(beta).sum()
        else:
            logL_R = -num_data*np.log(beta)
        logL = -(output_dim*(num_data*log_2_pi+logL_R+psi0_full-np.trace(LmInvPsi2LmInvT))+YRY_full-bbt)/2.-output_dim*logdet_L/2.

        # Compute dL_dKmm

        dL_dKmm =  dL_dpsi2R - output_dim*backsub_both_sides(Lm, LmInvPsi2LmInvT, transpose='left')/2.

        # Compute the Posterior distribution of inducing points p(u|Y)

        if not self.Y_speedup or het_noise:
            wd_inv = backsub_both_sides(Lm, np.eye(input_dim)- backsub_both_sides(LL, np.identity(input_dim), transpose='left'), transpose='left')
            post = Posterior(woodbury_inv=wd_inv, woodbury_vector=v, K=Kmm, mean=None, cov=None, K_chol=Lm)
        else:
            post = None

        # Compute dL_dthetaL for uncertian input and non-heter noise

            dL_dthetaL = (YRY_full*beta + beta*output_dim*psi0_full - num_data*output_dim*beta)/2. - beta*(dL_dpsi2R*psi2_full).sum() - beta*(v.T*psi1Y_full).sum()
            self.midRes['dL_dthetaL'] = dL_dthetaL

        return logL, dL_dKmm, post

    def inference_minibatch(self, kern, X, Z, likelihood, Y):
        """
        The second phase of inference: Computing the derivatives over a minibatch of Y
        Compute: dL_dpsi0, dL_dpsi1, dL_dpsi2, dL_dthetaL
        return a flag showing whether it reached the end of Y (isEnd)
        """

        else:

        #self.YYTfactor = beta*self.get_YYTfactor(Y)
            YYT_factor = self.get_YYTfactor(Y)
        else:
            YYT_factor = Y

        n_start = self.batch_pos
        n_end = min(batchsize+n_start, num_data)
        if n_end==num_data:
            isEnd = True
            self.batch_pos = 0
        else:
            isEnd = False
            self.batch_pos = n_end

        if batchsize==num_data:
            Y_slice = YYT_factor
            X_slice =X
        else:
            Y_slice = YYT_factor[n_start:n_end]
            X_slice = X[n_start:n_end]

            psi0 = kern.Kdiag(X_slice)
            psi1 = kern.K(X_slice, Z)
            psi2 = None
            betapsi1 = np.einsum('n,nm->nm',beta,psi1)
        elif het_noise:
            psi0 = kern.psi0(Z, X_slice)
            psi1 = kern.psi1(Z, X_slice)
            psi2 = kern.psi2(Z, X_slice)

            beta = beta[n_start] # assuming batchsize==1

        betaY = beta*Y_slice

        # Load Intermediate Results

        dL_dpsi2R = self.midRes['dL_dpsi2R']
        v = self.midRes['v']

        # Compute dL_dpsi

        dL_dpsi0 = -output_dim * (beta * np.ones((n_end-n_start,)))/2.

        dL_dpsi1 = np.dot(betaY,v.T)

            dL_dpsi2 = beta* dL_dpsi2R
        else:
            dL_dpsi1 += np.dot(betapsi1,dL_dpsi2R)*2.

        # Compute dL_dthetaL

                psiR = np.einsum('mo,mo->',dL_dpsi2R,psi2)
            else:
                psiR = np.einsum('nm,no,mo->',psi1,psi1,dL_dpsi2R)

            dL_dthetaL = ((np.square(betaY)).sum(axis=-1) + np.square(beta)*(output_dim*psi0)-output_dim*beta)/2. - np.square(beta)*psiR- (betaY*np.dot(betapsi1,v)).sum(axis=-1)
        else:
            if isEnd:
                dL_dthetaL = self.midRes['dL_dthetaL']
            else:
                dL_dthetaL = 0.

            grad_dict = {'dL_dpsi0':dL_dpsi0,
        else:
            grad_dict = {'dL_dKdiag':dL_dpsi0,

        return isEnd, (n_start,n_end), grad_dict

def update_gradients(model, mpi_comm=None):
    if mpi_comm == None:
        Y = model.Y
        X = model.X
    else:
        Y = model.Y_local
        X = model.X[model.N_range[0]:model.N_range[1]]

    model._log_marginal_likelihood, dL_dKmm, model.posterior = model.inference_method.inference_likelihood(model.kern, X, model.Z, model.likelihood, Y)

    het_noise = model.likelihood.variance.size > 1

        dL_dthetaL = np.empty((model.Y.shape[0],))
    else:
        dL_dthetaL = np.float64(0.)

    kern_grad = model.kern.gradient.copy()
    kern_grad[:] = 0.
    model.Z.gradient = 0.

    isEnd = False
    while not isEnd:
        isEnd, n_range, grad_dict = model.inference_method.inference_minibatch(model.kern, X, model.Z, model.likelihood, Y)
        if isinstance(model.X, VariationalPosterior):
            if (n_range[1]-n_range[0])==X.shape[0]:
                X_slice = X
            elif mpi_comm ==None:
                X_slice = model.X[n_range[0]:n_range[1]]
            else:
                X_slice = model.X[model.N_range[0]+n_range[0]:model.N_range[0]+n_range[1]]

            #gradients w.r.t. kernel
            model.kern.update_gradients_expectations(variational_posterior=X_slice, Z=model.Z, dL_dpsi0=grad_dict['dL_dpsi0'], dL_dpsi1=grad_dict['dL_dpsi1'], dL_dpsi2=grad_dict['dL_dpsi2'])
            kern_grad += model.kern.gradient

            #gradients w.r.t. Z
            model.Z.gradient += model.kern.gradients_Z_expectations(
                               dL_dpsi0=grad_dict['dL_dpsi0'], dL_dpsi1=grad_dict['dL_dpsi1'], dL_dpsi2=grad_dict['dL_dpsi2'], Z=model.Z, variational_posterior=X_slice)

            #gradients w.r.t. posterior parameters of X
            X_grad = model.kern.gradients_qX_expectations(variational_posterior=X_slice, Z=model.Z, dL_dpsi0=grad_dict['dL_dpsi0'], dL_dpsi1=grad_dict['dL_dpsi1'], dL_dpsi2=grad_dict['dL_dpsi2'])
            model.set_X_gradients(X_slice, X_grad)

                dL_dthetaL[n_range[0]:n_range[1]] = grad_dict['dL_dthetaL']
            else:
                dL_dthetaL += grad_dict['dL_dthetaL']

    # Gather the gradients from multiple MPI nodes
    if mpi_comm != None:
            raise "het_noise not implemented!"
        kern_grad_all = kern_grad.copy()
        Z_grad_all = model.Z.gradient.copy()
        mpi_comm.Allreduce([kern_grad, MPI.DOUBLE], [kern_grad_all, MPI.DOUBLE])
        mpi_comm.Allreduce([model.Z.gradient, MPI.DOUBLE], [Z_grad_all, MPI.DOUBLE])
        kern_grad = kern_grad_all
        model.Z.gradient = Z_grad_all

    #gradients w.r.t. kernel
    model.kern.update_gradients_full(dL_dKmm, model.Z, None)
    model.kern.gradient += kern_grad

    #gradients w.r.t. Z
    model.Z.gradient += model.kern.gradients_X(dL_dKmm, model.Z)

    # Update Log-likelihood
    KL_div = model.variational_prior.KL_divergence(X)
    # update for the KL divergence
    model.variational_prior.update_gradients_KL(X)

        KL_div_all = np.array(KL_div)
        mpi_comm.Allreduce([np.float64(KL_div), MPI.DOUBLE], [KL_div_all, MPI.DOUBLE])
        KL_div = KL_div_all
        [mpi_comm.Allgatherv([pp.copy(), MPI.DOUBLE], [pa, (model.N_list*pa.shape[-1], None), MPI.DOUBLE]) for pp,pa in zip(model.get_X_gradients(X),model.get_X_gradients(model.X))]
#         from ...models import SSGPLVM
#         if isinstance(model, SSGPLVM):
#             grad_pi = np.array(model.variational_prior.pi.gradient)
#             mpi_comm.Allreduce([grad_pi.copy(), MPI.DOUBLE], [model.variational_prior.pi.gradient, MPI.DOUBLE])
    model._log_marginal_likelihood -= KL_div

    # dL_dthetaL
    model.likelihood.update_gradients(dL_dthetaL)

def update_gradients_sparsegp(model, mpi_comm=None):
        Y = model.Y
        X = model.X
    else:

    else:
    
        if (n_range[1]-n_range[0])==X.shape[0]:
            X_slice = X
        elif mpi_comm ==None:
            X_slice = model.X[n_range[0]:n_range[1]]
        else:
            X_slice = model.X[model.N_range[0]+n_range[0]:model.N_range[0]+n_range[1]]
                
        model.kern.update_gradients_diag(grad_dict['dL_dKdiag'], X_slice)
        kern_grad += model.kern.gradient
        model.kern.update_gradients_full(grad_dict['dL_dKnm'], X_slice, model.Z)
            
        model.Z.gradient += model.kern.gradients_X(grad_dict['dL_dKnm'].T, model.Z, X_slice)
                
            dL_dthetaL[n_range[0]:n_range[1]] = grad_dict['dL_dthetaL']
        else:
            dL_dthetaL += grad_dict['dL_dthetaL']
    
# Copyright (c) 2012, 2013 GPy authors (see AUTHORS.txt).

import GPy
import sys

verbose = 0

class Kern_check_model(GPy.core.Model):
    """
    This is a dummy model class used as a base class for checking that the
    gradients of a given kernel are implemented correctly. It enables
    checkgrad() to be called independently on a kernel.
    """
    def __init__(self, kernel=None, dL_dK=None, X=None, X2=None):
        GPy.core.Model.__init__(self, 'kernel_test_model')
        if kernel==None:
            kernel = GPy.kern.RBF(1)
        kernel.randomize(loc=1, scale=0.1)
            X = np.random.randn(20, kernel.input_dim)
        if dL_dK is None:
                dL_dK = np.ones((X.shape[0], X.shape[0]))
            else:
                dL_dK = np.ones((X.shape[0], X2.shape[0]))

        self.kernel = kernel
        self.X = X
        self.X2 = X2
        self.dL_dK = dL_dK

    def is_positive_semi_definite(self):
        v = np.linalg.eig(self.kernel.K(self.X))[0]
        if any(v.real<=-1e-10):
            print v.real.min()
        else:
            return True

        return np.sum(self.dL_dK*self.kernel.K(self.X, self.X2))

class Kern_check_dK_dtheta(Kern_check_model):
    """
    This class allows gradient checks for the gradient of a kernel with
    respect to parameters.
    """
        Kern_check_model.__init__(self,kernel=kernel,dL_dK=dL_dK, X=X, X2=X2)
        self.link_parameter(self.kernel)

        return self.kernel.update_gradients_full(self.dL_dK, self.X, self.X2)

class Kern_check_dKdiag_dtheta(Kern_check_model):
    """
    This class allows gradient checks of the gradient of the diagonal of a
    kernel with respect to the parameters.
    """
    def __init__(self, kernel=None, dL_dK=None, X=None):
        Kern_check_model.__init__(self,kernel=kernel,dL_dK=dL_dK, X=X, X2=None)

        return (np.diag(self.dL_dK)*self.kernel.Kdiag(self.X)).sum()

        self.kernel.update_gradients_diag(np.diag(self.dL_dK), self.X)

class Kern_check_dK_dX(Kern_check_model):
    """This class allows gradient checks for the gradient of a kernel with respect to X. """
        self.X = Param('X',X)
        self.link_parameter(self.X)

        self.X.gradient[:] =  self.kernel.gradients_X(self.dL_dK, self.X, self.X2)

class Kern_check_dKdiag_dX(Kern_check_dK_dX):
    """This class allows gradient checks for the gradient of a kernel diagonal with respect to X. """
        Kern_check_dK_dX.__init__(self,kernel=kernel,dL_dK=dL_dK, X=X, X2=None)

        self.X.gradient[:] =  self.kernel.gradients_X_diag(self.dL_dK.diagonal(), self.X)

def check_kernel_gradient_functions(kern, X=None, X2=None, output_ind=None, verbose=False, fixed_X_dims=None):
    """
    This function runs on kernels to check the correctness of their
    implementation. It checks that the covariance function is positive definite
    for a randomly generated data set.

    :param kern: the kernel to be tested.
    :type kern: GPy.kern.Kernpart
    :param X: X input values to test the covariance function.
    :param X2: X2 input values to test the covariance function.
    :type X2: ndarray

    """
    pass_checks = True
    if X is None:
        X = np.random.randn(10, kern.input_dim)
        if output_ind is not None:
            X[:, output_ind] = np.random.randint(kern.output_dim, X.shape[0])
    if X2 is None:
        X2 = np.random.randn(20, kern.input_dim)
            X2[:, output_ind] = np.random.randint(kern.output_dim, X2.shape[0])

    if verbose:
        print("Checking covariance function is positive definite.")
    result = Kern_check_model(kern, X=X).is_positive_semi_definite()
    if result and verbose:
        print("Check passed.")
    if not result:
        print("Positive definite check failed for " + kern.name + " covariance function.")
        pass_checks = False
        assert(result)

        print("Checking gradients of K(X, X) wrt theta.")
    result = Kern_check_dK_dtheta(kern, X=X, X2=None).checkgrad(verbose=verbose)
        print("Gradient of K(X, X) wrt theta failed for " + kern.name + " covariance function. Gradient values as follows:")
        Kern_check_dK_dtheta(kern, X=X, X2=None).checkgrad(verbose=True)

        print("Checking gradients of K(X, X2) wrt theta.")
    result = Kern_check_dK_dtheta(kern, X=X, X2=X2).checkgrad(verbose=verbose)
        Kern_check_dK_dtheta(kern, X=X, X2=X2).checkgrad(verbose=True)

        print("Checking gradients of Kdiag(X) wrt theta.")
    try:
        result = Kern_check_dKdiag_dtheta(kern, X=X).checkgrad(verbose=verbose)
    except NotImplementedError:
        result=True
        if verbose:
            print("update_gradients_diag not implemented for " + kern.name)
        print("Gradient of Kdiag(X) wrt theta failed for " + kern.name + " covariance function. Gradient values as follows:")
        Kern_check_dKdiag_dtheta(kern, X=X).checkgrad(verbose=True)

        print("Checking gradients of K(X, X) wrt X.")
    try:
        testmodel = Kern_check_dK_dX(kern, X=X, X2=None)
        if fixed_X_dims is not None:
            testmodel.X[:,fixed_X_dims].fix()
        result = testmodel.checkgrad(verbose=verbose)
            print("gradients_X not implemented for " + kern.name)
        print("Gradient of K(X, X) wrt X failed for " + kern.name + " covariance function. Gradient values as follows:")
        testmodel.checkgrad(verbose=True)

        print("Checking gradients of K(X, X2) wrt X.")
    try:
        testmodel = Kern_check_dK_dX(kern, X=X, X2=X2)
        print("Gradient of K(X, X2) wrt X failed for " + kern.name + " covariance function. Gradient values as follows:")

        print("Checking gradients of Kdiag(X) wrt X.")
    try:
        testmodel = Kern_check_dKdiag_dX(kern, X=X)
        print("Gradient of Kdiag(X) wrt X failed for " + kern.name + " covariance function. Gradient values as follows:")
        Kern_check_dKdiag_dX(kern, X=X).checkgrad(verbose=True)

    return pass_checks

class KernelGradientTestsContinuous(unittest.TestCase):
        self.N, self.D = 10, 5
        self.X = np.random.randn(self.N,self.D)
        self.X2 = np.random.randn(self.N+10,self.D)

        continuous_kerns = ['RBF', 'Linear']
        self.kernclasses = [getattr(GPy.kern, s) for s in continuous_kerns]

    def test_Matern32(self):
        k = GPy.kern.Matern32(self.D)
        k.randomize()
        self.assertTrue(check_kernel_gradient_functions(k, X=self.X, X2=self.X2, verbose=verbose))

    def test_Prod(self):
        k = GPy.kern.Matern32(2, active_dims=[2,3]) * GPy.kern.RBF(2, active_dims=[0,4]) + GPy.kern.Linear(self.D)

    def test_Prod2(self):
        k = (GPy.kern.RBF(2, active_dims=[0,4]) * GPy.kern.Linear(self.D))

    def test_Prod3(self):

    def test_Add(self):
        k = GPy.kern.Matern32(2, active_dims=[2,3]) + GPy.kern.RBF(2, active_dims=[0,4]) + GPy.kern.Linear(self.D)
        k += GPy.kern.Matern32(2, active_dims=[2,3]) + GPy.kern.RBF(2, active_dims=[0,4]) + GPy.kern.Linear(self.D)

    def test_Add_dims(self):
        k = GPy.kern.Matern32(2, active_dims=[2,self.D]) + GPy.kern.RBF(2, active_dims=[0,4]) + GPy.kern.Linear(self.D)
        self.assertRaises(IndexError, k.K, self.X)
        k = GPy.kern.Matern32(2, active_dims=[2,self.D-1]) + GPy.kern.RBF(2, active_dims=[0,4]) + GPy.kern.Linear(self.D)
        # assert it runs:
        try:
            k.K(self.X)
        except AssertionError:
            raise AssertionError, "k.K(X) should run on self.D-1 dimension"

    def test_Matern52(self):
        k = GPy.kern.Matern52(self.D)

    def test_RBF(self):
        k = GPy.kern.RBF(self.D)

    def test_Linear(self):
        k = GPy.kern.Linear(self.D)

    def test_LinearFull(self):
        k = GPy.kern.LinearFull(self.D, self.D-1)

class KernelTestsMiscellaneous(unittest.TestCase):
        N, D = 100, 10
        self.X = np.linspace(-np.pi, +np.pi, N)[:,None] * np.random.uniform(-10,10,D)
        self.rbf = GPy.kern.RBF(2, active_dims=np.arange(0,4,2))
        self.linear = GPy.kern.Linear(2, active_dims=(3,9))
        self.matern = GPy.kern.Matern32(3, active_dims=np.array([1,7,9]))
        self.sumkern = self.rbf + self.linear
        self.sumkern += self.matern
        self.sumkern.randomize()

    def test_which_parts(self):
        self.assertTrue(np.allclose(self.sumkern.K(self.X, which_parts=[self.linear, self.matern]), self.linear.K(self.X)+self.matern.K(self.X)))
        self.assertTrue(np.allclose(self.sumkern.K(self.X, which_parts=[self.linear, self.rbf]), self.linear.K(self.X)+self.rbf.K(self.X)))
        self.assertTrue(np.allclose(self.sumkern.K(self.X, which_parts=self.sumkern.parts[0]), self.rbf.K(self.X)))

class KernelTestsNonContinuous(unittest.TestCase):
        N0 = 3
        N1 = 9
        N2 = 4
        N = N0+N1+N2
        self.D = 3
        self.X = np.random.randn(N, self.D+1)
        indices = np.random.random_integers(0, 2, size=N)
        self.X[indices==0, -1] = 0
        self.X[indices==1, -1] = 1
        self.X[indices==2, -1] = 2
        #self.X = self.X[self.X[:, -1].argsort(), :]
        self.X2 = np.random.randn((N0+N1)*2, self.D+1)
        self.X2[:(N0*2), -1] = 0
        self.X2[(N0*2):, -1] = 1

    def test_IndependentOutputs(self):
        k = GPy.kern.RBF(self.D, active_dims=range(self.D))
        kern = GPy.kern.IndependentOutputs(k, -1, 'ind_single')
        self.assertTrue(check_kernel_gradient_functions(kern, X=self.X, X2=self.X2, verbose=verbose, fixed_X_dims=-1))
        k = [GPy.kern.RBF(1, active_dims=[1], name='rbf1'), GPy.kern.RBF(self.D, active_dims=range(self.D), name='rbf012'), GPy.kern.RBF(2, active_dims=[0,2], name='rbf02')]
        kern = GPy.kern.IndependentOutputs(k, -1, name='ind_split')

    def test_Hierarchical(self):
        k = [GPy.kern.RBF(2, active_dims=[0,2], name='rbf1'), GPy.kern.RBF(2, active_dims=[0,2], name='rbf2')]

    def test_ODE_UY(self):
        kern = GPy.kern.ODE_UY(2, active_dims=[0, self.D])
        X = self.X[self.X[:,-1]!=2]
        X2 = self.X2[self.X2[:,-1]!=2]
        self.assertTrue(check_kernel_gradient_functions(kern, X=X, X2=X2, verbose=verbose, fixed_X_dims=-1))

class Coregionalize_weave_test(unittest.TestCase):
    """
    Make sure that the coregionalize kernel work with and without weave enabled
    """
        self.k = GPy.kern.Coregionalize(1, output_dim=12)
        self.N1, self.N2 = 100, 200
        self.X = np.random.randint(0,12,(self.N1,1))
        self.X2 = np.random.randint(0,12,(self.N2,1))

    def test_sym(self):
        dL_dK = np.random.randn(self.N1, self.N1)
        GPy.util.config.config.set('weave', 'working', 'True')
        K_weave = self.k.K(self.X)
        self.k.update_gradients_full(dL_dK, self.X)
        grads_weave = self.k.gradient.copy()

        GPy.util.config.config.set('weave', 'working', 'False')
        K_numpy = self.k.K(self.X)
        grads_numpy = self.k.gradient.copy()

        self.assertTrue(np.allclose(K_numpy, K_weave))
        self.assertTrue(np.allclose(grads_numpy, grads_weave))

    def test_nonsym(self):
        dL_dK = np.random.randn(self.N1, self.N2)
        K_weave = self.k.K(self.X, self.X2)
        self.k.update_gradients_full(dL_dK, self.X, self.X2)

        K_numpy = self.k.K(self.X, self.X2)

    #reset the weave state for any other tests
    GPy.util.config.config.set('weave', 'working', 'False')

    print "Running unit tests, please be (very) patient..."
#     np.random.seed(0)
#     N0 = 3
#     N1 = 9
#     N2 = 4
#     N = N0+N1+N2
#     D = 3
#     X = np.random.randn(N, D+1)
#     indices = np.random.random_integers(0, 2, size=N)
#     X[indices==0, -1] = 0
#     X[indices==1, -1] = 1
#     X[indices==2, -1] = 2
#     #X = X[X[:, -1].argsort(), :]
#     X2 = np.random.randn((N0+N1)*2, D+1)
#     X2[:(N0*2), -1] = 0
#     X2[(N0*2):, -1] = 1
#     k = [GPy.kern.RBF(1, active_dims=[1], name='rbf1'), GPy.kern.RBF(D, name='rbf012'), GPy.kern.RBF(2, active_dims=[0,2], name='rbf02')]
#     kern = GPy.kern.IndependentOutputs(k, -1, name='ind_split')
#     assert(check_kernel_gradient_functions(kern, X=X, X2=X2, verbose=verbose, fixed_X_dims=-1))
#     k = GPy.kern.RBF(D)
#     kern = GPy.kern.IndependentOutputs(k, -1, 'ind_single')

from ..core.parameterization.observable import Observable
import collections, weakref

class Cacher(object):
    def __init__(self, operation, limit=5, ignore_args=(), force_kwargs=()):
        """
        Parameters:
        ***********
        :param callable operation: function to cache
        :param int limit: depth of cacher
        :param [int] ignore_args: list of indices, pointing at arguments to ignore in *args of operation(*args). This includes self!
        :param [str] force_kwargs: list of kwarg names (strings). If a kwarg with that name is given, the cacher will force recompute and wont cache anything.
        :param int verbose: verbosity level. 0: no print outs, 1: casual print outs, 2: debug level print outs
        """
        self.limit = int(limit)
        self.ignore_args = ignore_args
        self.force_kwargs = force_kwargs
        self.operation = operation
        self.order = collections.deque()
        self.cached_inputs = {}  # point from cache_ids to a list of [ind_ids], which where used in cache cache_id

        # point from each ind_id to [ref(obj), cache_ids]
        # 0: a weak reference to the object itself
        # 1: the cache_ids in which this ind_id is used (len will be how many times we have seen this ind_id)
        self.cached_input_ids = {}

        self.cached_outputs = {}  # point from cache_ids to outputs
        self.inputs_changed = {}  # point from cache_ids to bools

    def id(self, obj):
        """returns the self.id of an object, to be used in caching individual self.ids"""
        return hex(id(obj))

    def combine_inputs(self, args, kw, ignore_args):
        "Combines the args and kw in a unique way, such that ordering of kwargs does not lead to recompute"
        inputs= args + tuple(c[1] for c in sorted(kw.items(), key=lambda x: x[0]))
        # REMOVE the ignored arguments from input and PREVENT it from being checked!!!
        return [a for i,a in enumerate(inputs) if i not in ignore_args]

    def prepare_cache_id(self, combined_args_kw):
        "get the cacheid (conc. string of argument self.ids in order)"
        cache_id = "".join(self.id(a) for a in combined_args_kw)
        return cache_id

    def ensure_cache_length(self, cache_id):
        "Ensures the cache is within its limits and has one place free"
        if len(self.order) == self.limit:
            # we have reached the limit, so lets release one element
            cache_id = self.order.popleft()
            combined_args_kw = self.cached_inputs[cache_id]
            for ind in combined_args_kw:
                if ind is not None:
                    ind_id = self.id(ind)
                    tmp = self.cached_input_ids.get(ind_id, None)
                    if tmp is not None:
                        ref, cache_ids = tmp
                        if len(cache_ids) == 1 and ref() is not None:
                            ref().remove_observer(self, self.on_cache_changed)
                            del self.cached_input_ids[ind_id]
                        else:
                            cache_ids.remove(cache_id)
                            self.cached_input_ids[ind_id] = [ref, cache_ids]
            del self.cached_outputs[cache_id]
            del self.inputs_changed[cache_id]
            del self.cached_inputs[cache_id]

    def add_to_cache(self, cache_id, inputs, output):
        """This adds cache_id to the cache, with inputs and output"""
        self.inputs_changed[cache_id] = False
        self.cached_outputs[cache_id] = output
        self.order.append(cache_id)
        self.cached_inputs[cache_id] = inputs
        for a in inputs:
            if a is not None:
                ind_id = self.id(a)
                v = self.cached_input_ids.get(ind_id, [weakref.ref(a), []])
                v[1].append(cache_id)
                if len(v[1]) == 1:
                    a.add_observer(self, self.on_cache_changed)
                self.cached_input_ids[ind_id] = v

    def __call__(self, *args, **kw):
        """
        A wrapper function for self.operation,
        """
        # !WARNING CACHE OFFSWITCH!
        # return self.operation(*args, **kw)

        # 1: Check whether we have forced recompute arguments:
        if len(self.force_kwargs) != 0:
            for k in self.force_kwargs:
                if k in kw and kw[k] is not None:
                    return self.operation(*args, **kw)

        # 2: prepare_cache_id and get the unique self.id string for this call
        inputs = self.combine_inputs(args, kw, self.ignore_args)
        cache_id = self.prepare_cache_id(inputs)
        # 2: if anything is not cachable, we will just return the operation, without caching
        if reduce(lambda a, b: a or (not (isinstance(b, Observable) or b is None)), inputs, False):
            #print 'WARNING: '+self.operation.__name__ + ' not cacheable!'
            #print [not (isinstance(b, Observable)) for b in inputs]
            return self.operation(*args, **kw)
        # 3&4: check whether this cache_id has been cached, then has it changed?
        try:
            if(self.inputs_changed[cache_id]):
                # 4: This happens, when elements have changed for this cache self.id
                self.inputs_changed[cache_id] = False
                self.cached_outputs[cache_id] = self.operation(*args, **kw)
        except KeyError:
            # 3: This is when we never saw this chache_id:
            self.ensure_cache_length(cache_id)
            self.add_to_cache(cache_id, inputs, self.operation(*args, **kw))
        except:
            self.reset()
            raise
        # 5: We have seen this cache_id and it is cached:
        return self.cached_outputs[cache_id]

    def on_cache_changed(self, direct, which=None):
        """
        A callback funtion, which sets local flags when the elements of some cached inputs change

        this function gets 'hooked up' to the inputs when we cache them, and upon their elements being changed we update here.
        """
        for what in [direct, which]:
            if what is not None:
                ind_id = self.id(what)
                _, cache_ids = self.cached_input_ids.get(ind_id, [None, []])
                for cache_id in cache_ids:
                    self.inputs_changed[cache_id] = True

        """
        Totally reset the cache
        """
        [a().remove_observer(self, self.on_cache_changed) if (a() is not None) else None for [a, _] in self.cached_input_ids.values()]
        self.cached_outputs = {}
        self.inputs_changed = {}

    def __deepcopy__(self, memo=None):
        return Cacher(self.operation, self.limit, self.ignore_args, self.force_kwargs)

    def __getstate__(self, memo=None):
        raise NotImplementedError, "Trying to pickle Cacher object with function {}, pickling functions not possible.".format(str(self.operation))

    def __setstate__(self, memo=None):

    @property
    def __name__(self):
        return self.operation.__name__

from functools import partial, update_wrapper

class Cacher_wrap(object):
    def __init__(self, f, limit, ignore_args, force_kwargs):
        self.f = f
        update_wrapper(self, self.f)
    def __get__(self, obj, objtype=None):
        return partial(self, obj)
    def __call__(self, *args, **kwargs):
        obj = args[0]
        # import ipdb;ipdb.set_trace()
        try:
            caches = obj.__cachers
        except AttributeError:
            caches = obj.__cachers = {}
        try:
            cacher = caches[self.f]
            cacher = caches[self.f] = Cacher(self.f, self.limit, self.ignore_args, self.force_kwargs)
        return cacher(*args, **kwargs)

class Cache_this(object):
    """
    A decorator which can be applied to bound methods in order to cache them
    """
    def __init__(self, limit=5, ignore_args=(), force_kwargs=()):
        self.force_args = force_kwargs
    def __call__(self, f):
        newf = Cacher_wrap(f, self.limit, self.ignore_args, self.force_args)
        update_wrapper(newf, f)
        return newf

try:
except:
    pass
from base_plots import gpplot, x_frame1D, x_frame2D
from ...models.gp_coregionalized_regression import GPCoregionalizedRegression
from ...models.sparse_gp_coregionalized_regression import SparseGPCoregionalizedRegression
from scipy import sparse

def plot_fit(model, plot_limits=None, which_data_rows='all',
        which_data_ycols='all', fixed_inputs=[],
        levels=20, samples=0, fignum=None, ax=None, resolution=None,
        plot_raw=False,
        linecol=Tango.colorsHex['darkBlue'],fillcol=Tango.colorsHex['lightBlue'], Y_metadata=None, data_symbol='kx'):
    """
    Plot the posterior of the GP.
      - In one dimension, the function is plotted with a shaded region identifying two standard deviations.
      - In two dimsensions, a contour-plot shows the mean predicted function
      - In higher dimensions, use fixed_inputs to plot the GP  with some of the inputs fixed.

    using which_data_rowsm which_data_ycols.

    :param which_data_rows: which of the training data to plot (default all)
    :type which_data_rows: 'all' or a slice object to slice model.X, model.Y
    :param which_data_ycols: when the data has several columns (independant outputs), only plot these
    :type which_data_rows: 'all' or a list of integers
    :type output: integer (first output is 0)
    :param fillcol: color of fill
    """
    #deal with optional arguments
    if which_data_rows == 'all':
        which_data_rows = slice(None)
    if which_data_ycols == 'all':
        which_data_ycols = np.arange(model.output_dim)
    #if len(which_data_ycols)==0:
        #raise ValueError('No data selected for plotting')

    if hasattr(model, 'has_uncertain_inputs') and model.has_uncertain_inputs():
        X = model.X.mean
        X_variance = model.X.variance
    else:
        X = model.X
    Y = model.Y
    if sparse.issparse(Y): Y = Y.todense().view(np.ndarray)

    if hasattr(model, 'Z'): Z = model.Z

    #work out what the inputs are for plotting (1D or 2D)
    fixed_dims = np.array([i for i,v in fixed_inputs])
    free_dims = np.setdiff1d(np.arange(model.input_dim),fixed_dims)
    plots = {}
    #one dimensional plotting
    if len(free_dims) == 1:

        #define the frame on which to plot
        Xnew, xmin, xmax = x_frame1D(X[:,free_dims], plot_limits=plot_limits, resolution=resolution or 200)
        Xgrid = np.empty((Xnew.shape[0],model.input_dim))
        Xgrid[:,free_dims] = Xnew

        #make a prediction on the frame and plot it
        if plot_raw:
            m, v = model._raw_predict(Xgrid)
            lower = m - 2*np.sqrt(v)
            upper = m + 2*np.sqrt(v)
        else:
            if isinstance(model,GPCoregionalizedRegression) or isinstance(model,SparseGPCoregionalizedRegression):
                meta = {'output_index': Xgrid[:,-1:].astype(np.int)}
            else:
                meta = None
            m, v = model.predict(Xgrid, full_cov=False, Y_metadata=meta)
            lower, upper = model.predict_quantiles(Xgrid, Y_metadata=meta)

        for d in which_data_ycols:
            plots['gpplot'] = gpplot(Xnew, m[:, d], lower[:, d], upper[:, d], ax=ax, edgecol=linecol, fillcol=fillcol)
            if not plot_raw: plots['dataplot'] = ax.plot(X[which_data_rows,free_dims], Y[which_data_rows, d], data_symbol, mew=1.5)

        #optionally plot some samples
        if samples: #NOTE not tested with fixed_inputs
            Ysim = model.posterior_samples(Xgrid, samples)
            for yi in Ysim.T:
                plots['posterior_samples'] = ax.plot(Xnew, yi[:,None], Tango.colorsHex['darkBlue'], linewidth=0.25)
                #ax.plot(Xnew, yi[:,None], marker='x', linestyle='--',color=Tango.colorsHex['darkBlue']) #TODO apply this line for discrete outputs.

        #add error bars for uncertain (if input uncertainty is being modelled)
        if hasattr(model,"has_uncertain_inputs") and model.has_uncertain_inputs():
            plots['xerrorbar'] = ax.errorbar(X[which_data_rows, free_dims].flatten(), Y[which_data_rows, which_data_ycols].flatten(),
                        xerr=2 * np.sqrt(X_variance[which_data_rows, free_dims].flatten()),
                        ecolor='k', fmt=None, elinewidth=.5, alpha=.5)

        #set the limits of the plot to some sensible values
        ymin, ymax = min(np.append(Y[which_data_rows, which_data_ycols].flatten(), lower)), max(np.append(Y[which_data_rows, which_data_ycols].flatten(), upper))
        ymin, ymax = ymin - 0.1 * (ymax - ymin), ymax + 0.1 * (ymax - ymin)
        ax.set_xlim(xmin, xmax)
        ax.set_ylim(ymin, ymax)

        #add inducing inputs (if a sparse model is used)
        if hasattr(model,"Z"):
            #Zu = model.Z[:,free_dims] * model._Xscale[:,free_dims] + model._Xoffset[:,free_dims]
            if isinstance(model,SparseGPCoregionalizedRegression):
                Z = Z[Z[:,-1] == Y_metadata['output_index'],:]
            Zu = Z[:,free_dims]
            z_height = ax.get_ylim()[0]
            plots['inducing_inputs'] = ax.plot(Zu, np.zeros_like(Zu) + z_height, 'r|', mew=1.5, markersize=12)

    #2D plotting
    elif len(free_dims) == 2:

        #define the frame for plotting on
        Xnew, _, _, xmin, xmax = x_frame2D(X[:,free_dims], plot_limits, resolution)

        #predict on the frame and plot
            m, _ = model._raw_predict(Xgrid)
        else:
            else:
                meta = None
            m_d = m[:,d].reshape(resolution, resolution).T
            plots['contour'] = ax.contour(x, y, m_d, levels, vmin=m.min(), vmax=m.max(), cmap=pb.cm.jet)
            if not plot_raw: plots['dataplot'] = ax.scatter(X[which_data_rows, free_dims[0]], X[which_data_rows, free_dims[1]], 40, Y[which_data_rows, d], cmap=pb.cm.jet, vmin=m.min(), vmax=m.max(), linewidth=0.)

        if samples:
            warnings.warn("Samples are rather difficult to plot for 2D inputs...")

            plots['inducing_inputs'] = ax.plot(Zu[:,free_dims[0]], Zu[:,free_dims[1]], 'wo')

    else:
    return plots

def plot_fit_f(model, *args, **kwargs):
    """
    Plot the GP's view of the world, where the data is normalized and before applying a likelihood.

    All args and kwargs are passed on to models_plots.plot.
    """
    kwargs['plot_raw'] = True
    plot_fit(model,*args, **kwargs)

"""Test case implementation"""

import collections
import sys
import functools
import difflib
import pprint
import re
import types

from . import result
from .util import (
    strclass, safe_repr, unorderable_list_difference,
    _count_diff_all_purpose, _count_diff_hashable
)

__unittest = True

DIFF_OMITTED = ('\nDiff is %s characters long. '
                 'Set self.maxDiff to None to see it.')

class SkipTest(Exception):
    """
    Raise this exception in a test to skip it.

    Usually you can use TestCase.skipTest() or one of the skipping decorators
    instead of raising this directly.
    """
    pass

class _ExpectedFailure(Exception):
    """
    Raise this when a test is expected to fail.

    This is an implementation detail.
    """

    def __init__(self, exc_info):
        super(_ExpectedFailure, self).__init__()
        self.exc_info = exc_info

class _UnexpectedSuccess(Exception):
    """
    The test was supposed to fail, but it didn't!
    """
    pass

def _id(obj):
    return obj

def skip(reason):
    """
    Unconditionally skip a test.
    """
    def decorator(test_item):
        if not isinstance(test_item, (type, types.ClassType)):
            @functools.wraps(test_item)
            def skip_wrapper(*args, **kwargs):
                raise SkipTest(reason)
            test_item = skip_wrapper

        test_item.__unittest_skip__ = True
        test_item.__unittest_skip_why__ = reason
        return test_item
    return decorator

def skipIf(condition, reason):
    """
    Skip a test if the condition is true.
    """
    if condition:
        return skip(reason)
    return _id

def skipUnless(condition, reason):
    """
    Skip a test unless the condition is true.
    """
    if not condition:
    return _id

def expectedFailure(func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        try:
            func(*args, **kwargs)
            raise _ExpectedFailure(sys.exc_info())
        raise _UnexpectedSuccess

class _AssertRaisesContext(object):
    """A context manager used to implement TestCase.assertRaises* methods."""

    def __init__(self, expected, test_case, expected_regexp=None):
        self.expected = expected
        self.failureException = test_case.failureException
        self.expected_regexp = expected_regexp

    def __exit__(self, exc_type, exc_value, tb):
        if exc_type is None:
            try:
                exc_name = self.expected.__name__
            except AttributeError:
                exc_name = str(self.expected)
            raise self.failureException(
                "{0} not raised".format(exc_name))
        if not issubclass(exc_type, self.expected):
            # let unexpected exceptions pass through
        self.exception = exc_value # store for later retrieval
        if self.expected_regexp is None:

        expected_regexp = self.expected_regexp
        if not expected_regexp.search(str(exc_value)):
            raise self.failureException('"%s" does not match "%s"' %
                     (expected_regexp.pattern, str(exc_value)))

class TestCase(object):
    """A class whose instances are single test cases.

    By default, the test code itself should be placed in a method named
    'runTest'.

    If the fixture may be used for many test cases, create as
    many test methods as are needed. When instantiating such a TestCase
    subclass, specify in the constructor arguments the name of the test method
    that the instance is to execute.

    Test authors should subclass TestCase for their own tests. Construction
    and deconstruction of the test's environment ('fixture') can be
    implemented by overriding the 'setUp' and 'tearDown' methods respectively.

    If it is necessary to override the __init__ method, the base class
    __init__ method must always be called. It is important that subclasses
    should not change the signature of their __init__ method, since instances
    of the classes are instantiated automatically by parts of the framework
    in order to be run.

    When subclassing TestCase, you can set these attributes:
    * failureException: determines which exception will be raised when
        the instance's assertion methods fail; test methods raising this
        exception will be deemed to have 'failed' rather than 'errored'.
    * longMessage: determines whether long messages (including repr of
        objects used in assert methods) will be printed on failure in *addition*
        to any explicit message passed.
    * maxDiff: sets the maximum length of a diff in failure messages
        by assert methods using difflib. It is looked up as an instance
        attribute so can be configured by individual tests if required.
    """

    failureException = AssertionError

    longMessage = False

    maxDiff = 80*8

    # If a string is longer than _diffThreshold, use normal comparison instead
    # of difflib.  See #11763.
    _diffThreshold = 2**16

    # Attribute used by TestSuite for classSetUp

    _classSetupFailed = False

    def __init__(self, methodName='runTest'):
        """Create an instance of the class that will use the named test
           method when executed. Raises a ValueError if the instance does
           not have a method with the specified name.
        """
        self._testMethodName = methodName
        self._resultForDoCleanups = None
        try:
            testMethod = getattr(self, methodName)
            raise ValueError("no such test method in %s: %s" %
                  (self.__class__, methodName))
        self._testMethodDoc = testMethod.__doc__
        self._cleanups = []

        # Map types to custom assertEqual functions that will compare
        # instances of said type in more detail to generate a more useful
        # error message.
        self._type_equality_funcs = {}
        self.addTypeEqualityFunc(dict, 'assertDictEqual')
        self.addTypeEqualityFunc(list, 'assertListEqual')
        self.addTypeEqualityFunc(tuple, 'assertTupleEqual')
        self.addTypeEqualityFunc(set, 'assertSetEqual')
        self.addTypeEqualityFunc(frozenset, 'assertSetEqual')
        try:
            self.addTypeEqualityFunc(unicode, 'assertMultiLineEqual')
        except NameError:
            # No unicode support in this build
            pass

    def addTypeEqualityFunc(self, typeobj, function):
        """Add a type specific assertEqual style function to compare a type.

        This method is for use by TestCase subclasses that need to register
        their own type equality functions to provide nicer error messages.

        Args:
            typeobj: The data type to call this function on when both values
                    are of the same type in assertEqual().
            function: The callable taking two arguments and an optional
                    msg= argument that raises self.failureException with a
                    useful error message when the two arguments are not equal.
        """
        self._type_equality_funcs[typeobj] = function

    def addCleanup(self, function, *args, **kwargs):
        """Add a function, with arguments, to be called when the test is
        completed. Functions added are called on a LIFO basis and are
        called after tearDown on test failure or success.

        Cleanup items are called even if setUp fails (unlike tearDown)."""
        self._cleanups.append((function, args, kwargs))

        "Hook method for setting up the test fixture before exercising it."
        pass

    def tearDown(self):
        "Hook method for deconstructing the test fixture after testing it."
        pass

    @classmethod
    def setUpClass(cls):
        "Hook method for setting up class fixture before running tests in the class."

    def tearDownClass(cls):
        "Hook method for deconstructing the class fixture after running all tests in the class."

    def countTestCases(self):
        return 1

    def defaultTestResult(self):
        return result.TestResult()

    def shortDescription(self):
        """Returns a one-line description of the test, or None if no
        description has been provided.

        The default implementation of this method returns the first line of
        the specified test method's docstring.
        """
        doc = self._testMethodDoc
        return doc and doc.split("\n")[0].strip() or None

    def id(self):
        return "%s.%s" % (strclass(self.__class__), self._testMethodName)

    def __eq__(self, other):
        if type(self) is not type(other):
            return NotImplemented

        return self._testMethodName == other._testMethodName

    def __ne__(self, other):
        return not self == other

    def __hash__(self):
        return hash((type(self), self._testMethodName))

        return "%s (%s)" % (self._testMethodName, strclass(self.__class__))

        return "<%s testMethod=%s>" % \
               (strclass(self.__class__), self._testMethodName)

    def _addSkip(self, result, reason):
        addSkip = getattr(result, 'addSkip', None)
        if addSkip is not None:
            addSkip(self, reason)
        else:
            warnings.warn("TestResult has no addSkip method, skips not reported",
                          RuntimeWarning, 2)
            result.addSuccess(self)

    def run(self, result=None):
        orig_result = result
        if result is None:
            result = self.defaultTestResult()
            startTestRun = getattr(result, 'startTestRun', None)
            if startTestRun is not None:
                startTestRun()

        self._resultForDoCleanups = result
        result.startTest(self)

        testMethod = getattr(self, self._testMethodName)
        if (getattr(self.__class__, "__unittest_skip__", False) or
            getattr(testMethod, "__unittest_skip__", False)):
            # If the class or method was skipped.
            try:
                skip_why = (getattr(self.__class__, '__unittest_skip_why__', '')
                            or getattr(testMethod, '__unittest_skip_why__', ''))
                self._addSkip(result, skip_why)
            finally:
                result.stopTest(self)
            return
        try:
            try:
                self.setUp()
            except SkipTest as e:
                self._addSkip(result, str(e))
                raise
            except:
                result.addError(self, sys.exc_info())
            else:
                try:
                    testMethod()
                except KeyboardInterrupt:
                    raise
                except self.failureException:
                    result.addFailure(self, sys.exc_info())
                except _ExpectedFailure as e:
                    addExpectedFailure = getattr(result, 'addExpectedFailure', None)
                    if addExpectedFailure is not None:
                        addExpectedFailure(self, e.exc_info)
                    else:
                        warnings.warn("TestResult has no addExpectedFailure method, reporting as passes",
                                      RuntimeWarning)
                        result.addSuccess(self)
                except _UnexpectedSuccess:
                    addUnexpectedSuccess = getattr(result, 'addUnexpectedSuccess', None)
                    if addUnexpectedSuccess is not None:
                        addUnexpectedSuccess(self)
                    else:
                        warnings.warn("TestResult has no addUnexpectedSuccess method, reporting as failures",
                        result.addFailure(self, sys.exc_info())
                except SkipTest as e:
                    self._addSkip(result, str(e))
                except:
                    result.addError(self, sys.exc_info())
                else:
                    success = True

                try:
                    self.tearDown()
                    raise
                except:
                    success = False

            cleanUpSuccess = self.doCleanups()
            success = success and cleanUpSuccess
            if success:
                result.addSuccess(self)
        finally:
            result.stopTest(self)
            if orig_result is None:
                stopTestRun = getattr(result, 'stopTestRun', None)
                if stopTestRun is not None:
                    stopTestRun()

    def doCleanups(self):
        """Execute all cleanup functions. Normally called for you after
        tearDown."""
        result = self._resultForDoCleanups
        ok = True
        while self._cleanups:
            function, args, kwargs = self._cleanups.pop(-1)
            try:
                function(*args, **kwargs)
                raise
            except:
                ok = False
        return ok

    def __call__(self, *args, **kwds):
        return self.run(*args, **kwds)

    def debug(self):
        """Run the test without collecting errors in a TestResult"""
        self.setUp()
        getattr(self, self._testMethodName)()
        self.tearDown()
            function(*args, **kwargs)

    def skipTest(self, reason):
        """Skip this test."""
        raise SkipTest(reason)

    def fail(self, msg=None):
        """Fail immediately, with the given message."""
        raise self.failureException(msg)

    def assertFalse(self, expr, msg=None):
        """Check that the expression is false."""
        if expr:
            msg = self._formatMessage(msg, "%s is not false" % safe_repr(expr))
            raise self.failureException(msg)

    def assertTrue(self, expr, msg=None):
        """Check that the expression is true."""
        if not expr:
            msg = self._formatMessage(msg, "%s is not true" % safe_repr(expr))

    def _formatMessage(self, msg, standardMsg):
        """Honour the longMessage attribute when generating failure messages.
        If longMessage is False this means:
        * Use only an explicit message if it is provided
        * Otherwise use the standard message for the assert

        If longMessage is True:
        * Use the standard message
        * If an explicit message is provided, plus ' : ' and the explicit message
        """
        if not self.longMessage:
            return msg or standardMsg
        if msg is None:
            return standardMsg
        try:
            # don't switch to '{}' formatting in Python 2.X
            # it changes the way unicode input is handled
            return '%s : %s' % (standardMsg, msg)
        except UnicodeDecodeError:
            return  '%s : %s' % (safe_repr(standardMsg), safe_repr(msg))

    def assertRaises(self, excClass, callableObj=None, *args, **kwargs):
        """Fail unless an exception of class excClass is raised
           by callableObj when invoked with arguments args and keyword
           arguments kwargs. If a different type of exception is
           raised, it will not be caught, and the test case will be
           deemed to have suffered an error, exactly as for an
           unexpected exception.

           If called with callableObj omitted or None, will return a
           context object used like this::

                with self.assertRaises(SomeException):
                    do_something()

           The context manager keeps a reference to the exception as
           the 'exception' attribute. This allows you to inspect the
           exception after the assertion::

               with self.assertRaises(SomeException) as cm:
                   do_something()
               the_exception = cm.exception
               self.assertEqual(the_exception.error_code, 3)
        """
        context = _AssertRaisesContext(excClass, self)
        if callableObj is None:
            return context
        with context:
            callableObj(*args, **kwargs)

    def _getAssertEqualityFunc(self, first, second):
        """Get a detailed comparison function for the types of the two args.

        Returns: A callable accepting (first, second, msg=None) that will
        raise a failure exception if first != second with a useful human
        readable error message for those types.
        """
        #
        # NOTE(gregory.p.smith): I considered isinstance(first, type(second))
        # and vice versa.  I opted for the conservative approach in case
        # subclasses are not intended to be compared in detail to their super
        # class instances using a type equality func.  This means testing
        # subtypes won't automagically use the detailed comparison.  Callers
        # should use their type specific assertSpamEqual method to compare
        # subclasses if the detailed comparison is desired and appropriate.
        # See the discussion in http://bugs.python.org/issue2578.
        #
        if type(first) is type(second):
            asserter = self._type_equality_funcs.get(type(first))
            if asserter is not None:
                if isinstance(asserter, basestring):
                    asserter = getattr(self, asserter)
                return asserter

        return self._baseAssertEqual

    def _baseAssertEqual(self, first, second, msg=None):
        """The default assertEqual implementation, not type specific."""
        if not first == second:
            standardMsg = '%s != %s' % (safe_repr(first), safe_repr(second))
            msg = self._formatMessage(msg, standardMsg)

    def assertEqual(self, first, second, msg=None):
        """Fail if the two objects are unequal as determined by the '=='
           operator.
        """
        assertion_func = self._getAssertEqualityFunc(first, second)
        assertion_func(first, second, msg=msg)

    def assertNotEqual(self, first, second, msg=None):
        """Fail if the two objects are equal as determined by the '!='
           operator.
        """
        if not first != second:
            msg = self._formatMessage(msg, '%s == %s' % (safe_repr(first),
                                                          safe_repr(second)))

    def assertAlmostEqual(self, first, second, places=None, msg=None, delta=None):
        """Fail if the two objects are unequal as determined by their
           difference rounded to the given number of decimal places
           (default 7) and comparing to zero, or by comparing that the
           between the two objects is more than the given delta.

           Note that decimal places (from zero) are usually not the same
           as significant digits (measured from the most signficant digit).

           If the two objects compare equal then they will automatically
           compare almost equal.
        """
        if first == second:
            # shortcut
            return
        if delta is not None and places is not None:
            raise TypeError("specify delta or places not both")

        if delta is not None:
            if abs(first - second) <= delta:
                return

            standardMsg = '%s != %s within %s delta' % (safe_repr(first),
                                                        safe_repr(second),
                                                        safe_repr(delta))
        else:
            if places is None:
                places = 7

            if round(abs(second-first), places) == 0:
                return

            standardMsg = '%s != %s within %r places' % (safe_repr(first),
                                                          safe_repr(second),
                                                          places)
        msg = self._formatMessage(msg, standardMsg)

    def assertNotAlmostEqual(self, first, second, places=None, msg=None, delta=None):
        """Fail if the two objects are equal as determined by their
           between the two objects is less than the given delta.

           Objects that are equal automatically fail.
        """
            if not (first == second) and abs(first - second) > delta:
                return
            standardMsg = '%s == %s within %s delta' % (safe_repr(first),
        else:
                places = 7
            if not (first == second) and round(abs(second-first), places) != 0:
                return
            standardMsg = '%s == %s within %r places' % (safe_repr(first),
                                                         safe_repr(second),
                                                         places)

    # Synonyms for assertion methods

    # The plurals are undocumented.  Keep them that way to discourage use.
    # Do not add more.  Do not remove.
    # Going through a deprecation cycle on these would annoy many people.
    assertEquals = assertEqual
    assertNotEquals = assertNotEqual
    assertAlmostEquals = assertAlmostEqual
    assertNotAlmostEquals = assertNotAlmostEqual
    assert_ = assertTrue

    # These fail* assertion method names are pending deprecation and will
    # be a DeprecationWarning in 3.2; http://bugs.python.org/issue2578
    def _deprecate(original_func):
        def deprecated_func(*args, **kwargs):
            warnings.warn(
                'Please use {0} instead.'.format(original_func.__name__),
                PendingDeprecationWarning, 2)
            return original_func(*args, **kwargs)
        return deprecated_func

    failUnlessEqual = _deprecate(assertEqual)
    failIfEqual = _deprecate(assertNotEqual)
    failUnlessAlmostEqual = _deprecate(assertAlmostEqual)
    failIfAlmostEqual = _deprecate(assertNotAlmostEqual)
    failUnless = _deprecate(assertTrue)
    failUnlessRaises = _deprecate(assertRaises)
    failIf = _deprecate(assertFalse)

    def assertSequenceEqual(self, seq1, seq2, msg=None, seq_type=None):
        """An equality assertion for ordered sequences (like lists and tuples).

        For the purposes of this function, a valid ordered sequence type is one
        which can be indexed, has a length, and has an equality operator.

        Args:
            seq1: The first sequence to compare.
            seq2: The second sequence to compare.
            seq_type: The expected datatype of the sequences, or None if no
                    datatype should be enforced.
            msg: Optional message to use on failure instead of a list of
                    differences.
        """
        if seq_type is not None:
            seq_type_name = seq_type.__name__
            if not isinstance(seq1, seq_type):
                raise self.failureException('First sequence is not a %s: %s'
                                        % (seq_type_name, safe_repr(seq1)))
            if not isinstance(seq2, seq_type):
                raise self.failureException('Second sequence is not a %s: %s'
                                        % (seq_type_name, safe_repr(seq2)))
        else:
            seq_type_name = "sequence"

        differing = None
        try:
            len1 = len(seq1)
        except (TypeError, NotImplementedError):
            differing = 'First %s has no length.    Non-sequence?' % (
                    seq_type_name)

        if differing is None:
            try:
                len2 = len(seq2)
            except (TypeError, NotImplementedError):
                differing = 'Second %s has no length.    Non-sequence?' % (
                        seq_type_name)

            if seq1 == seq2:
                return

            seq1_repr = safe_repr(seq1)
            seq2_repr = safe_repr(seq2)
            if len(seq1_repr) > 30:
                seq1_repr = seq1_repr[:30] + '...'
            if len(seq2_repr) > 30:
                seq2_repr = seq2_repr[:30] + '...'
            elements = (seq_type_name.capitalize(), seq1_repr, seq2_repr)
            differing = '%ss differ: %s != %s\n' % elements

            for i in xrange(min(len1, len2)):
                try:
                    item1 = seq1[i]
                except (TypeError, IndexError, NotImplementedError):
                    differing += ('\nUnable to index element %d of first %s\n' %
                                 (i, seq_type_name))
                    break

                try:
                    item2 = seq2[i]
                    differing += ('\nUnable to index element %d of second %s\n' %
                    break

                if item1 != item2:
                    differing += ('\nFirst differing element %d:\n%s\n%s\n' %
                                 (i, item1, item2))
                    break
            else:
                if (len1 == len2 and seq_type is None and
                    type(seq1) != type(seq2)):
                    # The sequences are the same, but have differing types.
                    return

            if len1 > len2:
                differing += ('\nFirst %s contains %d additional '
                             'elements.\n' % (seq_type_name, len1 - len2))
                try:
                    differing += ('First extra element %d:\n%s\n' %
                                  (len2, seq1[len2]))
                    differing += ('Unable to index element %d '
                                  'of first %s\n' % (len2, seq_type_name))
            elif len1 < len2:
                differing += ('\nSecond %s contains %d additional '
                             'elements.\n' % (seq_type_name, len2 - len1))
                try:
                                  (len1, seq2[len1]))
                                  'of second %s\n' % (len1, seq_type_name))
        standardMsg = differing
        diffMsg = '\n' + '\n'.join(
            difflib.ndiff(pprint.pformat(seq1).splitlines(),
                          pprint.pformat(seq2).splitlines()))
        standardMsg = self._truncateMessage(standardMsg, diffMsg)
        self.fail(msg)

    def _truncateMessage(self, message, diff):
        max_diff = self.maxDiff
        if max_diff is None or len(diff) <= max_diff:
            return message + diff
        return message + (DIFF_OMITTED % len(diff))

    def assertListEqual(self, list1, list2, msg=None):
        """A list-specific equality assertion.

        Args:
            list1: The first list to compare.
            list2: The second list to compare.

        """
        self.assertSequenceEqual(list1, list2, msg, seq_type=list)

    def assertTupleEqual(self, tuple1, tuple2, msg=None):
        """A tuple-specific equality assertion.

        Args:
            tuple1: The first tuple to compare.
            tuple2: The second tuple to compare.
        """
        self.assertSequenceEqual(tuple1, tuple2, msg, seq_type=tuple)

    def assertSetEqual(self, set1, set2, msg=None):
        """A set-specific equality assertion.

        Args:
            set1: The first set to compare.
            set2: The second set to compare.

        assertSetEqual uses ducktyping to support different types of sets, and
        is optimized for sets specifically (parameters must support a
        difference method).
        """
        try:
            difference1 = set1.difference(set2)
        except TypeError, e:
            self.fail('invalid type when attempting set difference: %s' % e)
        except AttributeError, e:
            self.fail('first argument does not support set difference: %s' % e)

        try:
            difference2 = set2.difference(set1)
            self.fail('second argument does not support set difference: %s' % e)

        if not (difference1 or difference2):
            return

        lines = []
        if difference1:
            lines.append('Items in the first set but not the second:')
            for item in difference1:
                lines.append(repr(item))
        if difference2:
            lines.append('Items in the second set but not the first:')
            for item in difference2:

        standardMsg = '\n'.join(lines)
        self.fail(self._formatMessage(msg, standardMsg))

    def assertIn(self, member, container, msg=None):
        """Just like self.assertTrue(a in b), but with a nicer default message."""
        if member not in container:
            standardMsg = '%s not found in %s' % (safe_repr(member),
                                                  safe_repr(container))
            self.fail(self._formatMessage(msg, standardMsg))

    def assertNotIn(self, member, container, msg=None):
        """Just like self.assertTrue(a not in b), but with a nicer default message."""
        if member in container:
            standardMsg = '%s unexpectedly found in %s' % (safe_repr(member),
                                                        safe_repr(container))

    def assertIs(self, expr1, expr2, msg=None):
        """Just like self.assertTrue(a is b), but with a nicer default message."""
        if expr1 is not expr2:
            standardMsg = '%s is not %s' % (safe_repr(expr1),
                                             safe_repr(expr2))

    def assertIsNot(self, expr1, expr2, msg=None):
        """Just like self.assertTrue(a is not b), but with a nicer default message."""
        if expr1 is expr2:
            standardMsg = 'unexpectedly identical: %s' % (safe_repr(expr1),)

    def assertDictEqual(self, d1, d2, msg=None):
        self.assertIsInstance(d1, dict, 'First argument is not a dictionary')
        self.assertIsInstance(d2, dict, 'Second argument is not a dictionary')

        if d1 != d2:
            standardMsg = '%s != %s' % (safe_repr(d1, True), safe_repr(d2, True))
            diff = ('\n' + '\n'.join(difflib.ndiff(
                           pprint.pformat(d1).splitlines(),
                           pprint.pformat(d2).splitlines())))
            standardMsg = self._truncateMessage(standardMsg, diff)

    def assertDictContainsSubset(self, expected, actual, msg=None):
        """Checks whether actual is a superset of expected."""
        missing = []
        mismatched = []
        for key, value in expected.iteritems():
            if key not in actual:
                missing.append(key)
            elif value != actual[key]:
                mismatched.append('%s, expected: %s, actual: %s' %
                                  (safe_repr(key), safe_repr(value),
                                   safe_repr(actual[key])))

        if not (missing or mismatched):
            return

        standardMsg = ''
        if missing:
            standardMsg = 'Missing: %s' % ','.join(safe_repr(m) for m in
                                                    missing)
        if mismatched:
            if standardMsg:
                standardMsg += '; '
            standardMsg += 'Mismatched values: %s' % ','.join(mismatched)

    def assertItemsEqual(self, expected_seq, actual_seq, msg=None):
        """An unordered sequence specific comparison. It asserts that
        actual_seq and expected_seq have the same element counts.
        Equivalent to::

            self.assertEqual(Counter(iter(actual_seq)),
                             Counter(iter(expected_seq)))

        Asserts that each element has the same count in both sequences.
        Example:
            - [0, 1, 1] and [1, 0, 1] compare equal.
            - [0, 0, 1] and [0, 1] compare unequal.
        """
        first_seq, second_seq = list(expected_seq), list(actual_seq)
        with warnings.catch_warnings():
            if sys.py3kwarning:
                # Silence Py3k warning raised during the sorting
                for _msg in ["(code|dict|type) inequality comparisons",
                             "builtin_function_or_method order comparisons",
                             "comparing unequal types"]:
                    warnings.filterwarnings("ignore", _msg, DeprecationWarning)
            try:
                first = collections.Counter(first_seq)
                second = collections.Counter(second_seq)
            except TypeError:
                # Handle case with unhashable elements
                differences = _count_diff_all_purpose(first_seq, second_seq)
            else:
                if first == second:
                    return
                differences = _count_diff_hashable(first_seq, second_seq)

        if differences:
            standardMsg = 'Element counts were not equal:\n'
            lines = ['First has %d, Second has %d:  %r' % diff for diff in differences]
            diffMsg = '\n'.join(lines)
            standardMsg = self._truncateMessage(standardMsg, diffMsg)
            self.fail(msg)

    def assertMultiLineEqual(self, first, second, msg=None):
        """Assert that two multi-line strings are equal."""
        self.assertIsInstance(first, basestring,
                'First argument is not a string')
        self.assertIsInstance(second, basestring,
                'Second argument is not a string')

        if first != second:
            # don't use difflib if the strings are too long
            if (len(first) > self._diffThreshold or
                len(second) > self._diffThreshold):
                self._baseAssertEqual(first, second, msg)
            firstlines = first.splitlines(True)
            secondlines = second.splitlines(True)
            if len(firstlines) == 1 and first.strip('\r\n') == first:
                firstlines = [first + '\n']
                secondlines = [second + '\n']
            standardMsg = '%s != %s' % (safe_repr(first, True),
                                        safe_repr(second, True))
            diff = '\n' + ''.join(difflib.ndiff(firstlines, secondlines))

    def assertLess(self, a, b, msg=None):
        """Just like self.assertTrue(a < b), but with a nicer default message."""
        if not a < b:
            standardMsg = '%s not less than %s' % (safe_repr(a), safe_repr(b))

    def assertLessEqual(self, a, b, msg=None):
        """Just like self.assertTrue(a <= b), but with a nicer default message."""
        if not a <= b:
            standardMsg = '%s not less than or equal to %s' % (safe_repr(a), safe_repr(b))

    def assertGreater(self, a, b, msg=None):
        """Just like self.assertTrue(a > b), but with a nicer default message."""
        if not a > b:
            standardMsg = '%s not greater than %s' % (safe_repr(a), safe_repr(b))

    def assertGreaterEqual(self, a, b, msg=None):
        """Just like self.assertTrue(a >= b), but with a nicer default message."""
        if not a >= b:
            standardMsg = '%s not greater than or equal to %s' % (safe_repr(a), safe_repr(b))

    def assertIsNone(self, obj, msg=None):
        """Same as self.assertTrue(obj is None), with a nicer default message."""
        if obj is not None:
            standardMsg = '%s is not None' % (safe_repr(obj),)

    def assertIsNotNone(self, obj, msg=None):
        """Included for symmetry with assertIsNone."""
        if obj is None:
            standardMsg = 'unexpectedly None'

    def assertIsInstance(self, obj, cls, msg=None):
        """Same as self.assertTrue(isinstance(obj, cls)), with a nicer
        default message."""
        if not isinstance(obj, cls):
            standardMsg = '%s is not an instance of %r' % (safe_repr(obj), cls)

    def assertNotIsInstance(self, obj, cls, msg=None):
        """Included for symmetry with assertIsInstance."""
        if isinstance(obj, cls):
            standardMsg = '%s is an instance of %r' % (safe_repr(obj), cls)

    def assertRaisesRegexp(self, expected_exception, expected_regexp,
                           callable_obj=None, *args, **kwargs):
        """Asserts that the message in a raised exception matches a regexp.

        Args:
            expected_exception: Exception class expected to be raised.
            expected_regexp: Regexp (re pattern object or string) expected
                    to be found in error message.
            callable_obj: Function to be called.
            args: Extra args.
            kwargs: Extra kwargs.
        """
        if expected_regexp is not None:
            expected_regexp = re.compile(expected_regexp)
        context = _AssertRaisesContext(expected_exception, self, expected_regexp)
        if callable_obj is None:
            callable_obj(*args, **kwargs)

    def assertRegexpMatches(self, text, expected_regexp, msg=None):
        """Fail the test unless the text matches the regular expression."""
        if isinstance(expected_regexp, basestring):
        if not expected_regexp.search(text):
            msg = msg or "Regexp didn't match"
            msg = '%s: %r not found in %r' % (msg, expected_regexp.pattern, text)

    def assertNotRegexpMatches(self, text, unexpected_regexp, msg=None):
        """Fail the test if the text matches the regular expression."""
        if isinstance(unexpected_regexp, basestring):
            unexpected_regexp = re.compile(unexpected_regexp)
        match = unexpected_regexp.search(text)
        if match:
            msg = msg or "Regexp matched"
            msg = '%s: %r matches %r in %r' % (msg,
                                               text[match.start():match.end()],
                                               unexpected_regexp.pattern,
                                               text)

class FunctionTestCase(TestCase):
    """A test case that wraps a test function.

    This is useful for slipping pre-existing test functions into the
    unittest framework. Optionally, set-up and tidy-up functions can be
    supplied. As with TestCase, the tidy-up ('tearDown') function will
    always be called if the set-up ('setUp') function ran successfully.
    """

    def __init__(self, testFunc, setUp=None, tearDown=None, description=None):
        super(FunctionTestCase, self).__init__()
        self._setUpFunc = setUp
        self._tearDownFunc = tearDown
        self._testFunc = testFunc
        self._description = description

        if self._setUpFunc is not None:
            self._setUpFunc()

        if self._tearDownFunc is not None:
            self._tearDownFunc()

    def runTest(self):
        self._testFunc()

        return self._testFunc.__name__

        if not isinstance(other, self.__class__):

        return self._setUpFunc == other._setUpFunc and \
               self._tearDownFunc == other._tearDownFunc and \
               self._testFunc == other._testFunc and \
               self._description == other._description

        return hash((type(self), self._setUpFunc, self._tearDownFunc,
                     self._testFunc, self._description))

        return "%s (%s)" % (strclass(self.__class__),
                            self._testFunc.__name__)

        return "<%s tec=%s>" % (strclass(self.__class__),
                                     self._testFunc)

        if self._description is not None:
            return self._description
        doc = self._testFunc.__doc__

## Copyright (C) 2010 Soren Hauberg
##
## Copyright James Hensman 2011
## 
## This program is free software; you can redistribute it and/or modify it
## under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 3 of the License, or (at
## your option) any later version.
##
## This program is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program; see the file COPYING.  If not, see
## <http://www.gnu.org/licenses/>.

def erfcx (arg):
	arg = np.atleast_1d(arg)
	assert(np.all(np.isreal(arg)),"erfcx: input must be real")

	## Get precision dependent thresholds -- or not :p
	xneg = -26.628;
	xmax = 2.53e+307;

	## Allocate output
	result = np.zeros (arg.shape)

	## Find values where erfcx can be evaluated
	idx_neg = (arg < xneg);
	idx_max = (arg > xmax);
	idx = ~(idx_neg | idx_max);

	arg = arg [idx];

	## Perform the actual computation
	t = 3.97886080735226 / (np.abs (arg) + 3.97886080735226);
	u = t - 0.5;
	y = (((((((((u * 0.00127109764952614092 + 1.19314022838340944e-4) * u \
	    - 0.003963850973605135)   * u - 8.70779635317295828e-4) * u +     \
	      0.00773672528313526668) * u + 0.00383335126264887303) * u -     \
	      0.0127223813782122755)  * u - 0.0133823644533460069)  * u +     \
	      0.0161315329733252248)  * u + 0.0390976845588484035)  * u +     \
	      0.00249367200053503304;
	y = ((((((((((((y * u - 0.0838864557023001992) * u -           \
	      0.119463959964325415) * u + 0.0166207924969367356) * u + \
	      0.357524274449531043) * u + 0.805276408752910567)  * u + \
	      1.18902982909273333)  * u + 1.37040217682338167)   * u + \
	      1.31314653831023098)  * u + 1.07925515155856677)   * u + \
	      0.774368199119538609) * u + 0.490165080585318424)  * u + \
	      0.275374741597376782) * t;

	y [arg < 0] = 2 * np.exp (arg [arg < 0]**2) - y [arg < 0];

	## Put the results back into something with the same size is the original input
	result [idx] = y;
	result [idx_neg] = np.inf;
	## result (idx_max) = 0; # not needed as we initialise with zeros
	return(result)

__doc__ = """
Inference over Gaussian process latent functions

In all our GP models, the consistency propery means that we have a Gaussian
prior over a finite set of points f. This prior is

  math:: N(f | 0, K)

where K is the kernel matrix.

We also have a likelihood (see GPy.likelihoods) which defines how the data are
related to the latent function: p(y | f).  If the likelihood is also a Gaussian,
the inference over f is tractable (see exact_gaussian_inference.py).

If the likelihood object is something other than Gaussian, then exact inference
is not tractable. We then resort to a Laplace approximation (laplace.py) or
expectation propagation (ep.py).

The inference methods return a
:class:`~GPy.inference.latent_function_inference.posterior.Posterior`
instance, which is a simple
structure which contains a summary of the posterior. The model classes can then
use this posterior object for making predictions, optimizing hyper-parameters,
etc.

"""

class LatentFunctionInference(object):
        """
        This function gets called, just before the optimization loop to start.
        """
        pass

        """
        This function gets called, just after the optimization loop ended.
        """
        pass

class InferenceMethodList(LatentFunctionInference, list):

        for inf in self:
            inf.on_optimization_start()

            inf.on_optimization_end()
    
        state = []
            state.append(inf)
    
        for inf in state:
            self.append(inf)

from exact_gaussian_inference import ExactGaussianInference
from laplace import Laplace
from GPy.inference.latent_function_inference.var_dtc import VarDTC
from expectation_propagation import EP
from expectation_propagation_dtc import EPDTC
from dtc import DTC
from fitc import FITC
from var_dtc_parallel import VarDTC_minibatch

# class FullLatentFunctionData(object):
#
#

# class EMLikeLatentFunctionInference(LatentFunctionInference):
#     def update_approximation(self):
#         """
#         This function gets called when the 
#         """
#     
#     def inference(self, kern, X, Z, likelihood, Y, Y_metadata=None):
#         """
#         Do inference on the latent functions given a covariance function `kern`,
#         inputs and outputs `X` and `Y`, inducing_inputs `Z`, and a likelihood `likelihood`.
#         Additional metadata for the outputs `Y` can be given in `Y_metadata`.
#         """
#         raise NotImplementedError, "Abstract base class for full inference"
# 
# class VariationalLatentFunctionInference(LatentFunctionInference):
#         """
#         """

import sys
from model import Model
from parameterization import ObsAr
from ..inference.latent_function_inference import exact_gaussian_inference, expectation_propagation
from parameterization.variational import VariationalPosterior

from GPy.util.normalizer import MeanNorm
logger = logging.getLogger("GP")

class GP(Model):
    """
    General purpose Gaussian process model

    :param Y: output observations
    :param likelihood: a GPy likelihood
    :param inference_method: The :class:`~GPy.inference.latent_function_inference.LatentFunctionInference` inference method to use for this GP
    :param Norm normalizer:
        normalize the outputs Y.
        Prediction will be un-normalized using this normalizer.
        If normalizer is None, we will normalize using MeanNorm.
        If normalizer is False, no normalization will be done.

    """
    def __init__(self, X, Y, kernel, likelihood, inference_method=None, name='gp', Y_metadata=None, normalizer=False):
        super(GP, self).__init__(name)

        assert X.ndim == 2
        if isinstance(X, (ObsAr, VariationalPosterior)):
            self.X = X.copy()
        else: self.X = ObsAr(X)

        assert Y.ndim == 2
        logger.info("initializing Y")

        if normalizer is True:
            self.normalizer = MeanNorm()
        elif normalizer is False:
            self.normalizer = None
        else:
            self.normalizer = normalizer

        if self.normalizer is not None:
            self.normalizer.scale_by(Y)
            self.Y_normalized = ObsAr(self.normalizer.normalize(Y))
            self.Y = Y
        else:
            self.Y = ObsAr(Y)
            self.Y_normalized = self.Y

        assert Y.shape[0] == self.num_data
        _, self.output_dim = self.Y.shape

        #TODO: check the type of this is okay?
        self.Y_metadata = Y_metadata

        assert isinstance(kernel, kern.Kern)
        #assert self.input_dim == kernel.input_dim

        assert isinstance(likelihood, likelihoods.Likelihood)
        self.likelihood = likelihood

        #find a sensible inference method
        logger.info("initializing inference method")
            if isinstance(likelihood, likelihoods.Gaussian) or isinstance(likelihood, likelihoods.MixedNoise):
                inference_method = exact_gaussian_inference.ExactGaussianInference()
            else:
                inference_method = expectation_propagation.EP()
                print "defaulting to ", inference_method, "for latent function inference"
        self.inference_method = inference_method

        logger.info("adding kernel and likelihood as parameters")

    def set_XY(self, X=None, Y=None):
        """
        Set the input / output data of the model
        This is useful if we wish to change our existing data but maintain the same model

        :type X: np.ndarray
        :param Y: output observations
        """
        if Y is not None:
            if self.normalizer is not None:
                self.normalizer.scale_by(Y)
                self.Y_normalized = ObsAr(self.normalizer.normalize(Y))
                self.Y = Y
            else:
                self.Y = ObsAr(Y)
                self.Y_normalized = self.Y
        if X is not None:
            if self.X in self.parameters:
                # LVM models
                if isinstance(self.X, VariationalPosterior):
                    assert isinstance(X, type(self.X)), "The given X must have the same type as the X in the model!"
                    self.unlink_parameter(self.X)
                    self.X = X
                    self.link_parameters(self.X)
                else:
                    from ..core import Param
                    self.X = Param('latent mean',X)
            else:
                self.X = ObsAr(X)

    def set_X(self,X):
        """
        Set the input data of the model

        """
        self.set_XY(X=X)

    def set_Y(self,Y):
        """
        Set the output data of the model

        :param X: output observations
        """
        self.set_XY(Y=Y)

        """
        Method that is called upon any changes to :class:`~GPy.core.parameterization.param.Param` variables within the model.
        In particular in the GP class this method reperforms inference, recalculating the posterior and log marginal likelihood and gradients of the model

        .. warning::
            This method is not designed to be called manually, the framework is set up to automatically call this method upon changes to parameters, if you call
            this method yourself, there may be unexpected consequences.
        """
        self.posterior, self._log_marginal_likelihood, self.grad_dict = self.inference_method.inference(self.kern, self.X, self.likelihood, self.Y_normalized, self.Y_metadata)
        self.likelihood.update_gradients(self.grad_dict['dL_dthetaL'])
        self.kern.update_gradients_full(self.grad_dict['dL_dK'], self.X)

        """
        The log marginal likelihood of the model, :math:`p(\mathbf{y})`, this is the objective function of the model being optimised
        """

    def _raw_predict(self, _Xnew, full_cov=False, kern=None):
        """
        For making predictions, does not account for normalization or likelihood

        full_cov is a boolean which defines whether the full covariance matrix
        of the prediction is computed. If full_cov is False (default), only the
        diagonal of the covariance is returned.

        .. math::
            p(f*|X*, X, Y) = \int^{\inf}_{\inf} p(f*|f,X*)p(f|X,Y) df
                        = N(f*| K_{x*x}(K_{xx} + \Sigma)^{-1}Y, K_{x*x*} - K_{xx*}(K_{xx} + \Sigma)^{-1}K_{xx*}
            \Sigma := \texttt{Likelihood.variance / Approximate likelihood covariance}
        """
        if kern is None:
            kern = self.kern

        Kx = kern.K(_Xnew, self.X).T
        WiKx = np.dot(self.posterior.woodbury_inv, Kx)
        mu = np.dot(Kx.T, self.posterior.woodbury_vector)
            Kxx = kern.K(_Xnew)
            var = Kxx - np.dot(Kx.T, WiKx)
        else:
            Kxx = kern.Kdiag(_Xnew)
            var = Kxx - np.sum(WiKx*Kx, 0)
            var = var.reshape(-1, 1)

        #force mu to be a column vector
        if len(mu.shape)==1: mu = mu[:,None]

    def predict(self, Xnew, full_cov=False, Y_metadata=None, kern=None):
        """

        :type Xnew: np.ndarray (Nnew x self.input_dim)
        :param full_cov: whether to return the full covariance matrix, or just
                         the diagonal
        :type full_cov: bool
        :param Y_metadata: metadata about the predicting point to pass to the likelihood
        :param kern: The kernel to use for prediction (defaults to the model
                     kern). this is useful for examining e.g. subprocesses.
        :returns: (mean, var, lower_upper):
            mean: posterior mean, a Numpy array, Nnew x self.input_dim
            var: posterior variance, a Numpy array, Nnew x 1 if full_cov=False, Nnew x Nnew otherwise
            lower_upper: lower and upper boundaries of the 95% confidence intervals, Numpy arrays,  Nnew x self.input_dim

           If full_cov and self.input_dim > 1, the return shape of var is Nnew x Nnew x self.input_dim. If self.input_dim == 1, the return shape is Nnew x Nnew.
           This is to allow for different normalizations of the output dimensions.
        """
        #predict the latent function values
        mu, var = self._raw_predict(Xnew, full_cov=full_cov, kern=kern)
            mu, var = self.normalizer.inverse_mean(mu), self.normalizer.inverse_variance(var)

        # now push through likelihood
        mean, var = self.likelihood.predictive_values(mu, var, full_cov, Y_metadata)
        return mean, var

    def predict_quantiles(self, X, quantiles=(2.5, 97.5), Y_metadata=None):
        """
        Get the predictive quantiles around the prediction at X

        :param X: The points at which to make a prediction
        :type X: np.ndarray (Xnew x self.input_dim)
        :param quantiles: tuple of quantiles, default is (2.5, 97.5) which is the 95% interval
        :type quantiles: tuple
        :returns: list of quantiles for each X and predictive quantiles for interval combination
        :rtype: [np.ndarray (Xnew x self.input_dim), np.ndarray (Xnew x self.input_dim)]
        """
        m, v = self._raw_predict(X,  full_cov=False)
            m, v = self.normalizer.inverse_mean(m), self.normalizer.inverse_variance(v)
        return self.likelihood.predictive_quantiles(m, v, quantiles, Y_metadata)

    def predictive_gradients(self, Xnew):
        """
        Compute the derivatives of the latent function with respect to X*

        Given a set of points at which to predict X* (size [N*,Q]), compute the
        derivatives of the mean and variance. Resulting arrays are sized:
         dmu_dX* -- [N*, Q ,D], where D is the number of output in this GP (usually one).

         dv_dX*  -- [N*, Q],    (since all outputs have the same variance)
        :param X: The points at which to get the predictive gradients
        :returns: dmu_dX, dv_dX
        :rtype: [np.ndarray (N*, Q ,D), np.ndarray (N*,Q) ]

        """
        dmu_dX = np.empty((Xnew.shape[0],Xnew.shape[1],self.output_dim))
            dmu_dX[:,:,i] = self.kern.gradients_X(self.posterior.woodbury_vector[:,i:i+1].T, Xnew, self.X)

        # gradients wrt the diagonal part k_{xx}
        dv_dX = self.kern.gradients_X(np.eye(Xnew.shape[0]), Xnew)
        #grads wrt 'Schur' part K_{xf}K_{ff}^{-1}K_{fx}
        alpha = -2.*np.dot(self.kern.K(Xnew, self.X),self.posterior.woodbury_inv)
        dv_dX += self.kern.gradients_X(alpha, Xnew, self.X)
        return dmu_dX, dv_dX

    def posterior_samples_f(self,X,size=10, full_cov=True):
        """
        Samples the posterior GP at the points X.

        :param X: The points at which to take the samples.
        :type X: np.ndarray (Nnew x self.input_dim)
        :param size: the number of a posteriori samples.
        :type size: int.
        :param full_cov: whether to return the full covariance matrix, or just the diagonal.
        :type full_cov: bool.
        :returns: Ysim: set of simulations
        :rtype: np.ndarray (N x samples)
        """
        m, v = self._raw_predict(X,  full_cov=full_cov)
        v = v.reshape(m.size,-1) if len(v.shape)==3 else v
        if not full_cov:
            Ysim = np.random.multivariate_normal(m.flatten(), np.diag(v.flatten()), size).T
        else:
            Ysim = np.random.multivariate_normal(m.flatten(), v, size).T

    def posterior_samples(self, X, size=10, full_cov=False, Y_metadata=None):
        """

        :param X: the points at which to take the samples.
        :type X: np.ndarray (Nnew x self.input_dim.)
        :param noise_model: for mixed noise likelihood, the noise model to use in the samples.
        :type noise_model: integer.
        :returns: Ysim: set of simulations, a Numpy array (N x samples).
        """
        Ysim = self.posterior_samples_f(X, size, full_cov=full_cov)
        Ysim = self.likelihood.samples(Ysim, Y_metadata)

    def plot_f(self, plot_limits=None, which_data_rows='all',
        plot_raw=True,
        linecol=None,fillcol=None, Y_metadata=None, data_symbol='kx'):
        """
        Plot the GP's view of the world, where the data is normalized and before applying a likelihood.
        This is a call to plot with plot_raw=True.
        Data will not be plotted in this, as the GP's view of the world
        may live in another space, or units then the data.

        Can plot only part of the data and part of the posterior functions
        using which_data_rowsm which_data_ycols.

        :param plot_limits: The limits of the plot. If 1D [xmin,xmax], if 2D [[xmin,ymin],[xmax,ymax]]. Defaluts to data limits
        :type plot_limits: np.array
        :param which_data_rows: which of the training data to plot (default all)
        :type which_data_rows: 'all' or a slice object to slice model.X, model.Y
        :param which_data_ycols: when the data has several columns (independant outputs), only plot these
        :type which_data_ycols: 'all' or a list of integers
        :param fixed_inputs: a list of tuple [(i,v), (i,v)...], specifying that input index i should be set to value v.
        :type fixed_inputs: a list of tuples
        :param resolution: the number of intervals to sample the GP on. Defaults to 200 in 1D and 50 (a 50x50 grid) in 2D
        :type resolution: int
        :param levels: number of levels to plot in a contour plot.
        :param levels: for 2D plotting, the number of contour levels to use is ax is None, create a new figure
        :type levels: int
        :param samples: the number of a posteriori samples to plot
        :type samples: int
        :param fignum: figure to plot on.
        :type fignum: figure number
        :param ax: axes to plot on.
        :type ax: axes handle
        :param linecol: color of line to plot [Tango.colorsHex['darkBlue']]
        :type linecol: color either as Tango.colorsHex object or character ('r' is red, 'g' is green) as is standard in matplotlib
        :param fillcol: color of fill [Tango.colorsHex['lightBlue']]
        :type fillcol: color either as Tango.colorsHex object or character ('r' is red, 'g' is green) as is standard in matplotlib
        :param Y_metadata: additional data associated with Y which may be needed
        :type Y_metadata: dict
        :param data_symbol: symbol as used matplotlib, by default this is a black cross ('kx')
        :type data_symbol: color either as Tango.colorsHex object or character ('r' is red, 'g' is green) alongside marker type, as is standard in matplotlib.
        """
        from ..plotting.matplot_dep import models_plots
        kw = {}
        if linecol is not None:
            kw['linecol'] = linecol
        if fillcol is not None:
            kw['fillcol'] = fillcol
        return models_plots.plot_fit(self, plot_limits, which_data_rows,
                                     which_data_ycols, fixed_inputs,
                                     levels, samples, fignum, ax, resolution,
                                     plot_raw=plot_raw, Y_metadata=Y_metadata,
                                     data_symbol=data_symbol, **kw)

    def plot(self, plot_limits=None, which_data_rows='all',
        """
        Plot the posterior of the GP.
          - In one dimension, the function is plotted with a shaded region identifying two standard deviations.
          - In two dimsensions, a contour-plot shows the mean predicted function
          - In higher dimensions, use fixed_inputs to plot the GP  with some of the inputs fixed.

        """
        kw = {}

        """
        Returns the sensitivity for each dimension of this model
        """
        return self.kern.input_sensitivity(summarize=summarize)

        """

        :param optimizer: which optimizer to use (defaults to self.preferred optimizer), a range of optimisers can be found in :module:`~GPy.inference.optimization`, they include 'scg', 'lbfgs', 'tnc'.
        """
        self.inference_method.on_optimization_start()
        try:
            super(GP, self).optimize(optimizer, start, **kwargs)
        except KeyboardInterrupt:
            print "KeyboardInterrupt caught, calling on_optimization_end() to round things up"
            self.inference_method.on_optimization_end()
            raise

    def infer_newX(self, Y_new, optimize=True, ):
        """
        Infer the distribution of X for the new observed data *Y_new*.

        :param Y_new: the new observed data for inference
        :type Y_new: numpy.ndarray
        :param optimize: whether to optimize the location of new X (True by default)
        :type optimize: boolean
        :return: a tuple containing the posterior estimation of X and the model that optimize X
        :rtype: (:class:`~GPy.core.parameterization.variational.VariationalPosterior` or numpy.ndarray, :class:`~GPy.core.model.Model`)
        """
        from ..inference.latent_function_inference.inferenceX import infer_newX
        return infer_newX(self, Y_new, optimize=optimize)

from ..core.parameterization.variational import SpikeAndSlabPrior, SpikeAndSlabPosterior
from ..kern._src.psi_comp.ssrbf_psi_gpucomp import PSICOMP_SSRBF_GPU

class SSGPLVM(SparseGP_MPI):
    """
    Spike-and-Slab Gaussian Process Latent Variable Model

    """
    def __init__(self, Y, input_dim, X=None, X_variance=None, Gamma=None, init='PCA', num_inducing=10,
                 Z=None, kernel=None, inference_method=None, likelihood=None, name='Spike_and_Slab GPLVM', group_spike=False, mpi_comm=None, pi=None, learnPi=True,normalizer=False, **kwargs):

        self.group_spike = group_spike
        
        if X == None:
        else:

        if X_variance is None: # The variance of the variational approximation (S)
            
        if Gamma is None:
            gamma = np.random.randn(X.shape[0], input_dim)
        else:
            gamma = Gamma.copy()
                
            kernel = kern.RBF(input_dim, lengthscale=fracs, ARD=True) # + kern.white(input_dim)
        if kernel.useGPU:
            kernel.psicomp = PSICOMP_SSRBF_GPU()
        
            inference_method = VarDTC_minibatch(mpi_comm=mpi_comm)

        if pi is None:
            pi = np.empty((input_dim))
            pi[:] = 0.5
        self.variational_prior = SpikeAndSlabPrior(pi=pi,learnPi=learnPi) # the prior probability of the latent binary variable b
        
        X = SpikeAndSlabPosterior(X, X_variance, gamma)
                
        super(SSGPLVM,self).__init__(X, Y, Z, kernel, likelihood, variational_prior=self.variational_prior, inference_method=inference_method, name=name, mpi_comm=mpi_comm, normalizer=normalizer, **kwargs)
#         self.X.unfix()
#         self.X.variance.constrain_positive()
                
        if self.group_spike:
            [self.X.gamma[:,i].tie('tieGamma'+str(i)) for i in xrange(self.X.gamma.shape[1])] # Tie columns together
        
        X.mean.gradient, X.variance.gradient, X.binary_prob.gradient = X_grad
    
        return X.mean.gradient, X.variance.gradient, X.binary_prob.gradient

        super(SSGPLVM,self).parameters_changed()
            return
        
        self._log_marginal_likelihood -= self.variational_prior.KL_divergence(self.X)

        self.X.mean.gradient, self.X.variance.gradient, self.X.binary_prob.gradient = self.kern.gradients_qX_expectations(variational_posterior=self.X, Z=self.Z, dL_dpsi0=self.grad_dict['dL_dpsi0'], dL_dpsi1=self.grad_dict['dL_dpsi1'], dL_dpsi2=self.grad_dict['dL_dpsi2'])

        self.variational_prior.update_gradients_KL(self.X)

        if self.kern.ARD:
            return self.kern.input_sensitivity()
        else:
            return self.variational_prior.pi

    def plot_latent(self, plot_inducing=True, *args, **kwargs):
        import sys

        return dim_reduction_plots.plot_latent(self, plot_inducing=plot_inducing, *args, **kwargs)

class StochasticStorage(object):
    '''
    This is a container for holding the stochastic parameters,
    such as subset indices or step length and so on.
    '''
    def __init__(self, model):
        """
        Initialize this stochastic container using the given model
        """

    def do_stochastics(self):
        """
        Update the internal state to the next batch of the stochastic
        descent algorithm.
        """
        pass

        """
        Reset the state of this stochastics generator.
        """

class SparseGPMissing(StochasticStorage):
    def __init__(self, model, batchsize=1):
        """
        Here we want to loop over all dimensions everytime.
        Thus, we can just make sure the loop goes over self.d every
        time.
        """
        self.d = xrange(model.Y_normalized.shape[1])

class SparseGPStochastics(StochasticStorage):
    """
    For the sparse gp we need to store the dimension we are in,
    and the indices corresponding to those
    """
        self.output_dim = model.Y.shape[1]
        self.do_stochastics()

        if self.batchsize == 1:
            self.current_dim = (self.current_dim+1)%self.output_dim
            self.d = [self.current_dim]
        else:
            import numpy as np
            self.d = np.random.choice(self.output_dim, size=self.batchsize, replace=False)

        self.current_dim = -1
        self.d = None

# Copyright (c) 2012, James Hesnsman

from kern import Kern, CombinationKernel

def index_to_slices(index):
    """
    take a numpy array of integers (index) and return a  nested list of slices such that the slices describe the start, stop points for each integer in the index. 

    e.g.
    >>> index = np.asarray([0,0,0,1,1,1,2,2,2])
    returns
    >>> [[slice(0,3,None)],[slice(3,6,None)],[slice(6,9,None)]]

    or, a more complicated example
    >>> index = np.asarray([0,0,1,1,0,2,2,2,1,1])
    returns
    >>> [[slice(0,2,None),slice(4,5,None)],[slice(2,4,None),slice(8,10,None)],[slice(5,8,None)]]
    """
    if len(index)==0:
        return[]

    #contruct the return structure
    ind = np.asarray(index,dtype=np.int)
    ret = [[] for i in range(ind.max()+1)]

    #find the switchpoints
    ind_ = np.hstack((ind,ind[0]+ind[-1]+1))
    switchpoints = np.nonzero(ind_ - np.roll(ind_,+1))[0]

    [ret[ind_i].append(slice(*indexes_i)) for ind_i,indexes_i in zip(ind[switchpoints[:-1]],zip(switchpoints,switchpoints[1:]))]
    return ret

class IndependentOutputs(CombinationKernel):
    """
    A kernel which can represent several independent functions.  this kernel
    'switches off' parts of the matrix where the output indexes are different.

    The index of the functions is given by the last column in the input X the
    rest of the columns of X are passed to the underlying kernel for
    computation (in blocks).

    :param kernels: either a kernel, or list of kernels to work with. If it is
    a list of kernels the indices in the index_dim, index the kernels you gave!
    """
    def __init__(self, kernels, index_dim=-1, name='independ'):
        assert isinstance(index_dim, int), "IndependentOutputs kernel is only defined with one input dimension being the index"
        if not isinstance(kernels, list):
            self.single_kern = True
            self.kern = kernels
            kernels = [kernels]
        else:
            self.single_kern = False
        super(IndependentOutputs, self).__init__(kernels=kernels, extra_dims=[index_dim], name=name)

        kerns = itertools.repeat(self.kern) if self.single_kern else self.kern
            [[target.__setitem__((s,ss), kern.K(X[s,:], X[ss,:])) for s,ss in itertools.product(slices_i, slices_i)] for kern, slices_i in zip(kerns, slices)]
        else:
            [[target.__setitem__((s,s2), kern.K(X[s,:],X2[s2,:])) for s,s2 in itertools.product(slices_i, slices_j)] for kern, slices_i,slices_j in zip(kerns, slices,slices2)]

        target = np.zeros(X.shape[0])
        [[np.copyto(target[s], kern.Kdiag(X[s])) for s in slices_i] for kern, slices_i in zip(kerns, slices)]

        if self.single_kern: 
            target = np.zeros(self.kern.size)
            kerns = itertools.repeat(self.kern)
        else: 
            kerns = self.kern
            target = [np.zeros(kern.size) for kern, _ in zip(kerns, slices)]
        def collate_grads(kern, i, dL, X, X2):
            kern.update_gradients_full(dL,X,X2)
            if self.single_kern: target[:] += kern.gradient
            else: target[i][:] += kern.gradient
            [[collate_grads(kern, i, dL_dK[s,ss], X[s], X[ss]) for s,ss in itertools.product(slices_i, slices_i)] for i,(kern,slices_i) in enumerate(zip(kerns,slices))]
        else:
            [[[collate_grads(kern, i, dL_dK[s,s2],X[s],X2[s2]) for s in slices_i] for s2 in slices_j] for i,(kern,slices_i,slices_j) in enumerate(zip(kerns,slices,slices2))]
        if self.single_kern: kern.gradient = target
        else:[kern.gradient.__setitem__(Ellipsis, target[i]) for i, [kern, _] in enumerate(zip(kerns, slices))]

    def gradients_X(self,dL_dK, X, X2=None):
        target = np.zeros(X.shape)
            # TODO: make use of index_to_slices
            values = np.unique(X[:,self.index_dim])
            slices = [X[:,self.index_dim]==i for i in values]
            [target.__setitem__(s, kern.gradients_X(dL_dK[s,s],X[s],None))
              for kern, s in zip(kerns, slices)]
            #slices = index_to_slices(X[:,self.index_dim])
            #[[np.add(target[s], kern.gradients_X(dL_dK[s,s], X[s]), out=target[s]) 
            #  for s in slices_i] for kern, slices_i in zip(kerns, slices)]
            #import ipdb;ipdb.set_trace()
            #[[(np.add(target[s ], kern.gradients_X(dL_dK[s ,ss],X[s ], X[ss]), out=target[s ]),
            #   np.add(target[ss], kern.gradients_X(dL_dK[ss,s ],X[ss], X[s ]), out=target[ss]))
            #  for s, ss in itertools.combinations(slices_i, 2)] for kern, slices_i in zip(kerns, slices)]
        else:
            slices2 = [X2[:,self.index_dim]==i for i in values]
            [target.__setitem__(s, kern.gradients_X(dL_dK[s, :][:, s2],X[s],X2[s2]))
              for kern, s, s2 in zip(kerns, slices, slices2)]
            # TODO: make work with index_to_slices
            #slices2 = index_to_slices(X2[:,self.index_dim])
            #[[target.__setitem__(s, target[s] + kern.gradients_X(dL_dK[s,s2], X[s], X2[s2])) for s, s2 in itertools.product(slices_i, slices_j)] for kern, slices_i,slices_j in zip(kerns, slices,slices2)]

        [[target.__setitem__(s, kern.gradients_X_diag(dL_dKdiag[s],X[s])) for s in slices_i] for kern, slices_i in zip(kerns, slices)]

        if self.single_kern: target = np.zeros(self.kern.size)
        else: target = [np.zeros(kern.size) for kern, _ in zip(kerns, slices)]
        def collate_grads(kern, i, dL, X):
            kern.update_gradients_diag(dL,X)
        [[collate_grads(kern, i, dL_dKdiag[s], X[s,:]) for s in slices_i] for i, (kern, slices_i) in enumerate(zip(kerns, slices))]

class Hierarchical(CombinationKernel):
    """
    A kernel which can represent a simple hierarchical model.

    See Hensman et al 2013, "Hierarchical Bayesian modelling of gene expression time
    series across irregularly sampled replicates and clusters"
    http://www.biomedcentral.com/1471-2105/14/252

    To construct this kernel, you must pass a list of kernels. the first kernel
    will be assumed to be the 'base' kernel, and will be computed everywhere.
    For every additional kernel, we assume another layer in the hierachy, with
    a corresponding column of the input matrix which indexes which function the
    data are in at that level.

    For more, see the ipython notebook documentation on Hierarchical
    covariances.
    """
    def __init__(self, kernels, name='hierarchy'):
        assert all([k.input_dim==kernels[0].input_dim for k in kernels])
        assert len(kernels) > 1
        self.levels = len(kernels) -1
        input_max = max([k.input_dim for k in kernels])
        super(Hierarchical, self).__init__(kernels=kernels, extra_dims = range(input_max, input_max + len(kernels)-1), name=name)

        K = self.parts[0].K(X, X2) # compute 'base' kern everywhere
        slices = [index_to_slices(X[:,i]) for i in self.extra_dims]
            [[[np.add(K[s,s], k.K(X[s], None), K[s, s]) for s in slices_i] for slices_i in slices_k] for k, slices_k in zip(self.parts[1:], slices)]
        else:
            slices2 = [index_to_slices(X2[:,i]) for i in self.extra_dims]
            [[[np.add(K[s,ss], k.K(X[s], X2[ss]), K[s, ss]) for s,ss in zip(slices_i, slices_j)] for slices_i, slices_j in zip(slices_k1, slices_k2)] for k, slices_k1, slices_k2 in zip(self.parts[1:], slices, slices2)]
        return K

            self.parts[0].update_gradients_full(dL_dK, X, None)
            for k, slices_k in zip(self.parts[1:], slices):
                target = np.zeros(k.size)
                def collate_grads(dL, X, X2, target):
                    k.update_gradients_full(dL,X,X2)
                    target += k.gradient
                [[collate_grads(dL_dK[s,s], X[s], None, target) for s in slices_i] for slices_i in slices_k]
                k.gradient[:] = target
        else:
            raise NotImplementedError

import pylab as pb
import Tango
from matplotlib.textpath import TextPath
from matplotlib.transforms import offset_copy
from .base_plots import ax_default

def add_bar_labels(fig, ax, bars, bottom=0):
    transOffset = offset_copy(ax.transData, fig=fig,
                              x=0., y= -2., units='points')
    transOffsetUp = offset_copy(ax.transData, fig=fig,
                              x=0., y=1., units='points')
    for bar in bars:
        for i, [patch, num] in enumerate(zip(bar.patches, np.arange(len(bar.patches)))):
            if len(bottom) == len(bar): b = bottom[i]
            else: b = bottom
            height = patch.get_height() + b
            xi = patch.get_x() + patch.get_width() / 2.
            va = 'top'
            c = 'w'
            t = TextPath((0, 0), "${xi}$".format(xi=xi), rotation=0, ha='center')
            transform = transOffset
            if patch.get_extents().height <= t.get_extents().height + 5:
                va = 'bottom'
                c = 'k'
                transform = transOffsetUp
            ax.text(xi, height, "${xi}$".format(xi=int(num)), color=c, rotation=0, ha='center', va=va, transform=transform)

    ax.set_xticks([])

def plot_bars(fig, ax, x, ard_params, color, name, bottom=0):
    return ax.bar(left=x, height=ard_params.view(np.ndarray), width=.8,
                  bottom=bottom, align='center',
                  color=color, edgecolor='k', linewidth=1.2,
                  label=name.replace("_"," "))

def plot_ARD(kernel, fignum=None, ax=None, title='', legend=False, filtering=None):
    """
    If an ARD kernel is present, plot a bar representation using matplotlib

    :param fignum: figure number of the plot
    :param ax: matplotlib axis to plot on
    :param title:
        title of the plot,
        pass '' to not print a title
        pass None for a generic title
    :param filtering: list of names, which to use for plotting ARD parameters.
                      Only kernels which match names in the list of names in filtering
                      will be used for plotting.
    :type filtering: list of names to use for ARD plot
    """
    fig, ax = ax_default(fignum,ax)

    if title is None:
        ax.set_title('ARD parameters, %s kernel' % kernel.name)
    else:
        ax.set_title(title)

    bars = []

    ard_params = np.atleast_2d(kernel.input_sensitivity(summarize=False))
    bottom = 0
    last_bottom = bottom

    x = np.arange(kernel.input_dim)

    if filtering is None:
        filtering = kernel.parameter_names(recursive=False)

    for i in range(ard_params.shape[0]):
        if kernel.parameters[i].name in filtering:
            c = Tango.nextMedium()
            bars.append(plot_bars(fig, ax, x, ard_params[i,:], c, kernel.parameters[i].name, bottom=bottom))
            last_bottom = ard_params[i,:]
            bottom += last_bottom
        else:
            print "filtering out {}".format(kernel.parameters[i].name)

    ax.set_xlim(-.5, kernel.input_dim - .5)
    add_bar_labels(fig, ax, [bars[-1]], bottom=bottom-last_bottom)

    if legend:
        if title is '':
            mode = 'expand'
            if len(bars) > 1:
                mode = 'expand'
            ax.legend(bbox_to_anchor=(0., 1.02, 1., 1.02), loc=3,
                      ncol=len(bars), mode=mode, borderaxespad=0.)
            fig.tight_layout(rect=(0, 0, 1, .9))
        else:
            ax.legend()
    return ax

def plot(kernel,x=None, fignum=None, ax=None, title=None, plot_limits=None, resolution=None, **mpl_kwargs):
    """
    plot a kernel.
    :param x: the value to use for the other kernel argument (kernels are a function of two variables!)
    :param title: the matplotlib title
    :param plot_limits: the range over which to plot the kernel
    :resolution: the resolution of the lines used in plotting
    :mpl_kwargs avalid keyword arguments to pass through to matplotlib (e.g. lw=7)
    """

        ax.set_title('%s kernel' % kernel.name)
    else:

    if kernel.input_dim == 1:
        if x is None:
            x = np.zeros((1, 1))
        else:
            x = np.asarray(x)
            assert x.size == 1, "The size of the fixed variable x is not 1"
            x = x.reshape((1, 1))

        if plot_limits == None:
            xmin, xmax = (x - 5).flatten(), (x + 5).flatten()
        elif len(plot_limits) == 2:
            xmin, xmax = plot_limits
        else:
            raise ValueError, "Bad limits for plotting"

        Xnew = np.linspace(xmin, xmax, resolution or 201)[:, None]
        Kx = kernel.K(Xnew, x)
        ax.plot(Xnew, Kx, **mpl_kwargs)
        ax.set_xlabel("x")
        ax.set_ylabel("k(x,%0.1f)" % x)

    elif kernel.input_dim == 2:
            x = np.zeros((1, 2))
        else:
            assert x.size == 2, "The size of the fixed variable x is not 2"
            x = x.reshape((1, 2))

        if plot_limits is None:
        else:

        resolution = resolution or 51
        xx, yy = np.mgrid[xmin[0]:xmax[0]:1j * resolution, xmin[1]:xmax[1]:1j * resolution]
        Xnew = np.vstack((xx.flatten(), yy.flatten())).T
        Kx = Kx.reshape(resolution, resolution).T
        ax.contour(xx, yy, Kx, vmin=Kx.min(), vmax=Kx.max(), cmap=pb.cm.jet, **mpl_kwargs) # @UndefinedVariable
        ax.set_xlabel("x1")
        ax.set_ylabel("x2")
        ax.set_title("k(x1,x2 ; %0.1f,%0.1f)" % (x[0, 0], x[0, 1]))
    else:
        raise NotImplementedError, "Cannot plot a kernel with more than two input dimensions"

import GPy

class PriorTests(unittest.TestCase):
    def test_lognormal(self):
        xmin, xmax = 1, 2.5*np.pi
        b, C, SNR = 1, 0, 0.1
        X = np.linspace(xmin, xmax, 500)
        y  = b*X + C + 1*np.sin(X)
        y += 0.05*np.random.randn(len(X))
        X, y = X[:, None], y[:, None]
        m = GPy.models.GPRegression(X, y)
        lognormal = GPy.priors.LogGaussian(1, 2)
        m.rbf.set_prior(lognormal)

    def test_Gamma(self):
        Gamma = GPy.priors.Gamma(1, 1)
        m.rbf.set_prior(Gamma)

    def test_incompatibility(self):
        gaussian = GPy.priors.Gaussian(1, 1)
        # setting a Gaussian prior on non-negative parameters
        # should raise an assertionerror.
        self.assertRaises(AssertionError, m.rbf.set_prior, gaussian)

    def test_set_prior(self):

        #m.rbf.set_prior(gaussian)

    def test_set_gaussian_for_reals(self):
        m = GPy.models.SparseGPRegression(X, y)

        m.Z.set_prior(gaussian)
        #self.assertRaises(AssertionError, m.Z.set_prior, gaussian)

    def test_fixed_domain_check(self):

        m.rbf.fix()

    def test_fixed_domain_check1(self):

        m.kern.lengthscale.fix()

try:
    from matplotlib.patches import Polygon
    from matplotlib.collections import PatchCollection
    #from matplotlib import cm
    pb.ion()
except:
    pass
import re

def plot(shape_records,facecolor='w',edgecolor='k',linewidths=.5, ax=None,xlims=None,ylims=None):
    """
    Plot the geometry of a shapefile

    :param shape_records: geometry and attributes list
    :type shape_records: ShapeRecord object (output of a shapeRecords() method)
    :param facecolor: color to be used to fill in polygons
    :param edgecolor: color to be used for lines
    """
    #Axes handle
        fig     = pb.figure()
        ax      = fig.add_subplot(111)

    #Iterate over shape_records
    for srec in shape_records:
        points = np.vstack(srec.shape.points)
        sparts = srec.shape.parts
        par = list(sparts) + [points.shape[0]]

        polygs = []
        for pj in xrange(len(sparts)):
            polygs.append(Polygon(points[par[pj]:par[pj+1]]))
        ax.add_collection(PatchCollection(polygs,facecolor=facecolor,edgecolor=edgecolor, linewidths=linewidths))

    #Plot limits
    _box = np.vstack([srec.shape.bbox for srec in shape_records])
    minx,miny = np.min(_box[:,:2],0)
    maxx,maxy = np.max(_box[:,2:],0)

    if xlims is not None:
        minx,maxx = xlims
    if ylims is not None:
        miny,maxy = ylims
    ax.set_xlim(minx,maxx)
    ax.set_ylim(miny,maxy)

def string_match(sf,regex,field=2):
    """
    Return the geometry and attributes of a shapefile whose fields match a regular expression given

    :param sf: shapefile
    :type sf: shapefile object
    :regex: regular expression to match
    :type regex: string
    :field: field number to be matched with the regex
    :type field: integer
    """
    index = []
    shape_records = []
    for rec in enumerate(sf.shapeRecords()):
        m = re.search(regex,rec[1].record[field])
        if m is not None:
            index.append(rec[0])
            shape_records.append(rec[1])
    return index,shape_records

def bbox_match(sf,bbox,inside_only=True):
    """
    Return the geometry and attributes of a shapefile that lie within (or intersect) a bounding box

    :param bbox: bounding box
    :type bbox: list of floats [x_min,y_min,x_max,y_max]
    :inside_only: True if the objects returned are those that lie within the bbox and False if the objects returned are any that intersect the bbox
    :type inside_only: Boolean
    """
    A,B,C,D = bbox
    index = []
        a,b,c,d = rec[1].shape.bbox
        if inside_only:
            if A <= a and B <= b and C >= c and D >= d:
                index.append(rec[0])
                shape_records.append(rec[1])
        else:
            cond1 = A <= a and B <= b and C >= a and D >= b
            cond2 = A <= c and B <= d and C >= c and D >= d
            cond3 = A <= a and D >= d and C >= a and B <= d
            cond4 = A <= c and D >= b and C >= c and B <= b
            cond5 = a <= C and b <= B and d >= D
            cond6 = c <= A and b <= B and d >= D
            cond7 = d <= B and a <= A and c >= C
            cond8 = b <= D and a <= A and c >= C
            if cond1 or cond2 or cond3 or cond4 or cond5 or cond6 or cond7 or cond8:

def plot_bbox(sf,bbox,inside_only=True):
    """
    Plot the geometry of a shapefile within a bbox

    """
    index,shape_records = bbox_match(sf,bbox,inside_only)
    plot(shape_records,xlims=[bbox[0],bbox[2]],ylims=[bbox[1],bbox[3]])

def plot_string_match(sf,regex,field,**kwargs):
    """
    Plot the geometry of a shapefile whose fields match a regular expression given

    """
    index,shape_records = string_match(sf,regex,field)
    plot(shape_records,**kwargs)

def new_shape_string(sf,name,regex,field=2,type=None):
    import shapefile
    if type is None:
        type = shapefile.POINT
    newshp = shapefile.Writer(shapeType = sf.shapeType)
    newshp.autoBalance = 1

    _fi = [sf.fields[j] for j in index]
    for f in _fi:
        newshp.field(name=f[0],fieldType=f[1],size=f[2],decimal=f[3])

    _shre = shape_records
    for sr in _shre:
        _points = []
        _parts = []
        for point in sr.shape.points:
            _points.append(point)
        _parts.append(_points)

        newshp.line(parts=_parts)
        newshp.records.append(sr.record)
        print len(sr.record)

    newshp.save(name)
    print index

def apply_bbox(sf,ax):
    """
    Use bbox as xlim and ylim in ax
    """
    limits = sf.bbox
    xlim = limits[0],limits[2]
    ylim = limits[1],limits[3]
    ax.set_xlim(xlim)
    ax.set_ylim(ylim)

"""
The module for some general debug tools
"""

def checkFinite(arr, name=None):
    if name is None:
        name = 'Array with ID['+str(id(arr))+']'

    if np.any(np.logical_not(np.isfinite(arr))):
        idx = np.where(np.logical_not(np.isfinite(arr)))[0]
        print name+' at indices '+str(idx)+' have not finite values: '+str(arr[idx])+'!'

def checkFullRank(m, tol=1e-10, name=None, force_check=False):
        name = 'Matrix with ID['+str(id(m))+']'
    assert len(m.shape)==2 and m.shape[0]==m.shape[1], 'The input of checkFullRank has to be a square matrix!'

    if not force_check and m.shape[0]>=10000:
        print 'The size of '+name+'is too big to check (>=10000)!'

    s = np.real(np.linalg.eigvals(m))

    if s.min()/s.max()<tol:
        print name+' is close to singlar!'
        print 'The eigen values of '+name+' is '+str(s)

'''
Created on Aug 27, 2014

@author: t-mazwie
'''

class Norm(object):
        pass
    def scale_by(self, Y):
        """
        Use data matrix Y as normalization space to work in.
        """
    def normalize(self, Y):
        """
        Project Y into normalized space
        """
    def inverse_mean(self, X):
        """
        Project the normalized object X into space of Y
        """
    def inverse_variance(self, var):
        return var
    def scaled(self):
        """
        Whether this Norm object has been initialized.
        """
class MeanNorm(Norm):
        self.mean = None
        Y = np.ma.masked_invalid(Y, copy=False)
        self.mean = Y.mean(0).view(np.ndarray)
        return Y-self.mean
        return X+self.mean
        return self.mean is not None

#Only works for scipy 0.12+
try:
    from scipy.special import erfcx, erf
    from scipy.special import erf
    from erfcx import erfcx

def ln_diff_erfs(x1, x2, return_sign=False):
    """Function for stably computing the log of difference of two erfs in a numerically stable manner.
    :param x1 : argument of the positive erf
    :type x1: ndarray
    :param x2 : argument of the negative erf
    :type x2: ndarray
    :return: tuple containing (log(abs(erf(x1) - erf(x2))), sign(erf(x1) - erf(x2)))
    
    Based on MATLAB code that was written by Antti Honkela and modified by David Luengo and originally derived from code by Neil Lawrence.
    """
    x1 = np.require(x1).real
    x2 = np.require(x2).real
    if x1.size==1:
        x1 = np.reshape(x1, (1, 1))
    if x2.size==1:
        x2 = np.reshape(x2, (1, 1))
        
    if x1.shape==x2.shape:
        v = np.zeros_like(x1)
    else:
        if x1.size==1:
            v = np.zeros(x2.shape)
        elif x2.size==1:
            v = np.zeros(x1.shape)
        else:
            raise ValueError, "This function does not broadcast unless provided with a scalar."
    
    if x1.size == 1:
        x1 = np.tile(x1, x2.shape)

    if x2.size == 1:
        x2 = np.tile(x2, x1.shape)

    sign = np.sign(x1 - x2)
        if sign== -1:
            swap = x1
            x1 = x2
            x2 = swap
    else:
        I = sign == -1
        swap = x1[I]
        x1[I] = x2[I]
        x2[I] = swap

    with np.errstate(divide='ignore'):
        # switch off log of zero warnings.

        # Case 0: arguments of different sign, no problems with loss of accuracy
        I0 = np.logical_or(np.logical_and(x1>0, x2<0), np.logical_and(x2>0, x1<0)) # I1=(x1*x2)<0

        # Case 1: x1 = x2 so we have log of zero.
        I1 = (x1 == x2)

        # Case 2: Both arguments are non-negative
        I2 = np.logical_and(x1 > 0, np.logical_and(np.logical_not(I0),
                                                   np.logical_not(I1)))
        # Case 3: Both arguments are non-positive
        I3 = np.logical_and(np.logical_and(np.logical_not(I0),
                                           np.logical_not(I1)),
                            np.logical_not(I2))
        _x2 = x2.flatten()
        _x1 = x1.flatten()
        for group, flags in zip((0, 1, 2, 3), (I0, I1, I2, I3)):

            if np.any(flags):
                if not x1.size==1:
                    _x1 = x1[flags]
                if not x2.size==1:
                    _x2 = x2[flags]
                if group==0:
                    v[flags] = np.log( erf(_x1) - erf(_x2) )
                elif group==1:
                    v[flags] = -np.inf
                elif group==2:
                    v[flags] = np.log(erfcx(_x2)
                                   -erfcx(_x1)*np.exp(_x2**2
                                                      -_x1**2)) - _x2**2
                elif group==3:
                    v[flags] = np.log(erfcx(-_x1)
                                   -erfcx(-_x2)*np.exp(_x1**2
                                                          -_x2**2))-_x1**2
        
    # TODO: switch back on log of zero warnings.

    if return_sign:
        return v, sign
    else:
        if v.size==1:
            if sign==-1:
                v = v.view('complex64')
                v += np.pi*1j
        else:
            # Need to add in a complex part because argument is negative.
            v = v.view('complex64')
            v[I] += np.pi*1j

    return v

'''
Created on 6 Nov 2013

'''

from parameterized import Parameterized
from param import Param
from transformations import Logexp, Logistic,__fixed__
from GPy.util.misc import param_to_array
from GPy.util.caching import Cache_this

class VariationalPrior(Parameterized):
    def __init__(self, name='latent space', **kw):
        super(VariationalPrior, self).__init__(name=name, **kw)

    def KL_divergence(self, variational_posterior):
        raise NotImplementedError, "override this for variational inference of latent space"

    def update_gradients_KL(self, variational_posterior):
        """
        updates the gradients for mean and variance **in place**
        """

class NormalPrior(VariationalPrior):
        var_mean = np.square(variational_posterior.mean).sum()
        var_S = (variational_posterior.variance - np.log(variational_posterior.variance)).sum()
        return 0.5 * (var_mean + var_S) - 0.5 * variational_posterior.input_dim * variational_posterior.num_data

        # dL:
        variational_posterior.mean.gradient -= variational_posterior.mean
        variational_posterior.variance.gradient -= (1. - (1. / (variational_posterior.variance))) * 0.5

class SpikeAndSlabPrior(VariationalPrior):
    def __init__(self, pi=None, learnPi=False, variance = 1.0, name='SpikeAndSlabPrior', **kw):
        super(SpikeAndSlabPrior, self).__init__(name=name, **kw)        
        self.variance = Param('variance',variance)
        self.learnPi = learnPi
        if learnPi:
            self.pi = Param('Pi', pi, Logistic(1e-10,1.-1e-10))
        else:
            self.pi = Param('Pi', pi, __fixed__)
        self.link_parameter(self.pi)

        gamma,gamma1 = variational_posterior.gamma_probabilities()
        if len(self.pi.shape)==2:
            idx = np.unique(gamma._raveled_index()/gamma.shape[-1])
            pi = self.pi[idx]
        else:
            pi = self.pi
            
        var_mean = np.square(mu)/self.variance
        var_S = (S/self.variance - np.log(S))
        var_gamma = (gamma*(log_gamma-np.log(pi))).sum()+(gamma1*(log_gamma1-np.log(1-pi))).sum()
        return var_gamma+ (gamma* (np.log(self.variance)-1. +var_mean + var_S)).sum()/2.

        else:

        variational_posterior.binary_prob.gradient -= (np.log((1-pi)/pi)+log_gamma-log_gamma1+((np.square(mu)+S)/self.variance-np.log(S)+np.log(self.variance)-1.)/2.)*gamma*gamma1
        mu.gradient -= gamma*mu/self.variance
        S.gradient -= (1./self.variance - 1./S) * gamma /2.
        if self.learnPi:
            if len(self.pi)==1:
                self.pi.gradient = (gamma/self.pi - (1.-gamma)/(1.-self.pi)).sum()
            elif len(self.pi.shape)==1:
                self.pi.gradient = (gamma/self.pi - (1.-gamma)/(1.-self.pi)).sum(axis=0)
            else:
                self.pi[idx].gradient = (gamma/self.pi[idx] - (1.-gamma)/(1.-self.pi[idx]))

class VariationalPosterior(Parameterized):
    def __init__(self, means=None, variances=None, name='latent space', *a, **kw):
        super(VariationalPosterior, self).__init__(name=name, *a, **kw)
        self.mean = Param("mean", means)
        self.variance = Param("variance", variances, Logexp())
        self.ndim = self.mean.ndim
        self.shape = self.mean.shape
        self.num_data, self.input_dim = self.mean.shape
        self.link_parameters(self.mean, self.variance)
        if self.has_uncertain_inputs():
            assert self.variance.shape == self.mean.shape, "need one variance per sample and dimenion"

    def set_gradients(self, grad):
        self.mean.gradient, self.variance.gradient = grad

    def _raveled_index(self):
        index = np.empty(dtype=int, shape=0)
        size = 0
        for p in self.parameters:
            index = np.hstack((index, p._raveled_index()+size))
            size += p._realsize_ if hasattr(p, '_realsize_') else p.size

    def has_uncertain_inputs(self):
        return not self.variance is None

    def __getitem__(self, s):
        if isinstance(s, (int, slice, tuple, list, np.ndarray)):
            import copy
            n = self.__new__(self.__class__, self.name)
            dc = self.__dict__.copy()
            dc['mean'] = self.mean[s]
            dc['variance'] = self.variance[s]
            dc['parameters'] = copy.copy(self.parameters)
            n.__dict__.update(dc)
            n.parameters[dc['mean']._parent_index_] = dc['mean']
            n.parameters[dc['variance']._parent_index_] = dc['variance']
            n._gradient_array_ = None
            oversize = self.size - self.mean.size - self.variance.size
            n.size = n.mean.size + n.variance.size + oversize
            n.ndim = n.mean.ndim
            n.shape = n.mean.shape
            n.num_data = n.mean.shape[0]
            n.input_dim = n.mean.shape[1] if n.ndim != 1 else 1
            return n
        else:
            return super(VariationalPosterior, self).__getitem__(s)

class NormalPosterior(VariationalPosterior):
    '''
    NormalPosterior distribution for variational approximations.

    holds the means and variances for a factorizing multivariate normal distribution
    '''

    def plot(self, *args):
        """
        Plot latent space X in 1D:

        See  GPy.plotting.matplot_dep.variational_plots
        """
        import sys
        from ...plotting.matplot_dep import variational_plots
        import matplotlib
        return variational_plots.plot(self,*args)

class SpikeAndSlabPosterior(VariationalPosterior):
    '''
    The SpikeAndSlab distribution for variational approximations.
    '''
    def __init__(self, means, variances, binary_prob, name='latent space'):
        """
        binary_prob : the probability of the distribution on the slab part.
        """
        super(SpikeAndSlabPosterior, self).__init__(means, variances, name)
        self.gamma = Param("binary_prob",binary_prob)
        self.link_parameter(self.gamma)
        
    @Cache_this(limit=5)
    def gamma_probabilities(self):
        prob = np.zeros_like(param_to_array(self.gamma))
        prob[self.gamma>-710] = 1./(1.+np.exp(-self.gamma[self.gamma>-710]))
        prob1 = -np.zeros_like(param_to_array(self.gamma))
        prob1[self.gamma<710] = 1./(1.+np.exp(self.gamma[self.gamma<710]))
        return prob, prob1
    
    def gamma_log_prob(self):
        loggamma = param_to_array(self.gamma).copy()
        loggamma[loggamma>-40] = -np.log1p(np.exp(-loggamma[loggamma>-40]))
        loggamma1 = -param_to_array(self.gamma).copy()
        loggamma1[loggamma1>-40] = -np.log1p(np.exp(-loggamma1[loggamma1>-40]))
        return loggamma,loggamma1

        self.mean.gradient, self.variance.gradient, self.gamma.gradient = grad

            dc['binary_prob'] = self.binary_prob[s]
            n.parameters[dc['binary_prob']._parent_index_] = dc['binary_prob']
            return n
        else:
            return super(VariationalPrior, self).__getitem__(s)

    def plot(self, *args, **kwargs):
        """

        """
        import sys
        return variational_plots.plot_SpikeSlab(self,*args, **kwargs)

from __future__ import division
# Copyright (c) 2012-2014 Ricardo Andrade, Alan Saul

class Poisson(Likelihood):
    """
    Poisson likelihood

    .. math::
        p(y_{i}|\\lambda(f_{i})) = \\frac{\\lambda(f_{i})^{y_{i}}}{y_{i}!}e^{-\\lambda(f_{i})}

    .. Note::
        Y is expected to take values in {0,1,2,...}
    """
    def __init__(self, gp_link=None):

        super(Poisson, self).__init__(gp_link, name='Poisson')

        """
        the expected value of y given a value of f
        """
        return self.gp_link.transf(gp)

        """

        .. math::
            p(y_{i}|\\lambda(f_{i})) = \\frac{\\lambda(f_{i})^{y_{i}}}{y_{i}!}e^{-\\lambda(f_{i})}

        :param Y_metadata: Y_metadata which is not used in poisson distribution
        """
        return np.prod(stats.poisson.pmf(y,link_f))

        """

        .. math::
            \\ln p(y_{i}|\lambda(f_{i})) = -\\lambda(f_{i}) + y_{i}\\log \\lambda(f_{i}) - \\log y_{i}!

        """
        return np.sum(-link_f + y*np.log(link_f) - special.gammaln(y+1))

        """

        .. math::
            \\frac{d \\ln p(y_{i}|\lambda(f_{i}))}{d\\lambda(f)} = \\frac{y_{i}}{\\lambda(f_{i})} - 1

        """
        return y/link_f - 1

        """

        .. math::
            \\frac{d^{2} \\ln p(y_{i}|\lambda(f_{i}))}{d^{2}\\lambda(f)} = \\frac{-y_{i}}{\\lambda(f_{i})^{2}}

        .. Note::
        """
        hess = -y/(link_f**2)
        #d2_df = self.gp_link.d2transf_df2(gp)
        #transf = self.gp_link.transf(gp)
        #return obs * ((self.gp_link.dtransf_df(gp)/transf)**2 - d2_df/transf) + d2_df

        """

        .. math::
            \\frac{d^{3} \\ln p(y_{i}|\lambda(f_{i}))}{d^{3}\\lambda(f)} = \\frac{2y_{i}}{\\lambda(f_{i})^{3}}

        """
        d3lik_dlink3 = 2*y/(link_f)**3

    def conditional_mean(self,gp):
        """
        """

    def conditional_variance(self,gp):
        """
        """

        """

        """
        Ysim = np.random.poisson(self.gp_link.transf(gp))

from scipy import stats, integrate
from scipy.special import gammaln, gamma

class StudentT(Likelihood):
    """
    Student T likelihood

    For nomanclature see Bayesian Data Analysis 2003 p576

    .. math::
        p(y_{i}|\\lambda(f_{i})) = \\frac{\\Gamma\\left(\\frac{v+1}{2}\\right)}{\\Gamma\\left(\\frac{v}{2}\\right)\\sqrt{v\\pi\\sigma^{2}}}\\left(1 + \\frac{1}{v}\\left(\\frac{(y_{i} - f_{i})^{2}}{\\sigma^{2}}\\right)\\right)^{\\frac{-v+1}{2}}

    """
    def __init__(self,gp_link=None, deg_free=5, sigma2=2):
            gp_link = link_functions.Identity()

        super(StudentT, self).__init__(gp_link, name='Student_T')
        # sigma2 is not a noise parameter, it is a squared scale.
        self.sigma2 = Param('t_scale2', float(sigma2), Logexp())
        self.v = Param('deg_free', float(deg_free))
        self.link_parameter(self.sigma2)
        self.link_parameter(self.v)
        self.v.constrain_fixed()

        self.variance = (self.v / float(self.v - 2)) * self.sigma2

    def update_gradients(self, grads):
        """
        Pull out the gradients, be careful as the order must match the order
        in which the parameters are added
        """
        self.sigma2.gradient = grads[0]
        self.v.gradient = grads[1]

        """

        .. math::
            p(y_{i}|\\lambda(f_{i})) = \\frac{\\Gamma\\left(\\frac{v+1}{2}\\right)}{\\Gamma\\left(\\frac{v}{2}\\right)\\sqrt{v\\pi\\sigma^{2}}}\\left(1 + \\frac{1}{v}\\left(\\frac{(y_{i} - \\lambda(f_{i}))^{2}}{\\sigma^{2}}\\right)\\right)^{\\frac{-v+1}{2}}

        :param inv_link_f: latent variables link(f)
        :type inv_link_f: Nx1 array
        :param Y_metadata: Y_metadata which is not used in student t distribution
        """
        assert np.atleast_1d(inv_link_f).shape == np.atleast_1d(y).shape
        e = y - inv_link_f
        #Careful gamma(big_number) is infinity!
        objective = ((np.exp(gammaln((self.v + 1)*0.5) - gammaln(self.v * 0.5))
                     / (np.sqrt(self.v * np.pi * self.sigma2)))
                     * ((1 + (1./float(self.v))*((e**2)/float(self.sigma2)))**(-0.5*(self.v + 1)))
                    )
        return np.prod(objective)

        """

        .. math::
            \\ln p(y_{i}|\lambda(f_{i})) = \\ln \\Gamma\\left(\\frac{v+1}{2}\\right) - \\ln \\Gamma\\left(\\frac{v}{2}\\right) - \\ln \\sqrt{v \\pi\\sigma^{2}} - \\frac{v+1}{2}\\ln \\left(1 + \\frac{1}{v}\\left(\\frac{(y_{i} - \lambda(f_{i}))^{2}}{\\sigma^{2}}\\right)\\right)

        :param inv_link_f: latent variables (link(f))

        """
        #FIXME:
        #Why does np.log(1 + (1/self.v)*((y-inv_link_f)**2)/self.sigma2) suppress the divide by zero?!
        #But np.log(1 + (1/float(self.v))*((y-inv_link_f)**2)/self.sigma2) throws it correctly
        #print - 0.5*(self.v + 1)*np.log(1 + (1/np.float(self.v))*((e**2)/self.sigma2))
        objective = (+ gammaln((self.v + 1) * 0.5)
                    - gammaln(self.v * 0.5)
                    - 0.5*np.log(self.sigma2 * self.v * np.pi)
                    - 0.5*(self.v + 1)*np.log(1 + (1/np.float(self.v))*((e**2)/self.sigma2))
                    )
        return np.sum(objective)

        """

        .. math::
            \\frac{d \\ln p(y_{i}|\lambda(f_{i}))}{d\\lambda(f)} = \\frac{(v+1)(y_{i}-\lambda(f_{i}))}{(y_{i}-\lambda(f_{i}))^{2} + \\sigma^{2}v}

        :param inv_link_f: latent variables (f)

        """
        grad = ((self.v + 1) * e) / (self.v * self.sigma2 + (e**2))

        """

        .. math::
            \\frac{d^{2} \\ln p(y_{i}|\lambda(f_{i}))}{d^{2}\\lambda(f)} = \\frac{(v+1)((y_{i}-\lambda(f_{i}))^{2} - \\sigma^{2}v)}{((y_{i}-\lambda(f_{i}))^{2} + \\sigma^{2}v)^{2}}

        :param inv_link_f: latent variables inv_link(f)

        .. Note::
        """
        hess = ((self.v + 1)*(e**2 - self.v*self.sigma2)) / ((self.sigma2*self.v + e**2)**2)

        """

        .. math::
            \\frac{d^{3} \\ln p(y_{i}|\lambda(f_{i}))}{d^{3}\\lambda(f)} = \\frac{-2(v+1)((y_{i} - \lambda(f_{i}))^3 - 3(y_{i} - \lambda(f_{i})) \\sigma^{2} v))}{((y_{i} - \lambda(f_{i})) + \\sigma^{2} v)^3}

        """
        d3lik_dlink3 = ( -(2*(self.v + 1)*(-e)*(e**2 - 3*self.v*self.sigma2)) /
                       ((e**2 + self.sigma2*self.v)**3)
                    )

    def dlogpdf_link_dvar(self, inv_link_f, y, Y_metadata=None):
        """
        Gradient of the log-likelihood function at y given f, w.r.t variance parameter (t_noise)

        .. math::
            \\frac{d \\ln p(y_{i}|\lambda(f_{i}))}{d\\sigma^{2}} = \\frac{v((y_{i} - \lambda(f_{i}))^{2} - \\sigma^{2})}{2\\sigma^{2}(\\sigma^{2}v + (y_{i} - \lambda(f_{i}))^{2})}

        :returns: derivative of likelihood evaluated at points f w.r.t variance parameter
        """
        dlogpdf_dvar = self.v*(e**2 - self.sigma2)/(2*self.sigma2*(self.sigma2*self.v + e**2))
        return np.sum(dlogpdf_dvar)

    def dlogpdf_dlink_dvar(self, inv_link_f, y, Y_metadata=None):
        """
        Derivative of the dlogpdf_dlink w.r.t variance parameter (t_noise)

        .. math::
            \\frac{d}{d\\sigma^{2}}(\\frac{d \\ln p(y_{i}|\lambda(f_{i}))}{df}) = \\frac{-2\\sigma v(v + 1)(y_{i}-\lambda(f_{i}))}{(y_{i}-\lambda(f_{i}))^2 + \\sigma^2 v)^2}

        :param inv_link_f: latent variables inv_link_f
        """
        dlogpdf_dlink_dvar = (self.v*(self.v+1)*(-e))/((self.sigma2*self.v + e**2)**2)
        return dlogpdf_dlink_dvar

    def d2logpdf_dlink2_dvar(self, inv_link_f, y, Y_metadata=None):
        """
        Gradient of the hessian (d2logpdf_dlink2) w.r.t variance parameter (t_noise)

        .. math::
            \\frac{d}{d\\sigma^{2}}(\\frac{d^{2} \\ln p(y_{i}|\lambda(f_{i}))}{d^{2}f}) = \\frac{v(v+1)(\\sigma^{2}v - 3(y_{i} - \lambda(f_{i}))^{2})}{(\\sigma^{2}v + (y_{i} - \lambda(f_{i}))^{2})^{3}}

        :returns: derivative of hessian evaluated at points f and f_j w.r.t variance parameter
        """
        d2logpdf_dlink2_dvar = ( (self.v*(self.v+1)*(self.sigma2*self.v - 3*(e**2)))
                              / ((self.sigma2*self.v + (e**2))**3)
                           )
        return d2logpdf_dlink2_dvar

    def dlogpdf_link_dtheta(self, f, y, Y_metadata=None):
        dlogpdf_dvar = self.dlogpdf_link_dvar(f, y, Y_metadata=Y_metadata)
        dlogpdf_dv = np.zeros_like(dlogpdf_dvar) #FIXME: Not done yet
        return np.hstack((dlogpdf_dvar, dlogpdf_dv))

    def dlogpdf_dlink_dtheta(self, f, y, Y_metadata=None):
        dlogpdf_dlink_dvar = self.dlogpdf_dlink_dvar(f, y, Y_metadata=Y_metadata)
        dlogpdf_dlink_dv = np.zeros_like(dlogpdf_dlink_dvar) #FIXME: Not done yet
        return np.hstack((dlogpdf_dlink_dvar, dlogpdf_dlink_dv))

    def d2logpdf_dlink2_dtheta(self, f, y, Y_metadata=None):
        d2logpdf_dlink2_dvar = self.d2logpdf_dlink2_dvar(f, y, Y_metadata=Y_metadata)
        d2logpdf_dlink2_dv = np.zeros_like(d2logpdf_dlink2_dvar) #FIXME: Not done yet
        return np.hstack((d2logpdf_dlink2_dvar, d2logpdf_dlink2_dv))

    def predictive_mean(self, mu, sigma, Y_metadata=None):
        # The comment here confuses mean and median. 
        return self.gp_link.transf(mu) # only true if link is monotonic, which it is.

        if self.deg_free<=2.:
            return np.empty(mu.shape)*np.nan # does not exist for degrees of freedom <= 2.
        else:
            return super(StudentT, self).predictive_variance(mu, variance, predictive_mean, Y_metadata)

        return self.deg_free/(self.deg_free - 2.)

        """

        """
        #FIXME: Very slow as we are computing a new random variable per input!
        #Can't get it to sample all at the same time
        #student_t_samples = np.array([stats.t.rvs(self.v, self.gp_link.transf(gpj),scale=np.sqrt(self.sigma2), size=1) for gpj in gp])
        dfs = np.ones_like(gp)*self.v
        scales = np.ones_like(gp)*np.sqrt(self.sigma2)
        student_t_samples = stats.t.rvs(dfs, loc=self.gp_link.transf(gp),
                                        scale=scales)
        return student_t_samples.reshape(orig_shape)

import GPy

class MiscTests(unittest.TestCase):
        self.N = 20
        self.N_new = 50
        self.D = 1
        self.X = np.random.uniform(-3., 3., (self.N, 1))
        self.Y = np.sin(self.X) + np.random.randn(self.N, self.D) * 0.05
        self.X_new = np.random.uniform(-3., 3., (self.N_new, 1))

    def test_raw_predict(self):
        k = GPy.kern.RBF(1)
        m = GPy.models.GPRegression(self.X, self.Y, kernel=k)
        m.likelihood.variance = .5
        Kinv = np.linalg.pinv(k.K(self.X) + np.eye(self.N) * m.likelihood.variance)
        K_hat = k.K(self.X_new) - k.K(self.X_new, self.X).dot(Kinv).dot(k.K(self.X, self.X_new))
        mu_hat = k.K(self.X_new, self.X).dot(Kinv).dot(m.Y_normalized)

        mu, covar = m._raw_predict(self.X_new, full_cov=True)
        self.assertEquals(mu.shape, (self.N_new, self.D))
        self.assertEquals(covar.shape, (self.N_new, self.N_new))
        np.testing.assert_almost_equal(K_hat, covar)
        np.testing.assert_almost_equal(mu_hat, mu)

        mu, var = m._raw_predict(self.X_new)
        self.assertEquals(var.shape, (self.N_new, 1))
        np.testing.assert_almost_equal(np.diag(K_hat)[:, None], var)

    def test_sparse_raw_predict(self):
        m = GPy.models.SparseGPRegression(self.X, self.Y, kernel=k)
        Z = m.Z[:]
        X = self.X[:]

        # Not easy to check if woodbury_inv is correct in itself as it requires a large derivation and expression
        Kinv = m.posterior.woodbury_inv
        K_hat = k.K(self.X_new) - k.K(self.X_new, Z).dot(Kinv).dot(k.K(Z, self.X_new))

        # np.testing.assert_almost_equal(mu_hat, mu)

    def test_likelihood_replicate(self):
        m = GPy.models.GPRegression(self.X, self.Y)
        m2 = GPy.models.GPRegression(self.X, self.Y)
        np.testing.assert_equal(m.log_likelihood(), m2.log_likelihood())
        m2[:] = m[''].values()
        np.testing.assert_almost_equal(m.log_likelihood(), m2.log_likelihood())
        m2[''] = m[:]
        m2[:] = m[:]
        m2[''] = m['']

        m.kern.lengthscale.randomize()

        m.Gaussian_noise.randomize()

        m['.*var'] = 2
        m2['.*var'] = m['.*var']

    def test_likelihood_set(self):

        m2.kern.lengthscale = m.kern.lengthscale

        m2['.*lengthscale'] = m.kern.lengthscale

        m2['.*lengthscale'] = m.kern['.*lengthscale']

        m2.kern.lengthscale = m.kern['.*lengthscale']

    def test_missing_data(self):
        from GPy import kern
        from GPy.models.bayesian_gplvm_minibatch import BayesianGPLVMMiniBatch
        from GPy.examples.dimensionality_reduction import _simulate_matern

        D1, D2, D3, N, num_inducing, Q = 13, 5, 8, 400, 3, 4
        _, _, Ylist = _simulate_matern(D1, D2, D3, N, num_inducing, False)
        Y = Ylist[0]

        inan = np.random.binomial(1, .9, size=Y.shape).astype(bool) # 80% missing data
        Ymissing = Y.copy()
        Ymissing[inan] = np.nan

        k = kern.Linear(Q, ARD=True) + kern.White(Q, np.exp(-2)) # + kern.bias(Q)
        m = BayesianGPLVMMiniBatch(Ymissing, Q, init="random", num_inducing=num_inducing,
                          kernel=k, missing_data=True)
        assert(m.checkgrad())

        k = kern.RBF(Q, ARD=True) + kern.White(Q, np.exp(-2)) # + kern.bias(Q)

    def test_likelihood_replicate_kern(self):
        m.kern.randomize()
        m2.kern[''] = m.kern[:]
        m2.kern[:] = m.kern[:]
        m2.kern[''] = m.kern['']
        m2.kern[:] = m.kern[''].values()

    def test_big_model(self):
        m = GPy.examples.dimensionality_reduction.mrd_simulation(optimize=0, plot=0, plot_sim=0)
        m.X.fix()
        print m
        m.unfix()
        m.checkgrad()
        print m
        m.fix()
        print m
        m.inducing_inputs.unfix()
        print m
        m.unfix()
        print m

    def test_model_set_params(self):
        lengthscale = np.random.uniform()
        m.kern.lengthscale = lengthscale
        np.testing.assert_equal(m.kern.lengthscale, lengthscale)
        m.kern.lengthscale *= 1
        m['.*var'] -= .1
        m.optimize()
        print m

    def test_model_optimize(self):
        X = np.random.uniform(-3., 3., (20, 1))
        Y = np.sin(X) + np.random.randn(20, 1) * 0.05
        m = GPy.models.GPRegression(X, Y)
        print m

class GradientTests(np.testing.TestCase):

        self.X1D = np.random.uniform(-3., 3., (20, 1))
        self.Y1D = np.sin(self.X1D) + np.random.randn(20, 1) * 0.05

        self.X2D = np.random.uniform(-3., 3., (40, 2))
        self.Y2D = np.sin(self.X2D[:, 0:1]) * np.sin(self.X2D[:, 1:2]) + np.random.randn(40, 1) * 0.05

    def check_model(self, kern, model_type='GPRegression', dimension=1, uncertain_inputs=False):
        # Get the correct gradients
        if dimension == 1:
            X = self.X1D
            Y = self.Y1D
        else:
            X = self.X2D
            Y = self.Y2D
        # Get model type (GPRegression, SparseGPRegression, etc)
        model_fit = getattr(GPy.models, model_type)

        # noise = GPy.kern.White(dimension)
        kern = kern  #  + noise
            m = model_fit(X, Y, kernel=kern, X_variance=np.random.rand(X.shape[0], X.shape[1]))
        else:
            m = model_fit(X, Y, kernel=kern)
        # contrain all parameters to be positive

    def test_GPRegression_rbf_1d(self):
        ''' Testing the GP regression with rbf kernel with white kernel on 1d data '''
        rbf = GPy.kern.RBF(1)
        self.check_model(rbf, model_type='GPRegression', dimension=1)

    def test_GPRegression_rbf_2D(self):
        ''' Testing the GP regression with rbf kernel on 2d data '''
        rbf = GPy.kern.RBF(2)
        self.check_model(rbf, model_type='GPRegression', dimension=2)

    def test_GPRegression_rbf_ARD_2D(self):
        k = GPy.kern.RBF(2, ARD=True)
        self.check_model(k, model_type='GPRegression', dimension=2)

    def test_GPRegression_mlp_1d(self):
        ''' Testing the GP regression with mlp kernel with white kernel on 1d data '''
        mlp = GPy.kern.MLP(1)
        self.check_model(mlp, model_type='GPRegression', dimension=1)

    # TODO:
    # def test_GPRegression_poly_1d(self):
    #    ''' Testing the GP regression with polynomial kernel with white kernel on 1d data '''
    #    mlp = GPy.kern.Poly(1, degree=5)
    #    self.check_model(mlp, model_type='GPRegression', dimension=1)

    def test_GPRegression_matern52_1D(self):
        ''' Testing the GP regression with matern52 kernel on 1d data '''
        matern52 = GPy.kern.Matern52(1)
        self.check_model(matern52, model_type='GPRegression', dimension=1)

    def test_GPRegression_matern52_2D(self):
        ''' Testing the GP regression with matern52 kernel on 2d data '''
        matern52 = GPy.kern.Matern52(2)
        self.check_model(matern52, model_type='GPRegression', dimension=2)

    def test_GPRegression_matern52_ARD_2D(self):
        matern52 = GPy.kern.Matern52(2, ARD=True)

    def test_GPRegression_matern32_1D(self):
        ''' Testing the GP regression with matern32 kernel on 1d data '''
        matern32 = GPy.kern.Matern32(1)
        self.check_model(matern32, model_type='GPRegression', dimension=1)

    def test_GPRegression_matern32_2D(self):
        ''' Testing the GP regression with matern32 kernel on 2d data '''
        matern32 = GPy.kern.Matern32(2)
        self.check_model(matern32, model_type='GPRegression', dimension=2)

    def test_GPRegression_matern32_ARD_2D(self):
        matern32 = GPy.kern.Matern32(2, ARD=True)

    def test_GPRegression_exponential_1D(self):
        ''' Testing the GP regression with exponential kernel on 1d data '''
        exponential = GPy.kern.Exponential(1)
        self.check_model(exponential, model_type='GPRegression', dimension=1)

    def test_GPRegression_exponential_2D(self):
        ''' Testing the GP regression with exponential kernel on 2d data '''
        exponential = GPy.kern.Exponential(2)
        self.check_model(exponential, model_type='GPRegression', dimension=2)

    def test_GPRegression_exponential_ARD_2D(self):
        exponential = GPy.kern.Exponential(2, ARD=True)

    def test_GPRegression_bias_kern_1D(self):
        ''' Testing the GP regression with bias kernel on 1d data '''
        bias = GPy.kern.Bias(1)
        self.check_model(bias, model_type='GPRegression', dimension=1)

    def test_GPRegression_bias_kern_2D(self):
        ''' Testing the GP regression with bias kernel on 2d data '''
        bias = GPy.kern.Bias(2)
        self.check_model(bias, model_type='GPRegression', dimension=2)

    def test_GPRegression_linear_kern_1D_ARD(self):
        ''' Testing the GP regression with linear kernel on 1d data '''
        linear = GPy.kern.Linear(1, ARD=True)
        self.check_model(linear, model_type='GPRegression', dimension=1)

    def test_GPRegression_linear_kern_2D_ARD(self):
        ''' Testing the GP regression with linear kernel on 2d data '''
        linear = GPy.kern.Linear(2, ARD=True)
        self.check_model(linear, model_type='GPRegression', dimension=2)

    def test_GPRegression_linear_kern_1D(self):
        linear = GPy.kern.Linear(1)

    def test_GPRegression_linear_kern_2D(self):
        linear = GPy.kern.Linear(2)

    def test_SparseGPRegression_rbf_white_kern_1d(self):
        ''' Testing the sparse GP regression with rbf kernel with white kernel on 1d data '''
        self.check_model(rbf, model_type='SparseGPRegression', dimension=1)

    def test_SparseGPRegression_rbf_white_kern_2D(self):
        ''' Testing the sparse GP regression with rbf kernel on 2d data '''
        self.check_model(rbf, model_type='SparseGPRegression', dimension=2)

    def test_SparseGPRegression_rbf_linear_white_kern_1D(self):
        rbflin = GPy.kern.RBF(1) + GPy.kern.Linear(1)
        self.check_model(rbflin, model_type='SparseGPRegression', dimension=1)

    def test_SparseGPRegression_rbf_linear_white_kern_2D(self):
        rbflin = GPy.kern.RBF(2) + GPy.kern.Linear(2)
        self.check_model(rbflin, model_type='SparseGPRegression', dimension=2)

    # @unittest.expectedFailure
    def test_SparseGPRegression_rbf_linear_white_kern_2D_uncertain_inputs(self):
        ''' Testing the sparse GP regression with rbf, linear kernel on 2d data with uncertain inputs'''
        raise unittest.SkipTest("This is not implemented yet!")
        self.check_model(rbflin, model_type='SparseGPRegression', dimension=2, uncertain_inputs=1)

    def test_SparseGPRegression_rbf_linear_white_kern_1D_uncertain_inputs(self):
        ''' Testing the sparse GP regression with rbf, linear kernel on 1d data with uncertain inputs'''
        self.check_model(rbflin, model_type='SparseGPRegression', dimension=1, uncertain_inputs=1)

    def test_GPLVM_rbf_bias_white_kern_2D(self):
        """ Testing GPLVM with rbf + bias kernel """
        N, input_dim, D = 50, 1, 2
        X = np.random.rand(N, input_dim)
        k = GPy.kern.RBF(input_dim, 0.5, 0.9 * np.ones((1,))) + GPy.kern.Bias(input_dim, 0.1) + GPy.kern.White(input_dim, 0.05)
        K = k.K(X)
        Y = np.random.multivariate_normal(np.zeros(N), K, input_dim).T
        m = GPy.models.GPLVM(Y, input_dim, kernel=k)

    def test_GPLVM_rbf_linear_white_kern_2D(self):
        k = GPy.kern.Linear(input_dim) + GPy.kern.Bias(input_dim, 0.1) + GPy.kern.White(input_dim, 0.05)
        K = k.K(X)
        m = GPy.models.GPLVM(Y, input_dim, init='PCA', kernel=k)

    def test_GP_EP_probit(self):
        N = 20
        X = np.hstack([np.random.normal(5, 2, N / 2), np.random.normal(10, 2, N / 2)])[:, None]
        Y = np.hstack([np.ones(N / 2), np.zeros(N / 2)])[:, None]
        kernel = GPy.kern.RBF(1)
        m = GPy.models.GPClassification(X, Y, kernel=kernel)

    def test_sparse_EP_DTC_probit(self):
        N = 20
        Z = np.linspace(0, 15, 4)[:, None]
        m = GPy.models.SparseGPClassification(X, Y, kernel=kernel, Z=Z)
        # distribution = GPy.likelihoods.likelihood_functions.Bernoulli()
        # likelihood = GPy.likelihoods.EP(Y, distribution)
        # m = GPy.core.SparseGP(X, likelihood, kernel, Z)
        # m.ensure_default_constraints()

    @unittest.expectedFailure
    def test_generalized_FITC(self):
        N = 20
        X = np.hstack([np.random.rand(N / 2) + 1, np.random.rand(N / 2) - 1])[:, None]
        k = GPy.kern.RBF(1) + GPy.kern.White(1)
        m = GPy.models.FITCClassification(X, Y, kernel=k)
        m.update_likelihood_approximation()

    def test_multioutput_regression_1D(self):
        X1 = np.random.rand(50, 1) * 8
        X2 = np.random.rand(30, 1) * 5
        X = np.vstack((X1, X2))
        Y1 = np.sin(X1) + np.random.randn(*X1.shape) * 0.05
        Y2 = -np.sin(X2) + np.random.randn(*X2.shape) * 0.05
        Y = np.vstack((Y1, Y2))

        k1 = GPy.kern.RBF(1)
        m = GPy.models.GPMultioutputRegression(X_list=[X1, X2], Y_list=[Y1, Y2], kernel_list=[k1])
        m.constrain_fixed('.*rbf_var', 1.)

    def test_multioutput_sparse_regression_1D(self):
        X1 = np.random.rand(500, 1) * 8
        X2 = np.random.rand(300, 1) * 5

        m = GPy.models.SparseGPMultioutputRegression(X_list=[X1, X2], Y_list=[Y1, Y2], kernel_list=[k1])

    def test_gp_heteroscedastic_regression(self):
        num_obs = 25
        X = np.random.randint(0, 140, num_obs)
        X = X[:, None]
        Y = 25. + np.sin(X / 20.) * 2. + np.random.rand(num_obs)[:, None]
        kern = GPy.kern.Bias(1) + GPy.kern.RBF(1)
        m = GPy.models.GPHeteroscedasticRegression(X, Y, kern)

    def test_sparse_gp_heteroscedastic_regression(self):
        Y_metadata = {'output_index':np.arange(num_obs)[:,None]}
        noise_terms = np.unique(Y_metadata['output_index'].flatten())
        likelihoods_list = [GPy.likelihoods.Gaussian(name="Gaussian_noise_%s" %j) for j in noise_terms]
        likelihood = GPy.likelihoods.MixedNoise(likelihoods_list=likelihoods_list)
        m = GPy.core.SparseGP(X, Y, X[np.random.choice(num_obs, 10)],
                              kern, likelihood,
                              GPy.inference.latent_function_inference.VarDTC(),
                              Y_metadata=Y_metadata)

    def test_gp_kronecker_gaussian(self):
        N1, N2 = 30, 20
        X1 = np.random.randn(N1, 1)
        X2 = np.random.randn(N2, 1)
        X1.sort(0); X2.sort(0)
        k1 = GPy.kern.RBF(1)  # + GPy.kern.White(1)
        k2 = GPy.kern.RBF(1)  # + GPy.kern.White(1)
        Y = np.random.randn(N1, N2)
        Y = Y - Y.mean(0)
        Y = Y / Y.std(0)
        m = GPy.models.GPKroneckerGaussianRegression(X1, X2, Y, k1, k2)

        # build the model the dumb way
        assert (N1 * N2 < 1000), "too much data for standard GPs!"
        yy, xx = np.meshgrid(X2, X1)
        Xgrid = np.vstack((xx.flatten(order='F'), yy.flatten(order='F'))).T
        kg = GPy.kern.RBF(1, active_dims=[0]) * GPy.kern.RBF(1, active_dims=[1])
        mm = GPy.models.GPRegression(Xgrid, Y.reshape(-1, 1, order='F'), kernel=kg)

        mm[:] = m[:]
        assert np.allclose(m.log_likelihood(), mm.log_likelihood())
        assert np.allclose(m.gradient, mm.gradient)
        X1test = np.random.randn(100, 1)
        X2test = np.random.randn(100, 1)
        mean1, var1 = m.predict(X1test, X2test)
        yy, xx = np.meshgrid(X2test, X1test)
        mean2, var2 = mm.predict(Xgrid)
        assert np.allclose(mean1, mean2)
        assert np.allclose(var1, var2)

    def test_gp_VGPC(self):
        m = GPy.models.GPVariationalGaussianApproximation(X, Y, kern)

'''
Created on 10 Sep 2012

@author: Max Zwiessele
@copyright: Max Zwiessele 2012
'''
try:
    import pylab
    import matplotlib
except:
    pass
from operator import setitem

class PCA(object):
    """
    PCA module with automatic primal/dual determination.
    """
    def __init__(self, X):
        self.mu = None
        self.sigma = None

        X = self.center(X)

        # self.X = input
        if X.shape[0] >= X.shape[1]:
            # print "N >= D: using primal"
            self.eigvals, self.eigvectors = self._primal_eig(X)
        else:
            # print "N < D: using dual"
            self.eigvals, self.eigvectors = self._dual_eig(X)
        self.sort = numpy.argsort(self.eigvals)[::-1]
        self.eigvals = self.eigvals[self.sort]
        self.eigvectors = self.eigvectors[:, self.sort]
        self.fracs = self.eigvals / self.eigvals.sum()
        self.Q = self.eigvals.shape[0]

    def center(self, X):
        """
        Center `X` in PCA space.
        """
        X = X.copy()
        inan = numpy.isnan(X)
        if self.mu is None:
            X_ = numpy.ma.masked_array(X, inan)
            self.mu = X_.mean(0).base
            self.sigma = X_.std(0).base
        reduce(lambda y,x: setitem(x[0], x[1], x[2]), itertools.izip(X.T, inan.T, self.mu), None)
        X = X - self.mu
        X = X / numpy.where(self.sigma == 0, 1e-30, self.sigma)
        return X

    def _primal_eig(self, X):
        return numpy.linalg.eigh(numpy.einsum('ji,jk->ik',X,X))

    def _dual_eig(self, X):
        dual_eigvals, dual_eigvects = numpy.linalg.eigh(numpy.einsum('ij,kj->ik',X,X))
        relevant_dimensions = numpy.argsort(numpy.abs(dual_eigvals))[-X.shape[1]:]
        eigvals = dual_eigvals[relevant_dimensions]
        eigvects = dual_eigvects[:, relevant_dimensions]
        eigvects = (1. / numpy.sqrt(X.shape[0] * numpy.abs(eigvals))) * X.T.dot(eigvects)
        eigvects /= numpy.sqrt(numpy.diag(eigvects.T.dot(eigvects)))
        return eigvals, eigvects

    def project(self, X, Q=None):
        """
        Project X into PCA space, defined by the Q highest eigenvalues.
        Y = X dot V
        """
        if Q is None:
            Q = self.Q
        if Q > X.shape[1]:
            raise IndexError("requested dimension larger then input dimension")
        return X.dot(self.eigvectors[:, :Q])

    def plot_fracs(self, Q=None, ax=None, fignum=None):
        """
        Plot fractions of Eigenvalues sorted in descending order.
        """
        from GPy.plotting.matplot_dep import Tango
        Tango.reset()
        col = Tango.nextMedium()
            fig = pylab.figure(fignum)
            Q = self.Q
        ticks = numpy.arange(Q)
        bar = ax.bar(ticks - .4, self.fracs[:Q], color=col)
        ax.set_xticks(ticks, map(lambda x: r"${}$".format(x), ticks + 1))
        ax.set_ylabel("Eigenvalue fraction")
        ax.set_xlabel("PC")
        ax.set_ylim(0, ax.get_ylim()[1])
        ax.set_xlim(ticks.min() - .5, ticks.max() + .5)
        try:
            pylab.tight_layout()
        except:
            pass
        return bar

    def plot_2d(self, X, labels=None, s=20, marker='o',
                dimensions=(0, 1), ax=None, colors=None,
                fignum=None, cmap=None, # @UndefinedVariable
                ** kwargs):
        """
        Plot dimensions `dimensions` with given labels against each other in
        PC space. Labels can be any sequence of labels of dimensions X.shape[0].
        Labels can be drawn with a subsequent call to legend()
        """
        if cmap is None:
            cmap = matplotlib.cm.jet
        if labels is None:
            labels = numpy.zeros(X.shape[0])
        ulabels = []
        for lab in labels:
            if not lab in ulabels:
                ulabels.append(lab)
        nlabels = len(ulabels)
        if colors is None:
            colors = iter([cmap(float(i) / nlabels) for i in range(nlabels)])
        else:
            colors = iter(colors)
        X_ = self.project(X, self.Q)[:,dimensions]
        kwargs.update(dict(s=s))
        plots = list()
        for i, l in enumerate(ulabels):
            kwargs.update(dict(color=colors.next(), marker=marker[i % len(marker)]))
            plots.append(ax.scatter(*X_[labels == l, :].T, label=str(l), **kwargs))
        ax.set_xlabel(r"PC$_1$")
        ax.set_ylabel(r"PC$_2$")
        try:
        except:
            pass

from gradient_descent_update_rules import FletcherReeves, \
    PolakRibiere
from Queue import Empty
from multiprocessing import Value
from multiprocessing.queues import Queue
from multiprocessing.synchronize import Event
from scipy.optimize.linesearch import line_search_wolfe1, line_search_wolfe2
from threading import Thread
import sys
import time

RUNNING = "running"
CONVERGED = "converged"
MAXITER = "maximum number of iterations reached"
MAX_F_EVAL = "maximum number of function calls reached"
LINE_SEARCH = "line search failed"
KBINTERRUPT = "interrupted"

class _Async_Optimization(Thread):

    def __init__(self, f, df, x0, update_rule, runsignal, SENTINEL,
                 report_every=10, messages=0, maxiter=5e3, max_f_eval=15e3,
                 gtol=1e-6, outqueue=None, *args, **kw):
        """
        Helper Process class for async optimization
        
        f_call and df_call are Multiprocessing Values, for synchronized assignment
        """
        self.f_call = Value('i', 0)
        self.df_call = Value('i', 0)
        self.f = self.f_wrapper(f, self.f_call)
        self.df = self.f_wrapper(df, self.df_call)
        self.x0 = x0
        self.update_rule = update_rule
        self.report_every = report_every
        self.messages = messages
        self.maxiter = maxiter
        self.max_f_eval = max_f_eval
        self.gtol = gtol
        self.SENTINEL = SENTINEL
        self.runsignal = runsignal
#         self.parent = parent
#         self.result = None
        self.outq = outqueue
        super(_Async_Optimization, self).__init__(target=self.run,
                                            name="CG Optimization",
                                            *args, **kw)

#     def __enter__(self):
#         return self
#
#     def __exit__(self, type, value, traceback):
#         return isinstance(value, TypeError)

    def f_wrapper(self, f, counter):
        def f_w(*a, **kw):
            counter.value += 1
            return f(*a, **kw)
        return f_w

    def callback(self, *a):
        if self.outq is not None:
            self.outq.put(a)
#         self.parent and self.parent.callback(*a, **kw)
        pass
        # print "callback done"

    def callback_return(self, *a):
        self.callback(*a)
            self.outq.put(self.SENTINEL)
        if self.messages:
            print ""
        self.runsignal.clear()

    def run(self, *args, **kwargs):
        raise NotImplementedError("Overwrite this with optimization (for async use)")
        pass

class _CGDAsync(_Async_Optimization):

    def reset(self, xi, *a, **kw):
        gi = -self.df(xi, *a, **kw)
        si = gi
        ur = self.update_rule(gi)
        return gi, ur, si

    def run(self, *a, **kw):
        status = RUNNING

        fi = self.f(self.x0)
        fi_old = fi + 5000

        gi, ur, si = self.reset(self.x0, *a, **kw)
        xi = self.x0
        xi_old = numpy.nan
        it = 0

        while it < self.maxiter:
            if not self.runsignal.is_set():
                break

            if self.f_call.value > self.max_f_eval:
                status = MAX_F_EVAL

            gi = -self.df(xi, *a, **kw)
            if numpy.dot(gi.T, gi) <= self.gtol:
                status = CONVERGED
                break
            if numpy.isnan(numpy.dot(gi.T, gi)):
                if numpy.any(numpy.isnan(xi_old)):
                    status = CONVERGED
                    break
                self.reset(xi_old)

            gammai = ur(gi)
            if gammai < 1e-6 or it % xi.shape[0] == 0:
                gi, ur, si = self.reset(xi, *a, **kw)
            si = gi + gammai * si
            alphai, _, _, fi2, fi_old2, gfi = line_search_wolfe1(self.f,
                                                                 self.df,
                                                                 xi,
                                                                 si, gi,
                                                                 fi, fi_old)
            if alphai is None:
                alphai, _, _, fi2, fi_old2, gfi = \
                         line_search_wolfe2(self.f, self.df,
                                            xi, si, gi,
                                            fi, fi_old)
                if alphai is None:
                    # This line search also failed to find a better solution.
                    status = LINE_SEARCH
                    break
            if fi2 < fi:
                fi, fi_old = fi2, fi_old2
            if gfi is not None:
                gi = gfi

            if numpy.isnan(fi) or fi_old < fi:

            else:
                xi += numpy.dot(alphai, si)
                if self.messages:
                    sys.stdout.write("\r")
                    sys.stdout.flush()
                    sys.stdout.write("iteration: {0:> 6g}  f:{1:> 12e}  |g|:{2:> 12e}".format(it, fi, numpy.dot(gi.T, gi)))

            if it % self.report_every == 0:
                self.callback(xi, fi, gi, it, self.f_call.value, self.df_call.value, status)
            it += 1
        else:
            status = MAXITER
        self.callback_return(xi, fi, gi, it, self.f_call.value, self.df_call.value, status)
        self.result = [xi, fi, gi, it, self.f_call.value, self.df_call.value, status]

class Async_Optimize(object):
    callback = lambda *x: None
    runsignal = Event()
    SENTINEL = "SENTINEL"

    def async_callback_collect(self, q):
        while self.runsignal.is_set():
            try:
                for ret in iter(lambda: q.get(timeout=1), self.SENTINEL):
                    self.callback(*ret)
                self.runsignal.clear()
            except Empty:
                pass

    def opt_async(self, f, df, x0, callback, update_rule=PolakRibiere,
                   messages=0, maxiter=5e3, max_f_eval=15e3, gtol=1e-6,
                   report_every=10, *args, **kwargs):
        self.runsignal.set()
        c = None
        outqueue = None
        if callback:
            outqueue = Queue()
            self.callback = callback
            c = Thread(target=self.async_callback_collect, args=(outqueue,))
            c.start()
        p = _CGDAsync(f, df, x0, update_rule, self.runsignal, self.SENTINEL,
                 report_every=report_every, messages=messages, maxiter=maxiter,
                 max_f_eval=max_f_eval, gtol=gtol, outqueue=outqueue, *args, **kwargs)
        p.start()
        return p, c

    def opt(self, f, df, x0, callback=None, update_rule=FletcherReeves,
        p, c = self.opt_async(f, df, x0, callback, update_rule, messages,
                            maxiter, max_f_eval, gtol,
                            report_every, *args, **kwargs)
            try:
                p.join(1)
                if c: c.join(1)
                # print "^C"
                p.join()
                if c: c.join()
        if c and c.is_alive():
#             self.runsignal.set()
#             while self.runsignal.is_set():
#                 try:
#                     c.join(.1)
#                 except KeyboardInterrupt:
#                     # print "^C"
#                     self.runsignal.clear()
#                     c.join()
            print "WARNING: callback still running, optimisation done!"
        return p.result

class CGD(Async_Optimize):
    '''
    Conjugate gradient descent algorithm to minimize
    function f with gradients df, starting at x0
    with update rule update_rule
    
    if df returns tuple (grad, natgrad) it will optimize according 
    to natural gradient rules
    '''
    opt_name = "Conjugate Gradient Descent"

    def opt_async(self, *a, **kw):
        """
        opt_async(self, f, df, x0, callback, update_rule=FletcherReeves,
               messages=0, maxiter=5e3, max_f_eval=15e3, gtol=1e-6,
               report_every=10, \*args, \*\*kwargs)
        
        callback gets called every `report_every` iterations

            callback(xi, fi, gi, iteration, function_calls, gradient_calls, status_message)
        
        if df returns tuple (grad, natgrad) it will optimize according 
        to natural gradient rules
    
        f, and df will be called with
            
            f(xi, \*args, \*\*kwargs)
            df(xi, \*args, \*\*kwargs)
        
        **Returns:**
        
            Started `Process` object, optimizing asynchronously 
        
        **Calls:** 
        
            callback(x_opt, f_opt, g_opt, iteration, function_calls, gradient_calls, status_message)
        
        at end of optimization!
        """
        return super(CGD, self).opt_async(*a, **kw)

    def opt(self, *a, **kw):
        """
        opt(self, f, df, x0, callback=None, update_rule=FletcherReeves,
        
        Minimize f, calling callback every `report_every` iterations with following syntax:
        
        **returns** 
        
            x_opt, f_opt, g_opt, iteration, function_calls, gradient_calls, status_message
        
        at end of optimization
        """
        return super(CGD, self).opt(*a, **kw)

def view(A, offset=0):
    """
    Get a view on the diagonal elements of a 2D array.

    This is actually a view (!) on the diagonal of the array, so you can
    in-place adjust the view.

    :param :class:`ndarray` A: 2 dimensional numpy array
    :param int offset: view offset to give back (negative entries allowed)
    :rtype: :class:`ndarray` view of diag(A)

    >>> X = np.arange(9).reshape(3,3)
    >>> view(X)
    array([0, 4, 8])
    >>> d = view(X)
    >>> d += 2
    array([ 2,  6, 10])
    >>> view(X, offset=-1)
    array([3, 7])
    >>> subtract(X, 3, offset=-1)
    array([[ 2,  1,  2],
           [ 0,  6,  5],
           [ 6,  4, 10]])
    """
    from numpy.lib.stride_tricks import as_strided
    assert A.ndim == 2, "only implemented for 2 dimensions"
    assert A.shape[0] == A.shape[1], "attempting to get the view of non-square matrix?!"
    if offset > 0:
        return as_strided(A[0, offset:], shape=(A.shape[0] - offset, ), strides=((A.shape[0]+1)*A.itemsize, ))
    elif offset < 0:
        return as_strided(A[-offset:, 0], shape=(A.shape[0] + offset, ), strides=((A.shape[0]+1)*A.itemsize, ))
    else:
        return as_strided(A, shape=(A.shape[0], ), strides=((A.shape[0]+1)*A.itemsize, ))

def offdiag_view(A, offset=0):
    Af = as_strided(A, shape=(A.size,), strides=(A.itemsize,))
    return as_strided(Af[(1+offset):], shape=(A.shape[0]-1, A.shape[1]), strides=(A.strides[0] + A.itemsize, A.strides[1]))

def _diag_ufunc(A,b,offset,func):
    dA = view(A, offset); func(dA,b,dA)
    return A

def times(A, b, offset=0):
    """
    Times the view of A with b in place (!).
    Returns modified A
    Broadcasting is allowed, thus b can be scalar.

    if offset is not zero, make sure b is of right shape!

    :param ndarray A: 2 dimensional array
    :param ndarray-like b: either one dimensional or scalar
    :param int offset: same as in view.
    :rtype: view of A, which is adjusted inplace
    """
    return _diag_ufunc(A, b, offset, np.multiply)
multiply = times

def divide(A, b, offset=0):
    """
    Divide the view of A by b in place (!).

    """
    return _diag_ufunc(A, b, offset, np.divide)

def add(A, b, offset=0):
    """
    Add b to the view of A in place (!).
    Returns modified A.

    """
    return _diag_ufunc(A, b, offset, np.add)

def subtract(A, b, offset=0):
    """
    Subtract b from the view of A in place (!).

    """
    return _diag_ufunc(A, b, offset, np.subtract)

from GPy.util.univariate_Gaussian import std_norm_pdf,std_norm_cdf,inv_std_norm_cdf

_exp_lim_val = np.finfo(np.float64).max
_lim_val = np.log(_exp_lim_val)

class GPTransformation(object):
    """
    Link function class for doing non-Gaussian likelihoods approximation

    :param Y: observed output (Nx1 numpy.darray)

    .. note:: Y values allowed depend on the likelihood_function used

    """
        pass

    def transf(self,f):
        """
        Gaussian process tranformation function, latent space -> output space
        """

    def dtransf_df(self,f):
        """
        derivative of transf(f) w.r.t. f
        """

    def d2transf_df2(self,f):
        """
        second derivative of transf(f) w.r.t. f
        """

    def d3transf_df3(self,f):
        """
        third derivative of transf(f) w.r.t. f
        """

class Identity(GPTransformation):
    """
    .. math::

        g(f) = f

    """
        return f

        return np.ones_like(f)

        return np.zeros_like(f)

class Probit(GPTransformation):
    """
    .. math::

        g(f) = \\Phi^{-1} (mu)
    
    """
        return std_norm_cdf(f)

        return std_norm_pdf(f)

        #FIXME
        return -f * std_norm_pdf(f)

        #FIXME
        f2 = f**2
        return -(1/(np.sqrt(2*np.pi)))*np.exp(-0.5*(f2))*(1-f2)

class Cloglog(GPTransformation):
    """
    Complementary log-log link
    .. math::

        p(f) = 1 - e^{-e^f}

        or

        f = \log (-\log(1-p))
    
    """
        return 1-np.exp(-np.exp(f))

        return np.exp(f-np.exp(f))

        ef = np.exp(f)
        return -np.exp(f-ef)*(ef-1.)

        return np.exp(f-ef)*(1.-3*ef + ef**2)

class Log(GPTransformation):
    """
    .. math::

        g(f) = \\log(\\mu)

    """
        return np.exp(np.clip(f, -_lim_val, _lim_val))

class Log_ex_1(GPTransformation):
    """
    .. math::

        g(f) = \\log(\\exp(\\mu) - 1)

    """
        return np.log(1.+np.exp(f))

        return np.exp(f)/(1.+np.exp(f))

        aux = np.exp(f)/(1.+np.exp(f))
        return aux*(1.-aux)

        daux_df = aux*(1.-aux)
        return daux_df - (2.*aux*daux_df)

class Reciprocal(GPTransformation):
        return 1./f

        return -1./(f**2)

        return 2./(f**3)

        return -6./(f**4)

class Heaviside(GPTransformation):
    """

    .. math::

        g(f) = I_{x \\in A}

    """
        #transformation goes here
        return np.where(f>0, 1, 0)

        raise NotImplementedError, "This function is not differentiable!"

from ..core import GP

class GPRegression(GP):
    """

    This is a thin wrapper around the models.GP class, with a set of sensible defaults

    :param kernel: a GPy kernel, defaults to rbf
    :param Norm normalizer: [False]

        Normalize Y with the norm given.
        If normalizer is False, no normalization will be done
        If it is None, we use GaussianNorm(alization)

    """

    def __init__(self, X, Y, kernel=None, Y_metadata=None, normalizer=None):

            kernel = kern.RBF(X.shape[1])

        super(GPRegression, self).__init__(X, Y, kernel, likelihood, name='GP regression', Y_metadata=Y_metadata, normalizer=normalizer)

# ## Copyright (c) 2014, Zhenwen Dai

class HMC:
    """
    An implementation of Hybrid Monte Carlo (HMC) for GPy models
    
    Initialize an object for HMC sampling. Note that the status of the model (model parameters) will be changed during sampling.
    
    :param model: the GPy model that will be sampled
    :type model: GPy.core.Model
    :param M: the mass matrix (an identity matrix by default)
    :type M: numpy.ndarray
    :param stepsize: the step size for HMC sampling
    :type stepsize: float
    """
    def __init__(self, model, M=None,stepsize=1e-1):
        self.stepsize = stepsize
        self.p = np.empty_like(model.optimizer_array.copy())
        if M is None:
            self.M = np.eye(self.p.size)
        else:
            self.M = M
        self.Minv = np.linalg.inv(self.M)

    def sample(self, num_samples=1000, hmc_iters=20):
        """
        Sample the (unfixed) model parameters.
        
        :param num_samples: the number of samples to draw (1000 by default)
        :type num_samples: int
        :param hmc_iters: the number of leap-frog iterations (20 by default)
        :type hmc_iters: int
        :return: the list of parameters samples with the size N x P (N - the number of samples, P - the number of parameters to sample) 
        :rtype: numpy.ndarray
        """
        params = np.empty((num_samples,self.p.size))
        for i in xrange(num_samples):
            self.p[:] = np.random.multivariate_normal(np.zeros(self.p.size),self.M)
            H_old = self._computeH()
            theta_old = self.model.optimizer_array.copy()
            params[i] = self.model.unfixed_param_array
            #Matropolis
            self._update(hmc_iters)
            H_new = self._computeH()

            if H_old>H_new:
                k = 1.
            else:
                k = np.exp(H_old-H_new)
            if np.random.rand()<k:
                params[i] = self.model.unfixed_param_array
            else:
                self.model.optimizer_array = theta_old
        return params

    def _update(self, hmc_iters):
        for i in xrange(hmc_iters):
            self.p[:] += -self.stepsize/2.*self.model._transform_gradients(self.model.objective_function_gradients())
            self.model.optimizer_array = self.model.optimizer_array + self.stepsize*np.dot(self.Minv, self.p)

    def _computeH(self,):
        return self.model.objective_function()+self.p.size*np.log(2*np.pi)/2.+np.log(np.linalg.det(self.M))/2.+np.dot(self.p, np.dot(self.Minv,self.p[:,None]))/2.

class HMC_shortcut:
    def __init__(self,model,M=None,stepsize_range=[1e-6, 1e-1],groupsize=5, Hstd_th=[1e-5, 3.]):
        self.stepsize_range = np.log(stepsize_range)
        self.groupsize = groupsize
        self.Hstd_th = Hstd_th
        else:
            self.M = M

    def sample(self, m_iters=1000, hmc_iters=20):
        params = np.empty((m_iters,self.p.size))
        for i in xrange(m_iters):
            # sample a stepsize from the uniform distribution
            stepsize = np.exp(np.random.rand()*(self.stepsize_range[1]-self.stepsize_range[0])+self.stepsize_range[0])
            self._update(hmc_iters, stepsize)

                k = 1.
            else:
            else:

    def _update(self, hmc_iters, stepsize):
        theta_buf = np.empty((2*hmc_iters+1,self.model.optimizer_array.size))
        p_buf = np.empty((2*hmc_iters+1,self.p.size))
        H_buf = np.empty((2*hmc_iters+1,))
        # Set initial position
        theta_buf[hmc_iters] = self.model.optimizer_array
        p_buf[hmc_iters] = self.p
        H_buf[hmc_iters] = self._computeH()

        reversal = []
        pos = 1
        i=0
        while i<hmc_iters:
            self.p[:] += -stepsize/2.*self.model._transform_gradients(self.model.objective_function_gradients())
            self.model.optimizer_array = self.model.optimizer_array + stepsize*np.dot(self.Minv, self.p)

            theta_buf[hmc_iters+pos] = self.model.optimizer_array
            p_buf[hmc_iters+pos] = self.p
            H_buf[hmc_iters+pos] = self._computeH()
            i+=1

            if i<self.groupsize:
                pos += 1
                continue
            else:
                if len(reversal)==0:
                    Hlist = range(hmc_iters+pos,hmc_iters+pos-self.groupsize,-1)
                    if self._testH(H_buf[Hlist]):
                        pos += 1
                    else:
                        # Reverse the trajectory for the 1st time
                        reversal.append(pos)
                        if hmc_iters-i>pos:
                            pos = -1
                            i += pos
                            self.model.optimizer_array = theta_buf[hmc_iters]
                            self.p[:] = -p_buf[hmc_iters]
                        else:
                            pos_new = pos-hmc_iters+i
                            self.model.optimizer_array = theta_buf[hmc_iters+pos_new]
                            self.p[:] = -p_buf[hmc_iters+pos_new]
                            break
                else:
                    Hlist = range(hmc_iters+pos,hmc_iters+pos+self.groupsize)

                        pos += -1
                    else:
                        # Reverse the trajectory for the 2nd time
                        r = (hmc_iters - i)%((reversal[0]-pos)*2)
                        if r>(reversal[0]-pos):
                            pos_new = 2*reversal[0] - r - pos
                        else:
                            pos_new = pos + r
                        self.model.optimizer_array = theta_buf[hmc_iters+pos_new]
                        self.p[:] = p_buf[hmc_iters+pos_new] # the sign of momentum might be wrong!
                        break

    def _testH(self, Hlist):
        Hstd = np.std(Hlist)
        if Hstd<self.Hstd_th[0] or Hstd>self.Hstd_th[1]:
        else:

import GPy

class Additive(Mapping):
    """
    Mapping based on adding two existing mappings together.

    .. math::

       f(\mathbf{x}*) = f_1(\mathbf{x}*) + f_2(\mathbf(x)*)

    :param mapping1: first mapping to add together.
    :type mapping1: GPy.mappings.Mapping
    :param mapping2: second mapping to add together.
    :type mapping2: GPy.mappings.Mapping
    :param tensor: whether or not to use the tensor product of input spaces
    :type tensor: bool

    """

    def __init__(self, mapping1, mapping2, tensor=False):
        if tensor:
            input_dim = mapping1.input_dim + mapping2.input_dim
        else:
            input_dim = mapping1.input_dim
            assert(mapping1.input_dim==mapping2.input_dim)
        assert(mapping1.output_dim==mapping2.output_dim)
        output_dim = mapping1.output_dim
        self.mapping1 = mapping1
        self.mapping2 = mapping2
        self.num_params = self.mapping1.num_params + self.mapping2.num_params
        self.name = self.mapping1.name + '+' + self.mapping2.name
        return self.mapping1._get_param_names + self.mapping2._get_param_names

        return np.hstack((self.mapping1._get_params(), self.mapping2._get_params()))

        self.mapping1._set_params(x[:self.mapping1.num_params])
        self.mapping2._set_params(x[self.mapping1.num_params:])
        
        self.mapping1._randomize()
        self.mapping2._randomize()

        return self.mapping1.f(X) + self.mapping2.f(X)

        self._df_dA = (dL_df[:, :, None]*self.kern.K(X, self.X)[:, None, :]).sum(0).T
        self._df_dbias = (dL_df.sum(0))
        return np.hstack((self._df_dA.flatten(), self._df_dbias))

        return self.kern.dK_dX((dL_df[:, None, :]*self.A[None, :, :]).sum(2), X, self.X) 

"""
The module for psi-statistics for RBF kernel for Spike-and-Slab GPLVM
"""

from ....util.caching import Cache_this
from . import PSICOMP_RBF
from ....util import gpu_init

try:
    import pycuda.gpuarray as gpuarray
    from pycuda.compiler import SourceModule
    from ....util.linalg_gpu import sum_axis
except:
    pass    

gpu_code = """
    // define THREADNUM

    #define IDX_NMQ(n,m,q) ((q*M+m)*N+n)
    #define IDX_NMM(n,m1,m2) ((m2*M+m1)*N+n)
    #define IDX_NQ(n,q) (q*N+n)
    #define IDX_NM(n,m) (m*N+n)
    #define IDX_MQ(m,q) (q*M+m)
    #define IDX_MM(m1,m2) (m2*M+m1)
    #define IDX_NQB(n,q,b) ((b*Q+q)*N+n)
    #define IDX_QB(q,b) (b*Q+q)

    // Divide data evenly
    __device__ void divide_data(int total_data, int psize, int pidx, int *start, int *end) {
        int residue = (total_data)%psize;
        if(pidx<residue) {
            int size = total_data/psize+1;
            *start = size*pidx;
            *end = *start+size;
        } else {
            int size = total_data/psize;
            *start = size*pidx+residue;
        }
    }
    
    __device__ void reduce_sum(double* array, int array_size) {
        int s;
        if(array_size >= blockDim.x) {
            for(int i=blockDim.x+threadIdx.x; i<array_size; i+= blockDim.x) {
                array[threadIdx.x] += array[i];
            }
            array_size = blockDim.x;
        }
        __syncthreads();
        for(int i=1; i<=array_size;i*=2) {s=i;}
        if(threadIdx.x < array_size-s) {array[threadIdx.x] += array[s+threadIdx.x];}
        for(s=s/2;s>=1;s=s/2) {
            if(threadIdx.x < s) {array[threadIdx.x] += array[s+threadIdx.x];}
            __syncthreads();
        }
    }

    __global__ void compDenom(double *log_denom1, double *log_denom2, double *log_gamma, double*log_gamma1, double *gamma, double *l, double *S, int N, int Q)
    {
        int n_start, n_end;
        divide_data(N, gridDim.x, blockIdx.x, &n_start, &n_end);
        
        for(int i=n_start*Q+threadIdx.x; i<n_end*Q; i+=blockDim.x) {
            int n=i/Q;
            int q=i%Q;

            double Snq = S[IDX_NQ(n,q)];
            double lq = l[q]*l[q];
            double gnq = gamma[IDX_NQ(n,q)];
            log_denom1[IDX_NQ(n,q)] = log(Snq/lq+1.);
            log_denom2[IDX_NQ(n,q)] = log(2.*Snq/lq+1.);
            log_gamma[IDX_NQ(n,q)] = log(gnq);
            log_gamma1[IDX_NQ(n,q)] = log(1.-gnq);
        }
    }

    __global__ void psi1computations(double *psi1, double *log_denom1, double *log_gamma, double*log_gamma1, double var, double *l, double *Z, double *mu, double *S, int N, int M, int Q)
    {
        int m_start, m_end;
        divide_data(M, gridDim.x, blockIdx.x, &m_start, &m_end);
        
        for(int m=m_start; m<m_end; m++) {
            for(int n=threadIdx.x; n<N; n+= blockDim.x) {            
                double log_psi1 = 0;
                for(int q=0;q<Q;q++) {
                    double Zmq = Z[IDX_MQ(m,q)];
                    double muZ = mu[IDX_NQ(n,q)]-Zmq;
                    double Snq = S[IDX_NQ(n,q)];
                    double lq = l[q]*l[q];
                    double exp1 = log_gamma[IDX_NQ(n,q)]-(muZ*muZ/(Snq+lq)+log_denom1[IDX_NQ(n,q)])/(2.);
                    double exp2 = log_gamma1[IDX_NQ(n,q)]-Zmq*Zmq/(2.*lq);
                    log_psi1 += (exp1>exp2)?exp1+log1p(exp(exp2-exp1)):exp2+log1p(exp(exp1-exp2));
                }
                psi1[IDX_NM(n,m)] = var*exp(log_psi1);
            }
        }
    }
    
    __global__ void psi2computations(double *psi2, double *psi2n, double *log_denom2, double *log_gamma, double*log_gamma1, double var, double *l, double *Z, double *mu, double *S, int N, int M, int Q)
    {
        int psi2_idx_start, psi2_idx_end;
        __shared__ double psi2_local[THREADNUM];
        divide_data((M+1)*M/2, gridDim.x, blockIdx.x, &psi2_idx_start, &psi2_idx_end);
        
        for(int psi2_idx=psi2_idx_start; psi2_idx<psi2_idx_end; psi2_idx++) {
            int m1 = int((sqrt(8.*psi2_idx+1.)-1.)/2.);
            int m2 = psi2_idx - (m1+1)*m1/2;
            
            psi2_local[threadIdx.x] = 0;
            for(int n=threadIdx.x;n<N;n+=blockDim.x) {
                double log_psi2_n = 0;
                    double Zm1q = Z[IDX_MQ(m1,q)];
                    double Zm2q = Z[IDX_MQ(m2,q)];
                    double dZ = Zm1q - Zm2q;
                    double muZhat = mu[IDX_NQ(n,q)]- (Zm1q+Zm2q)/2.;
                    double Z2 = Zm1q*Zm1q+Zm2q*Zm2q;
                    double exp1 = dZ*dZ/(-4.*lq)-muZhat*muZhat/(2.*Snq+lq) - log_denom2[IDX_NQ(n,q)]/2. + log_gamma[IDX_NQ(n,q)];
                    double exp2 = log_gamma1[IDX_NQ(n,q)] - Z2/(2.*lq);
                    log_psi2_n += (exp1>exp2)?exp1+log1p(exp(exp2-exp1)):exp2+log1p(exp(exp1-exp2));
                }
                double exp_psi2_n = exp(log_psi2_n);
                psi2n[IDX_NMM(n,m1,m2)] = var*var*exp_psi2_n;
                if(m1!=m2) { psi2n[IDX_NMM(n,m2,m1)] = var*var*exp_psi2_n;}
                psi2_local[threadIdx.x] += exp_psi2_n;
            }
            reduce_sum(psi2_local, THREADNUM);
            if(threadIdx.x==0) {
                psi2[IDX_MM(m1,m2)] = var*var*psi2_local[0];
                if(m1!=m2) { psi2[IDX_MM(m2,m1)] = var*var*psi2_local[0]; }
            }
        }
    }
    
    __global__ void psi1compDer(double *dvar, double *dl, double *dZ, double *dmu, double *dS, double *dgamma, double *dL_dpsi1, double *psi1, double *log_denom1, double *log_gamma, double*log_gamma1, double var, double *l, double *Z, double *mu, double *S, double *gamma, int N, int M, int Q)
    {
        __shared__ double g_local[THREADNUM];
        int P = int(ceil(double(N)/THREADNUM));

        double dvar_local = 0;
        for(int q=0;q<Q;q++) {
            double lq_sqrt = l[q];
            double lq = lq_sqrt*lq_sqrt;
            double dl_local = 0;
            for(int p=0;p<P;p++) {
                int n = p*THREADNUM + threadIdx.x;
                double dmu_local = 0;
                double dS_local = 0;
                double dgamma_local = 0;
                double Snq,mu_nq,gnq,log_gnq,log_gnq1,log_de;
                if(n<N) {Snq = S[IDX_NQ(n,q)]; mu_nq=mu[IDX_NQ(n,q)]; gnq = gamma[IDX_NQ(n,q)];
                        log_gnq = log_gamma[IDX_NQ(n,q)]; log_gnq1 = log_gamma1[IDX_NQ(n,q)];
                        log_de = log_denom1[IDX_NQ(n,q)];}
                for(int m=m_start; m<m_end; m++) {
                    if(n<N) {
                        double lpsi1 = psi1[IDX_NM(n,m)]*dL_dpsi1[IDX_NM(n,m)];
                        if(q==0) {dvar_local += lpsi1;}
                        
                        double Zmq = Z[IDX_MQ(m,q)];
                        double Zmu = Zmq - mu_nq;
                        double denom = Snq+lq;
                        double Zmu2_denom = Zmu*Zmu/denom;
                        
                        double exp1 = log_gnq-(Zmu*Zmu/(Snq+lq)+log_de)/(2.);
                        double exp2 = log_gnq1-Zmq*Zmq/(2.*lq);
                        } else {
                        }
                        
                        dmu_local += lpsi1*Zmu*d_exp1/(denom*exp_sum);
                        dS_local += lpsi1*(Zmu2_denom-1.)*d_exp1/(denom*exp_sum);
                        dgamma_local += lpsi1*(d_exp1/gnq-d_exp2/(1.-gnq))/exp_sum;
                        dl_local += lpsi1*((Zmu2_denom+Snq/lq)/denom*d_exp1+Zmq*Zmq/(lq*lq)*d_exp2)/(2.*exp_sum);
                        g_local[threadIdx.x] = lpsi1*(-Zmu/denom*d_exp1-Zmq/lq*d_exp2)/exp_sum;
                    }
                    __syncthreads();
                    reduce_sum(g_local, p<P-1?THREADNUM:N-(P-1)*THREADNUM);
                    if(threadIdx.x==0) {dZ[IDX_MQ(m,q)] += g_local[0];}
                }
                if(n<N) {
                    dmu[IDX_NQB(n,q,blockIdx.x)] += dmu_local;
                    dS[IDX_NQB(n,q,blockIdx.x)] += dS_local/2.;
                    dgamma[IDX_NQB(n,q,blockIdx.x)] += dgamma_local;
                }
                __threadfence_block();
            }
            g_local[threadIdx.x] = dl_local*2.*lq_sqrt;
            reduce_sum(g_local, THREADNUM);
            if(threadIdx.x==0) {dl[IDX_QB(q,blockIdx.x)] += g_local[0];}
        }
        g_local[threadIdx.x] = dvar_local;
        reduce_sum(g_local, THREADNUM);
        if(threadIdx.x==0) {dvar[blockIdx.x] += g_local[0]/var;}
    }
    
    __global__ void psi2compDer(double *dvar, double *dl, double *dZ, double *dmu, double *dS, double *dgamma, double *dL_dpsi2, double *psi2n, double *log_denom2, double *log_gamma, double*log_gamma1, double var, double *l, double *Z, double *mu, double *S, double *gamma, int N, int M, int Q)
    {

                        log_de = log_denom2[IDX_NQ(n,q)];}
                for(int m1=m_start; m1<m_end; m1++) {
                    g_local[threadIdx.x] = 0;
                    for(int m2=0;m2<M;m2++) {
                        if(n<N) {
                            double lpsi2 = psi2n[IDX_NMM(n,m1,m2)]*dL_dpsi2[IDX_MM(m1,m2)];
                            if(q==0) {dvar_local += lpsi2;}
                            
                            double Zm1q = Z[IDX_MQ(m1,q)];
                            double Zm2q = Z[IDX_MQ(m2,q)];
                            double dZ = Zm1q - Zm2q;
                            double Z2 = Zm1q*Zm1q+Zm2q*Zm2q;
                            double muZhat =  mu_nq - (Zm1q + Zm2q)/2.;
                            double denom = 2.*Snq+lq;
                            double muZhat2_denom = muZhat*muZhat/denom;
                            
                            double exp1 = dZ*dZ/(-4.*lq)-muZhat*muZhat/(2.*Snq+lq) - log_de/2. + log_gnq;
                            double exp2 = log_gnq1 - Z2/(2.*lq);
                            } else {
                            }
                            
                            dmu_local += lpsi2*muZhat/denom*d_exp1/exp_sum;
                            dS_local += lpsi2*(2.*muZhat2_denom-1.)/denom*d_exp1/exp_sum;
                            dgamma_local += lpsi2*(d_exp1/gnq-d_exp2/(1.-gnq))/exp_sum;
                            dl_local += lpsi2*(((Snq/lq+muZhat2_denom)/denom+dZ*dZ/(4.*lq*lq))*d_exp1+Z2/(2.*lq*lq)*d_exp2)/exp_sum;
                            g_local[threadIdx.x] += 2.*lpsi2*((muZhat/denom-dZ/(2*lq))*d_exp1-Zm1q/lq*d_exp2)/exp_sum;
                        }
                    }
                    if(threadIdx.x==0) {dZ[IDX_MQ(m1,q)] += g_local[0];}
                }
                if(n<N) {
                    dmu[IDX_NQB(n,q,blockIdx.x)] += -2.*dmu_local;
                    dS[IDX_NQB(n,q,blockIdx.x)] += dS_local;
                }
            }
        }
        if(threadIdx.x==0) {dvar[blockIdx.x] += g_local[0]*2/var;}
    }
    """

class PSICOMP_SSRBF_GPU(PSICOMP_RBF):

    def __init__(self, threadnum=128, blocknum=15, GPU_direct=False):
        self.GPU_direct = GPU_direct
        self.gpuCache = None
        
        self.threadnum = threadnum
        self.blocknum = blocknum
        module = SourceModule("#define THREADNUM "+str(self.threadnum)+"\n"+gpu_code)
        self.g_psi1computations = module.get_function('psi1computations')
        self.g_psi1computations.prepare('PPPPdPPPPiii')
        self.g_psi2computations = module.get_function('psi2computations')
        self.g_psi2computations.prepare('PPPPPdPPPPiii')
        self.g_psi1compDer = module.get_function('psi1compDer')
        self.g_psi1compDer.prepare('PPPPPPPPPPPdPPPPPiii')
        self.g_psi2compDer = module.get_function('psi2compDer')
        self.g_psi2compDer.prepare('PPPPPPPPPPPdPPPPPiii')
        self.g_compDenom = module.get_function('compDenom')
        self.g_compDenom.prepare('PPPPPPPii')

    def __deepcopy__(self, memo):
        s = PSICOMP_SSRBF_GPU(threadnum=self.threadnum, blocknum=self.blocknum, GPU_direct=self.GPU_direct)
        memo[id(self)] = s 
        return s

    def _initGPUCache(self, N, M, Q):            
        if self.gpuCache == None:
            self.gpuCache = {
                             'l_gpu'                :gpuarray.empty((Q,),np.float64,order='F'),
                             'Z_gpu'                :gpuarray.empty((M,Q),np.float64,order='F'),
                             'mu_gpu'               :gpuarray.empty((N,Q),np.float64,order='F'),
                             'S_gpu'                :gpuarray.empty((N,Q),np.float64,order='F'),
                             'gamma_gpu'            :gpuarray.empty((N,Q),np.float64,order='F'),
                             'psi1_gpu'             :gpuarray.empty((N,M),np.float64,order='F'),
                             'psi2_gpu'             :gpuarray.empty((M,M),np.float64,order='F'),
                             'psi2n_gpu'            :gpuarray.empty((N,M,M),np.float64,order='F'),
                             'dL_dpsi1_gpu'         :gpuarray.empty((N,M),np.float64,order='F'),
                             'dL_dpsi2_gpu'         :gpuarray.empty((M,M),np.float64,order='F'),
                             'log_denom1_gpu'       :gpuarray.empty((N,Q),np.float64,order='F'),
                             'log_denom2_gpu'       :gpuarray.empty((N,Q),np.float64,order='F'),
                             'log_gamma_gpu'        :gpuarray.empty((N,Q),np.float64,order='F'),
                             'log_gamma1_gpu'       :gpuarray.empty((N,Q),np.float64,order='F'),
                             # derivatives
                             'dvar_gpu'             :gpuarray.empty((self.blocknum,),np.float64, order='F'),
                             'dl_gpu'               :gpuarray.empty((Q,self.blocknum),np.float64, order='F'),
                             'dZ_gpu'               :gpuarray.empty((M,Q),np.float64, order='F'),
                             'dmu_gpu'              :gpuarray.empty((N,Q,self.blocknum),np.float64, order='F'),
                             'dS_gpu'               :gpuarray.empty((N,Q,self.blocknum),np.float64, order='F'),
                             'dgamma_gpu'           :gpuarray.empty((N,Q,self.blocknum),np.float64, order='F'),
                             # grad
                             'grad_l_gpu'               :gpuarray.empty((Q,),np.float64, order='F'),
                             'grad_mu_gpu'              :gpuarray.empty((N,Q,),np.float64, order='F'),
                             'grad_S_gpu'               :gpuarray.empty((N,Q,),np.float64, order='F'),
                             'grad_gamma_gpu'           :gpuarray.empty((N,Q,),np.float64, order='F'),
                             }
        else:
            assert N==self.gpuCache['mu_gpu'].shape[0]
            assert M==self.gpuCache['Z_gpu'].shape[0]
            assert Q==self.gpuCache['l_gpu'].shape[0]
    
    def sync_params(self, lengthscale, Z, mu, S, gamma):
        if len(lengthscale)==1:
            self.gpuCache['l_gpu'].fill(lengthscale)
        else:
            self.gpuCache['l_gpu'].set(np.asfortranarray(lengthscale))
        self.gpuCache['Z_gpu'].set(np.asfortranarray(Z))
        self.gpuCache['mu_gpu'].set(np.asfortranarray(mu))
        self.gpuCache['S_gpu'].set(np.asfortranarray(S))
        self.gpuCache['gamma_gpu'].set(np.asfortranarray(gamma))
        N,Q = self.gpuCache['S_gpu'].shape
        self.g_compDenom.prepared_call((self.blocknum,1),(self.threadnum,1,1), self.gpuCache['log_denom1_gpu'].gpudata,self.gpuCache['log_denom2_gpu'].gpudata,self.gpuCache['log_gamma_gpu'].gpudata,self.gpuCache['log_gamma1_gpu'].gpudata,self.gpuCache['gamma_gpu'].gpudata,self.gpuCache['l_gpu'].gpudata,self.gpuCache['S_gpu'].gpudata, np.int32(N), np.int32(Q))
        
    def reset_derivative(self):
        self.gpuCache['dvar_gpu'].fill(0.)
        self.gpuCache['dl_gpu'].fill(0.)
        self.gpuCache['dZ_gpu'].fill(0.)
        self.gpuCache['dmu_gpu'].fill(0.)
        self.gpuCache['dS_gpu'].fill(0.)
        self.gpuCache['dgamma_gpu'].fill(0.)
        self.gpuCache['grad_l_gpu'].fill(0.)
        self.gpuCache['grad_mu_gpu'].fill(0.)
        self.gpuCache['grad_S_gpu'].fill(0.)
        self.gpuCache['grad_gamma_gpu'].fill(0.)
    
    def get_dimensions(self, Z, variational_posterior):
        return variational_posterior.mean.shape[0], Z.shape[0], Z.shape[1]

    def psicomputations(self, variance, lengthscale, Z, variational_posterior):
        """
        Z - MxQ
        mu - NxQ
        S - NxQ
        """
        N,M,Q = self.get_dimensions(Z, variational_posterior)
        self._initGPUCache(N,M,Q)
        self.sync_params(lengthscale, Z, variational_posterior.mean, variational_posterior.variance, variational_posterior.binary_prob)
        
        psi1_gpu = self.gpuCache['psi1_gpu']
        psi2_gpu = self.gpuCache['psi2_gpu']
        psi2n_gpu = self.gpuCache['psi2n_gpu']
        l_gpu = self.gpuCache['l_gpu']
        Z_gpu = self.gpuCache['Z_gpu']
        mu_gpu = self.gpuCache['mu_gpu']
        S_gpu = self.gpuCache['S_gpu']
        log_denom1_gpu = self.gpuCache['log_denom1_gpu']
        log_denom2_gpu = self.gpuCache['log_denom2_gpu']
        log_gamma_gpu = self.gpuCache['log_gamma_gpu']
        log_gamma1_gpu = self.gpuCache['log_gamma1_gpu']

        psi0 = np.empty((N,))
        self.g_psi1computations.prepared_call((self.blocknum,1),(self.threadnum,1,1),psi1_gpu.gpudata, log_denom1_gpu.gpudata, log_gamma_gpu.gpudata, log_gamma1_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata, np.int32(N), np.int32(M), np.int32(Q))
        self.g_psi2computations.prepared_call((self.blocknum,1),(self.threadnum,1,1),psi2_gpu.gpudata, psi2n_gpu.gpudata, log_denom2_gpu.gpudata, log_gamma_gpu.gpudata, log_gamma1_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata, np.int32(N), np.int32(M), np.int32(Q))
        
        if self.GPU_direct:
            return psi0, psi1_gpu, psi2_gpu
        else:
            return psi0, psi1_gpu.get(), psi2_gpu.get()

    @Cache_this(limit=1, ignore_args=(0,1,2,3))
    def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior):
        
        gamma_gpu = self.gpuCache['gamma_gpu']
        dvar_gpu = self.gpuCache['dvar_gpu']
        dl_gpu = self.gpuCache['dl_gpu']
        dZ_gpu = self.gpuCache['dZ_gpu']
        dmu_gpu = self.gpuCache['dmu_gpu']
        dS_gpu = self.gpuCache['dS_gpu']
        dgamma_gpu = self.gpuCache['dgamma_gpu']
        grad_l_gpu = self.gpuCache['grad_l_gpu']
        grad_mu_gpu = self.gpuCache['grad_mu_gpu']
        grad_S_gpu = self.gpuCache['grad_S_gpu']
        grad_gamma_gpu = self.gpuCache['grad_gamma_gpu']
        
            dL_dpsi1_gpu = dL_dpsi1
            dL_dpsi2_gpu = dL_dpsi2
            dL_dpsi0_sum = gpuarray.sum(dL_dpsi0).get()
        else:
            dL_dpsi1_gpu = self.gpuCache['dL_dpsi1_gpu']
            dL_dpsi2_gpu = self.gpuCache['dL_dpsi2_gpu']
            dL_dpsi1_gpu.set(np.asfortranarray(dL_dpsi1))
            dL_dpsi2_gpu.set(np.asfortranarray(dL_dpsi2))
            dL_dpsi0_sum = dL_dpsi0.sum()

        self.reset_derivative()
        # t=self.g_psi1compDer(dvar_gpu,dl_gpu,dZ_gpu,dmu_gpu,dS_gpu,dL_dpsi1_gpu,psi1_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True)
        # print 'g_psi1compDer '+str(t)
        # t=self.g_psi2compDer(dvar_gpu,dl_gpu,dZ_gpu,dmu_gpu,dS_gpu,dL_dpsi2_gpu,psi2n_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True)
        # print 'g_psi2compDer '+str(t)
        self.g_psi1compDer.prepared_call((self.blocknum,1),(self.threadnum,1,1),dvar_gpu.gpudata,dl_gpu.gpudata,dZ_gpu.gpudata,dmu_gpu.gpudata,dS_gpu.gpudata,dgamma_gpu.gpudata,dL_dpsi1_gpu.gpudata,psi1_gpu.gpudata, log_denom1_gpu.gpudata, log_gamma_gpu.gpudata, log_gamma1_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata,gamma_gpu.gpudata,np.int32(N), np.int32(M), np.int32(Q))
        self.g_psi2compDer.prepared_call((self.blocknum,1),(self.threadnum,1,1),dvar_gpu.gpudata,dl_gpu.gpudata,dZ_gpu.gpudata,dmu_gpu.gpudata,dS_gpu.gpudata,dgamma_gpu.gpudata,dL_dpsi2_gpu.gpudata,psi2n_gpu.gpudata, log_denom2_gpu.gpudata, log_gamma_gpu.gpudata, log_gamma1_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata,gamma_gpu.gpudata,np.int32(N), np.int32(M), np.int32(Q))

        dL_dvar = dL_dpsi0_sum + gpuarray.sum(dvar_gpu).get()
        sum_axis(grad_mu_gpu,dmu_gpu,N*Q,self.blocknum)
        dL_dmu = grad_mu_gpu.get()
        sum_axis(grad_S_gpu,dS_gpu,N*Q,self.blocknum)
        dL_dS = grad_S_gpu.get()
        sum_axis(grad_gamma_gpu,dgamma_gpu,N*Q,self.blocknum)
        dL_dgamma = grad_gamma_gpu.get()
        dL_dZ = dZ_gpu.get()
        if ARD:
            sum_axis(grad_l_gpu,dl_gpu,Q,self.blocknum)
            dL_dlengscale = grad_l_gpu.get()
        else:
            dL_dlengscale = gpuarray.sum(dl_gpu).get()
            
# Copyright (c) 2014, Alan Saul
import GPy
from GPy.models import GradientChecker
import inspect
from GPy.likelihoods import link_functions
from GPy.core.parameterization import Param
from functools import partial
#np.random.seed(300)
#np.random.seed(7)

#np.seterr(divide='raise')
def dparam_partial(inst_func, *args):
    """
    If we have a instance method that needs to be called but that doesn't
    take the parameter we wish to change to checkgrad, then this function
    will change the variable using set params.

    inst_func: should be a instance function of an object that we would like
                to change
    param: the param that will be given to set_params
    args: anything else that needs to be given to the function (for example
          the f or Y that are being used in the function whilst we tweak the
          param
    """
    def param_func(param_val, param_name, inst_func, args):
        #inst_func.im_self._set_params(param)
        #inst_func.im_self.add_parameter(Param(param_name, param_val))
        inst_func.im_self[param_name] = param_val
        return inst_func(*args)
    return functools.partial(param_func, inst_func=inst_func, args=args)

def dparam_checkgrad(func, dfunc, params, params_names, args, constraints=None, randomize=False, verbose=False):
    """
    checkgrad expects a f: R^N -> R^1 and df: R^N -> R^N
    However if we are holding other parameters fixed and moving something else
    We need to check the gradient of each of the fixed parameters
    (f and y for example) seperately,  whilst moving another parameter.
    Otherwise f: gives back R^N and
              df: gives back R^NxM where M is
    The number of parameters and N is the number of data
    Need to take a slice out from f and a slice out of df
    """
    print "\n{} likelihood: {} vs {}".format(func.im_self.__class__.__name__,
                                           func.__name__, dfunc.__name__)
    partial_f = dparam_partial(func, *args)
    partial_df = dparam_partial(dfunc, *args)
    gradchecking = True
    zipped_params = zip(params, params_names)
    for param_ind, (param_val, param_name) in enumerate(zipped_params):
        #Check one parameter at a time, make sure it is 2d (as some gradients only return arrays) then strip out the parameter
        fnum = np.atleast_2d(partial_f(param_val, param_name))[:, param_ind].shape[0]
        dfnum = np.atleast_2d(partial_df(param_val, param_name))[:, param_ind].shape[0]
        for fixed_val in range(dfnum):
            #dlik and dlik_dvar gives back 1 value for each
            f_ind = min(fnum, fixed_val+1) - 1
            print "fnum: {} dfnum: {} f_ind: {} fixed_val: {}".format(fnum, dfnum, f_ind, fixed_val)
            #Make grad checker with this param moving, note that set_params is NOT being called
            #The parameter is being set directly with __setattr__
            #Check only the parameter and function value we wish to check at a time
            grad = GradientChecker(lambda p_val: np.atleast_2d(partial_f(p_val, param_name))[f_ind, param_ind],
                                   lambda p_val: np.atleast_2d(partial_df(p_val, param_name))[fixed_val, param_ind],
                                   param_val, [param_name])

            if constraints is not None:
                for constrain_param, constraint in constraints:
                    if grad.grep_param_names(constrain_param):
                        constraint(constrain_param, grad)
                    else:
                        print "parameter didn't exist"
                    print constrain_param, " ", constraint
            if randomize:
                grad.randomize()
            if verbose:
                print grad
                grad.checkgrad(verbose=1)
            if not grad.checkgrad(verbose=True):
                gradchecking = False

    return gradchecking

from nose.tools import with_setup
class TestNoiseModels(object):
    """
    Generic model checker
    """
        self.N = 15
        self.D = 3
        self.X = np.random.rand(self.N, self.D)*10

        self.real_std = 0.1
        noise = np.random.randn(*self.X[:, 0].shape)*self.real_std
        self.Y = (np.sin(self.X[:, 0]*2*np.pi) + noise)[:, None]
        self.f = np.random.rand(self.N, 1)
        self.binary_Y = np.asarray(np.random.rand(self.N) > 0.5, dtype=np.int)[:, None]
        self.positive_Y = np.exp(self.Y.copy())
        tmp = np.round(self.X[:, 0]*3-3)[:, None] + np.random.randint(0,3, self.X.shape[0])[:, None]
        self.integer_Y = np.where(tmp > 0, tmp, 0)

        self.var = 0.2

        self.var = np.random.rand(1)

        #Make a bigger step as lower bound can be quite curved
        self.step = 1e-4

        self.Y = None
        self.f = None
        self.X = None

    def test_scale2_models(self):

        ####################################################
        # Constraint wrappers so we can just list them off #
        def constrain_fixed(regex, model):
            model[regex].constrain_fixed()

        def constrain_negative(regex, model):
            model[regex].constrain_negative()

        def constrain_positive(regex, model):
            model[regex].constrain_positive()

        def constrain_bounded(regex, model, lower, upper):
            """
            Used like: partial(constrain_bounded, lower=0, upper=1)
            """
            model[regex].constrain_bounded(lower, upper)

        """
        Dictionary where we nest models we would like to check
            Name: {
                "model": model_instance,
                "grad_params": {
                    "names": [names_of_params_we_want, to_grad_check],
                    "vals": [values_of_params, to_start_at],
                    "constrain": [constraint_wrappers, listed_here]
                    },
                "laplace": boolean_of_whether_model_should_work_for_laplace,
                "ep": boolean_of_whether_model_should_work_for_laplace,
                "link_f_constraints": [constraint_wrappers, listed_here]
                }
        """
        noise_models = {"Student_t_default": {
                            "model": GPy.likelihoods.StudentT(deg_free=5, sigma2=self.var),
                            "grad_params": {
                                "names": [".*t_scale2"],
                                "vals": [self.var],
                                "constraints": [(".*t_scale2", constrain_positive), (".*deg_free", constrain_fixed)]
                                #"constraints": [("t_scale2", constrain_positive), ("deg_free", partial(constrain_fixed, value=5))]
                                },
                            "laplace": True
                            },
                        "Student_t_1_var": {
                                "vals": [1.0],
                                },
                            },
                        "Student_t_small_deg_free": {
                            "model": GPy.likelihoods.StudentT(deg_free=1.5, sigma2=self.var),
                                },
                            },
                        "Student_t_small_var": {
                                "vals": [0.001],
                                },
                            },
                        "Student_t_large_var": {
                                "vals": [10.0],
                                },
                            },
                        "Student_t_approx_gauss": {
                            "model": GPy.likelihoods.StudentT(deg_free=1000, sigma2=self.var),
                                },
                            },
                        "Student_t_log": {
                            "model": GPy.likelihoods.StudentT(gp_link=link_functions.Log(), deg_free=5, sigma2=self.var),
                                },
                            },
                        "Gaussian_default": {
                            "model": GPy.likelihoods.Gaussian(variance=self.var),
                                "names": [".*variance"],
                                "constraints": [(".*variance", constrain_positive)]
                                },
                            "laplace": True,
                            "ep": False # FIXME: Should be True when we have it working again
                            },
                        #"Gaussian_log": {
                            #"model": GPy.likelihoods.gaussian(gp_link=link_functions.Log(), variance=self.var, D=self.D, N=self.N),
                            #"grad_params": {
                                #"names": ["noise_model_variance"],
                                #"vals": [self.var],
                                #"constraints": [constrain_positive]
                                #},
                            #"laplace": True
                            #},
                        #"Gaussian_probit": {
                            #"model": GPy.likelihoods.gaussian(gp_link=link_functions.Probit(), variance=self.var, D=self.D, N=self.N),
                                #},
                            #},
                        #"Gaussian_log_ex": {
                            #"model": GPy.likelihoods.gaussian(gp_link=link_functions.Log_ex_1(), variance=self.var, D=self.D, N=self.N),
                                #},
                            #},
                        "Bernoulli_default": {
                            "model": GPy.likelihoods.Bernoulli(),
                            "link_f_constraints": [partial(constrain_bounded, lower=0, upper=1)],
                            "Y": self.binary_Y,
                            },
                        "Exponential_default": {
                            "model": GPy.likelihoods.Exponential(),
                            "link_f_constraints": [constrain_positive],
                            "Y": self.positive_Y,
                        },
                        "Poisson_default": {
                            "model": GPy.likelihoods.Poisson(),
                            "Y": self.integer_Y,
                            "ep": False #Should work though...
                        }#,
                        #GAMMA needs some work!"Gamma_default": {
                            #"model": GPy.likelihoods.Gamma(),
                            #"link_f_constraints": [constrain_positive],
                            #"Y": self.positive_Y,
                        #}
                    }

        for name, attributes in noise_models.iteritems():
            model = attributes["model"]
            if "grad_params" in attributes:
                params = attributes["grad_params"]
                param_vals = params["vals"]
                param_names= params["names"]
                param_constraints = params["constraints"]
            else:
                params = []
                param_vals = []
                param_names = []
                constrain_positive = []
                param_constraints = [] # ??? TODO: Saul to Fix.
            if "link_f_constraints" in attributes:
                link_f_constraints = attributes["link_f_constraints"]
            else:
                link_f_constraints = []
            if "Y" in attributes:
                Y = attributes["Y"].copy()
            else:
                Y = self.Y.copy()
            if "f" in attributes:
                f = attributes["f"].copy()
            else:
                f = self.f.copy()
            if "laplace" in attributes:
                laplace = attributes["laplace"]
            else:
                laplace = False
            if "ep" in attributes:
                ep = attributes["ep"]
            else:
                ep = False

            #if len(param_vals) > 1:
                #raise NotImplementedError("Cannot support multiple params in likelihood yet!")

            #Required by all
            #Normal derivatives
            yield self.t_logpdf, model, Y, f
            yield self.t_dlogpdf_df, model, Y, f
            yield self.t_d2logpdf_df2, model, Y, f
            #Link derivatives
            yield self.t_dlogpdf_dlink, model, Y, f, link_f_constraints
            yield self.t_d2logpdf_dlink2, model, Y, f, link_f_constraints
            if laplace:
                #Laplace only derivatives
                yield self.t_d3logpdf_df3, model, Y, f
                yield self.t_d3logpdf_dlink3, model, Y, f, link_f_constraints
                #Params
                yield self.t_dlogpdf_dparams, model, Y, f, param_vals, param_names, param_constraints
                yield self.t_dlogpdf_df_dparams, model, Y, f, param_vals, param_names, param_constraints
                yield self.t_d2logpdf2_df2_dparams, model, Y, f, param_vals, param_names, param_constraints
                #Link params
                yield self.t_dlogpdf_link_dparams, model, Y, f, param_vals, param_names, param_constraints
                yield self.t_dlogpdf_dlink_dparams, model, Y, f, param_vals, param_names, param_constraints
                yield self.t_d2logpdf2_dlink2_dparams, model, Y, f, param_vals, param_names, param_constraints

                #laplace likelihood gradcheck
                yield self.t_laplace_fit_rbf_white, model, self.X, Y, f, self.step, param_vals, param_names, param_constraints
            if ep:
                #ep likelihood gradcheck
                yield self.t_ep_fit_rbf_white, model, self.X, Y, f, self.step, param_vals, param_names, param_constraints

    #############
    # dpdf_df's #
    @with_setup(setUp, tearDown)
    def t_logpdf(self, model, Y, f):
        print "\n{}".format(inspect.stack()[0][3])
        print model
        #print model._get_params()
        np.testing.assert_almost_equal(
                model.pdf(f.copy(), Y.copy()).prod(),
                               np.exp(model.logpdf(f.copy(), Y.copy()).sum())
                               )

    def t_dlogpdf_df(self, model, Y, f):
        self.description = "\n{}".format(inspect.stack()[0][3])
        logpdf = functools.partial(model.logpdf, y=Y)
        dlogpdf_df = functools.partial(model.dlogpdf_df, y=Y)
        grad = GradientChecker(logpdf, dlogpdf_df, f.copy(), 'g')
        grad.randomize()
        assert grad.checkgrad(verbose=1)

    def t_d2logpdf_df2(self, model, Y, f):
        d2logpdf_df2 = functools.partial(model.d2logpdf_df2, y=Y)
        grad = GradientChecker(dlogpdf_df, d2logpdf_df2, f.copy(), 'g')

    def t_d3logpdf_df3(self, model, Y, f):
        d3logpdf_df3 = functools.partial(model.d3logpdf_df3, y=Y)
        grad = GradientChecker(d2logpdf_df2, d3logpdf_df3, f.copy(), 'g')

    ##############
    # df_dparams #
    def t_dlogpdf_dparams(self, model, Y, f, params, params_names, param_constraints):
        assert (
                dparam_checkgrad(model.logpdf, model.dlogpdf_dtheta,
                    params, params_names, args=(f, Y), constraints=param_constraints,
                    randomize=False, verbose=True)
                )

    def t_dlogpdf_df_dparams(self, model, Y, f, params, params_names, param_constraints):
        assert (
                dparam_checkgrad(model.dlogpdf_df, model.dlogpdf_df_dtheta,
                )

    def t_d2logpdf2_df2_dparams(self, model, Y, f, params, params_names, param_constraints):
        assert (
                dparam_checkgrad(model.d2logpdf_df2, model.d2logpdf_df2_dtheta,
                )

    ################
    # dpdf_dlink's #
    def t_dlogpdf_dlink(self, model, Y, f, link_f_constraints):
        logpdf = functools.partial(model.logpdf_link, y=Y)
        dlogpdf_dlink = functools.partial(model.dlogpdf_dlink, y=Y)
        grad = GradientChecker(logpdf, dlogpdf_dlink, f.copy(), 'g')

        #Apply constraints to link_f values
        for constraint in link_f_constraints:
            constraint('g', grad)

        print grad

    def t_d2logpdf_dlink2(self, model, Y, f, link_f_constraints):
        d2logpdf_dlink2 = functools.partial(model.d2logpdf_dlink2, y=Y)
        grad = GradientChecker(dlogpdf_dlink, d2logpdf_dlink2, f.copy(), 'g')

        print grad

    def t_d3logpdf_dlink3(self, model, Y, f, link_f_constraints):
        d3logpdf_dlink3 = functools.partial(model.d3logpdf_dlink3, y=Y)
        grad = GradientChecker(d2logpdf_dlink2, d3logpdf_dlink3, f.copy(), 'g')

        print grad

    #################
    # dlink_dparams #
    def t_dlogpdf_link_dparams(self, model, Y, f, params, param_names, param_constraints):
        assert (
                dparam_checkgrad(model.logpdf_link, model.dlogpdf_link_dtheta,
                    params, param_names, args=(f, Y), constraints=param_constraints,
                )

    def t_dlogpdf_dlink_dparams(self, model, Y, f, params, param_names, param_constraints):
        assert (
                dparam_checkgrad(model.dlogpdf_dlink, model.dlogpdf_dlink_dtheta,
                )

    def t_d2logpdf2_dlink2_dparams(self, model, Y, f, params, param_names, param_constraints):
        assert (
                dparam_checkgrad(model.d2logpdf_dlink2, model.d2logpdf_dlink2_dtheta,
                )

    # laplace test #
    def t_laplace_fit_rbf_white(self, model, X, Y, f, step, param_vals, param_names, constraints):
        #Normalize
        Y = Y/Y.max()
        white_var = 1e-6
        kernel = GPy.kern.RBF(X.shape[1]) + GPy.kern.White(X.shape[1])
        laplace_likelihood = GPy.inference.latent_function_inference.Laplace()
        m = GPy.core.GP(X.copy(), Y.copy(), kernel, likelihood=model, inference_method=laplace_likelihood)
        m['.*white'].constrain_fixed(white_var)

        #Set constraints
        for constrain_param, constraint in constraints:
            constraint(constrain_param, m)

        print m

        #Set params
        for param_num in range(len(param_names)):
            name = param_names[param_num]
            m[name] = param_vals[param_num]

        #m.optimize(max_iters=8)
        print m
        #if not m.checkgrad(step=step):
            #m.checkgrad(verbose=1, step=step)
            #NOTE this test appears to be stochastic for some likelihoods (student t?)
            # appears to all be working in test mode right now...
        #if isinstance(model, GPy.likelihoods.StudentT):
        assert m.checkgrad(verbose=1, step=step)

    ###########
    # EP test #
    def t_ep_fit_rbf_white(self, model, X, Y, f, step, param_vals, param_names, constraints):
        ep_inf = GPy.inference.latent_function_inference.EP()
        m = GPy.core.GP(X.copy(), Y.copy(), kernel=kernel, likelihood=model, inference_method=ep_inf)

            constraints[param_num](name, m)

        print m

class LaplaceTests(unittest.TestCase):
    """
    Specific likelihood tests, not general enough for the above tests
    """

        self.N = 5
        self.D = 3

        self.stu_t = GPy.likelihoods.StudentT(deg_free=5, sigma2=self.var)
        #TODO: gaussians with on Identity link. self.gauss = GPy.likelihoods.Gaussian(gp_link=link_functions.Log(), variance=self.var)
        self.gauss = GPy.likelihoods.Gaussian(variance=self.var)

        self.step = 1e-6

        self.stu_t = None
        self.gauss = None

    def test_gaussian_d2logpdf_df2_2(self):

        self.N = 2
        self.D = 1
        self.X = np.linspace(0, self.D, self.N)[:, None]
        self.real_std = 0.2
        noise = np.random.randn(*self.X.shape)*self.real_std
        self.Y = np.sin(self.X*2*np.pi) + noise

        dlogpdf_df = functools.partial(self.gauss.dlogpdf_df, y=self.Y)
        d2logpdf_df2 = functools.partial(self.gauss.d2logpdf_df2, y=self.Y)
        grad = GradientChecker(dlogpdf_df, d2logpdf_df2, self.f.copy(), 'g')

        self.assertTrue(grad.checkgrad(verbose=1))

    def test_laplace_log_likelihood(self):
        debug = False
        real_std = 0.1
        initial_var_guess = 0.5

        #Start a function, any function
        X = np.linspace(0.0, np.pi*2, 100)[:, None]
        Y = np.sin(X) + np.random.randn(*X.shape)*real_std
        #Yc = Y.copy()
        #Yc[75:80] += 1
        kernel1 = GPy.kern.RBF(X.shape[1]) + GPy.kern.White(X.shape[1])
        #FIXME: Make sure you can copy kernels when params is fixed
        #kernel2 = kernel1.copy()
        kernel2 = GPy.kern.RBF(X.shape[1]) + GPy.kern.White(X.shape[1])

        gauss_distr1 = GPy.likelihoods.Gaussian(variance=initial_var_guess)
        exact_inf = GPy.inference.latent_function_inference.ExactGaussianInference()
        m1 = GPy.core.GP(X, Y.copy(), kernel=kernel1, likelihood=gauss_distr1, inference_method=exact_inf)
        m1['.*white'].constrain_fixed(1e-6)
        m1['.*rbf.variance'] = initial_var_guess
        m1['.*rbf.variance'].constrain_bounded(1e-4, 10)
        m1.randomize()

        gauss_distr2 = GPy.likelihoods.Gaussian(variance=initial_var_guess)
        laplace_inf = GPy.inference.latent_function_inference.Laplace()
        m2 = GPy.core.GP(X, Y.copy(), kernel=kernel2, likelihood=gauss_distr2, inference_method=laplace_inf)
        m2['.*white'].constrain_fixed(1e-6)
        m2['.*rbf.variance'].constrain_bounded(1e-4, 10)
        m2.randomize()

        if debug:
            print m1
            print m2
        optimizer = 'scg'
        print "Gaussian"
        m1.optimize(optimizer, messages=debug)
        print "Laplace Gaussian"
        m2.optimize(optimizer, messages=debug)
        if debug:
            print m1
            print m2

        m2[:] = m1[:]

        #Predict for training points to get posterior mean and variance
        post_mean, post_var = m1.predict(X)
        post_mean_approx, post_var_approx, = m2.predict(X)

        if debug:
            import pylab as pb
            pb.figure(5)
            pb.title('posterior means')
            pb.scatter(X, post_mean, c='g')
            pb.scatter(X, post_mean_approx, c='r', marker='x')

            pb.figure(6)
            pb.title('plot_f')
            m1.plot_f(fignum=6)
            m2.plot_f(fignum=6)
            fig, axes = pb.subplots(2, 1)
            fig.suptitle('Covariance matricies')
            a1 = pb.subplot(121)
            a1.matshow(m1.likelihood.covariance_matrix)
            a2 = pb.subplot(122)
            a2.matshow(m2.likelihood.covariance_matrix)

            pb.figure(8)
            pb.scatter(X, m1.likelihood.Y, c='g')
            pb.scatter(X, m2.likelihood.Y, c='r', marker='x')

        #Check Y's are the same
        np.testing.assert_almost_equal(m1.Y, m2.Y, decimal=5)
        #Check marginals are the same
        np.testing.assert_almost_equal(m1.log_likelihood(), m2.log_likelihood(), decimal=2)
        #Check marginals are the same with random

        #Check they are checkgradding
        #m1.checkgrad(verbose=1)
        #m2.checkgrad(verbose=1)
        self.assertTrue(m1.checkgrad(verbose=True))
        self.assertTrue(m2.checkgrad(verbose=True))

    print "Running unit tests"

try:
except:
    pass
#import numpy as np
#import Tango
#from base_plots import gpplot, x_frame1D, x_frame2D

def plot_optimizer(optimizer):
    if optimizer.trace == None:
        print "No trace present so I can't plot it. Please check that the optimizer actually supplies a trace."
    else:
        pb.figure()
        pb.plot(optimizer.trace)
        pb.xlabel('Iteration')
        pb.ylabel('f(x)')

def plot_sgd_traces(optimizer):
    pb.figure()
    pb.subplot(211)
    pb.title('Parameters')
    for k in optimizer.param_traces.keys():
        pb.plot(optimizer.param_traces[k], label=k)
    pb.legend(loc=0)
    pb.subplot(212)
    pb.title('Objective function')
    pb.plot(optimizer.fopt_trace)

four_over_tau = 2./np.pi

class MLP(Kern):
    """

    Multi layer perceptron kernel (also known as arc sine kernel or neural network kernel)

    .. math::

          k(x,y) = \\sigma^{2}\\frac{2}{\\pi }  \\text{asin} \\left ( \\frac{ \\sigma_w^2 x^\\top y+\\sigma_b^2}{\\sqrt{\\sigma_w^2x^\\top x + \\sigma_b^2 + 1}\\sqrt{\\sigma_w^2 y^\\top y \\sigma_b^2 +1}} \\right )

    :param variance: the variance :math:`\sigma^2`
    :param weight_variance: the vector of the variances of the prior over input weights in the neural network :math:`\sigma^2_w`
    :type weight_variance: array or list of the appropriate size (or float if there is only one weight variance parameter)
    :param bias_variance: the variance of the prior over bias parameters :math:`\sigma^2_b`
    :param ARD: Auto Relevance Determination. If equal to "False", the kernel is isotropic (ie. one weight variance parameter \sigma^2_w), otherwise there is one weight variance parameter per dimension.
    :rtype: Kernpart object

    """

    def __init__(self, input_dim, variance=1., weight_variance=1., bias_variance=100., active_dims=None, name='mlp'):
        super(MLP, self).__init__(input_dim, active_dims, name)
        self.weight_variance = Param('weight_variance', weight_variance, Logexp())
        self.bias_variance = Param('bias_variance', bias_variance, Logexp())
        self.link_parameters(self.variance, self.weight_variance, self.bias_variance)

        self._K_computations(X, X2)
        return self.variance*self._K_dvar

        """Compute the diagonal of the covariance matrix for X."""
        self._K_diag_computations(X)
        return self.variance*self._K_diag_dvar

        """Derivative of the covariance with respect to the parameters."""
        self.variance.gradient = np.sum(self._K_dvar*dL_dK)

        denom3 = self._K_denom**3
        base = four_over_tau*self.variance/np.sqrt(1-self._K_asin_arg*self._K_asin_arg)
        base_cov_grad = base*dL_dK

            vec = np.diag(self._K_inner_prod)
            self.weight_variance.gradient = ((self._K_inner_prod/self._K_denom
                           -.5*self._K_numer/denom3
                           *(np.outer((self.weight_variance*vec+self.bias_variance+1.), vec)
                             +np.outer(vec,(self.weight_variance*vec+self.bias_variance+1.))))*base_cov_grad).sum()
            self.bias_variance.gradient = ((1./self._K_denom
                           *((vec[None, :]+vec[:, None])*self.weight_variance
                           +2.*self.bias_variance + 2.))*base_cov_grad).sum()
        else:
            vec1 = (X*X).sum(1)
            vec2 = (X2*X2).sum(1)
                           *(np.outer((self.weight_variance*vec1+self.bias_variance+1.), vec2) + np.outer(vec1, self.weight_variance*vec2 + self.bias_variance+1.)))*base_cov_grad).sum()
                           *((vec1[:, None]+vec2[None, :])*self.weight_variance
                             + 2*self.bias_variance + 2.))*base_cov_grad).sum()

    def update_gradients_diag(self, X):
        raise NotImplementedError, "TODO"

    def gradients_X(self, dL_dK, X, X2):
        """Derivative of the covariance matrix with respect to X"""
        arg = self._K_asin_arg
        numer = self._K_numer
        denom = self._K_denom
        denom3 = denom*denom*denom
            vec2 = (X2*X2).sum(1)*self.weight_variance+self.bias_variance + 1.
            return four_over_tau*self.weight_variance*self.variance*((X2[None, :, :]/denom[:, :, None] - vec2[None, :, None]*X[:, None, :]*(numer/denom3)[:, :, None])*(dL_dK/np.sqrt(1-arg*arg))[:, :, None]).sum(1)
        else:
            vec = (X*X).sum(1)*self.weight_variance+self.bias_variance + 1.
            return 2*four_over_tau*self.weight_variance*self.variance*((X[None, :, :]/denom[:, :, None] - vec[None, :, None]*X[:, None, :]*(numer/denom3)[:, :, None])*(dL_dK/np.sqrt(1-arg*arg))[:, :, None]).sum(1)

        """Gradient of diagonal of covariance with respect to X"""
        arg = self._K_diag_asin_arg
        denom = self._K_diag_denom
        #numer = self._K_diag_numer
        return four_over_tau*2.*self.weight_variance*self.variance*X*(1./denom*(1. - arg)*dL_dKdiag/(np.sqrt(1-arg*arg)))[:, None]

    def _K_computations(self, X, X2):
        """Pre-computations for the covariance matrix (used for computing the covariance and its gradients."""
            self._K_inner_prod = np.dot(X,X.T)
            self._K_numer = self._K_inner_prod*self.weight_variance + self.bias_variance
            vec = np.diag(self._K_numer) + 1.
            self._K_denom = np.sqrt(np.outer(vec,vec))
        else:
            self._K_inner_prod = np.dot(X,X2.T)
            vec1 = (X*X).sum(1)*self.weight_variance + self.bias_variance + 1.
            vec2 = (X2*X2).sum(1)*self.weight_variance + self.bias_variance + 1.
            self._K_denom = np.sqrt(np.outer(vec1,vec2))
        self._K_asin_arg = self._K_numer/self._K_denom
        self._K_dvar = four_over_tau*np.arcsin(self._K_asin_arg)

    def _K_diag_computations(self, X):
        """Pre-computations concerning the diagonal terms (used for computation of diagonal and its gradients)."""
        self._K_diag_numer = (X*X).sum(1)*self.weight_variance + self.bias_variance
        self._K_diag_denom = self._K_diag_numer+1.
        self._K_diag_asin_arg = self._K_diag_numer/self._K_diag_denom
        self._K_diag_dvar = four_over_tau*np.arcsin(self._K_diag_asin_arg)

from ..util.univariate_Gaussian import std_norm_pdf, std_norm_cdf

class Bernoulli(Likelihood):
    """
    Bernoulli likelihood

    .. math::
        p(y_{i}|\\lambda(f_{i})) = \\lambda(f_{i})^{y_{i}}(1-f_{i})^{1-y_{i}}

    .. Note::
        Y takes values in either {-1, 1} or {0, 1}.
        link function should have the domain [0, 1], e.g. probit (default) or Heaviside

    .. See also::
        likelihood.py, for the parent class
    """
            gp_link = link_functions.Probit()

        super(Bernoulli, self).__init__(gp_link, 'Bernoulli')

        if isinstance(gp_link , (link_functions.Heaviside, link_functions.Probit)):
            self.log_concave = True

    def _preprocess_values(self, Y):
        """
        Check if the values of the observations correspond to the values
        assumed by the likelihood function.

        ..Note:: Binary classification algorithm works better with classes {-1, 1}
        """
        Y_prep = Y.copy()
        Y1 = Y[Y.flatten()==1].size
        Y2 = Y[Y.flatten()==0].size
        assert Y1 + Y2 == Y.size, 'Bernoulli likelihood is meant to be used only with outputs in {0, 1}.'
        Y_prep[Y.flatten() == 0] = -1
        return Y_prep

    def moments_match_ep(self, Y_i, tau_i, v_i):
        """
        Moments match of the marginal approximation in EP algorithm

        :param i: number of observation (int)
        :param tau_i: precision of the cavity distribution (float)
        :param v_i: mean/variance of the cavity distribution (float)
        """
        if Y_i == 1:
            sign = 1.
        elif Y_i == 0 or Y_i == -1:
            sign = -1
        else:
            raise ValueError("bad value for Bernoulli observation (0, 1)")
        if isinstance(self.gp_link, link_functions.Probit):
            z = sign*v_i/np.sqrt(tau_i**2 + tau_i)
            Z_hat = std_norm_cdf(z)
            phi = std_norm_pdf(z)
            mu_hat = v_i/tau_i + sign*phi/(Z_hat*np.sqrt(tau_i**2 + tau_i))
            sigma2_hat = 1./tau_i - (phi/((tau_i**2+tau_i)*Z_hat))*(z+phi/Z_hat)

        elif isinstance(self.gp_link, link_functions.Heaviside):
            a = sign*v_i/np.sqrt(tau_i)
            Z_hat = std_norm_cdf(a)
            N = std_norm_pdf(a)
            mu_hat = v_i/tau_i + sign*N/Z_hat/np.sqrt(tau_i)
            sigma2_hat = (1. - a*N/Z_hat - np.square(N/Z_hat))/tau_i
        else:
            #TODO: do we want to revert to numerical quadrature here?
            raise ValueError("Exact moment matching not available for link {}".format(self.gp_link.__name__))

        return Z_hat, mu_hat, sigma2_hat

            return stats.norm.cdf(mu/np.sqrt(1+variance))

            return stats.norm.cdf(mu/np.sqrt(variance))

        else:

    def predictive_variance(self, mu, variance, pred_mean, Y_metadata=None):

        if isinstance(self.gp_link, link_functions.Heaviside):
            return 0.
        else:
            return np.nan

        """
        Likelihood function given inverse link of f.

        .. math::
            p(y_{i}|\\lambda(f_{i})) = \\lambda(f_{i})^{y_{i}}(1-f_{i})^{1-y_{i}}

        :param inv_link_f: latent variables inverse link of f.
        :param Y_metadata: Y_metadata not used in bernoulli

        .. Note:
            Each y_i must be in {0, 1}
        """
        #objective = (inv_link_f**y) * ((1.-inv_link_f)**(1.-y))
        return np.where(y, inv_link_f, 1.-inv_link_f)

        """
        Log Likelihood function given inverse link of f.

        .. math::
            \\ln p(y_{i}|\\lambda(f_{i})) = y_{i}\\log\\lambda(f_{i}) + (1-y_{i})\\log (1-f_{i})

        :returns: log likelihood evaluated at points inverse link of f.
        """
        #objective = y*np.log(inv_link_f) + (1.-y)*np.log(inv_link_f)
        p = np.where(y==1, inv_link_f, 1.-inv_link_f)
        return np.log(np.clip(p, 1e-6 ,np.inf))

        """
        Gradient of the pdf at y, given inverse link of f w.r.t inverse link of f.

        .. math::
            \\frac{d\\ln p(y_{i}|\\lambda(f_{i}))}{d\\lambda(f)} = \\frac{y_{i}}{\\lambda(f_{i})} - \\frac{(1 - y_{i})}{(1 - \\lambda(f_{i}))}

        :returns: gradient of log likelihood evaluated at points inverse link of f.
        """
        #grad = (y/inv_link_f) - (1.-y)/(1-inv_link_f)
        #grad = np.where(y, 1./inv_link_f, -1./(1-inv_link_f))
        ff = np.clip(inv_link_f, 1e-6, 1-1e-6)
        denom = np.where(y, ff, -(1-ff))
        return 1./denom

        """
        Hessian at y, given inv_link_f, w.r.t inv_link_f the hessian will be 0 unless i == j
        i.e. second derivative logpdf at y given inverse link of f_i and inverse link of f_j  w.r.t inverse link of f_i and inverse link of f_j.

        .. math::
            \\frac{d^{2}\\ln p(y_{i}|\\lambda(f_{i}))}{d\\lambda(f)^{2}} = \\frac{-y_{i}}{\\lambda(f)^{2}} - \\frac{(1-y_{i})}{(1-\\lambda(f))^{2}}

        :returns: Diagonal of log hessian matrix (second derivative of log likelihood evaluated at points inverse link of f.

        .. Note::
            (the distribution for y_i depends only on inverse link of f_i not on inverse link of f_(j!=i)
        """
        #d2logpdf_dlink2 = -y/(inv_link_f**2) - (1-y)/((1-inv_link_f)**2)
        #d2logpdf_dlink2 = np.where(y, -1./np.square(inv_link_f), -1./np.square(1.-inv_link_f))
        arg = np.where(y, inv_link_f, 1.-inv_link_f)
        ret =  -1./np.square(np.clip(arg, 1e-3, np.inf))
        if np.any(np.isinf(ret)):
            stop
        return ret

        """
        Third order derivative log-likelihood function at y given inverse link of f w.r.t inverse link of f

        .. math::
            \\frac{d^{3} \\ln p(y_{i}|\\lambda(f_{i}))}{d^{3}\\lambda(f)} = \\frac{2y_{i}}{\\lambda(f)^{3}} - \\frac{2(1-y_{i}}{(1-\\lambda(f))^{3}}

        :param inv_link_f: latent variables passed through inverse link of f.
        :returns: third derivative of log likelihood evaluated at points inverse_link(f)
        """
        #d3logpdf_dlink3 = 2*(y/(inv_link_f**3) - (1-y)/((1-inv_link_f)**3))
        state = np.seterr(divide='ignore')
        # TODO check y \in {0, 1} or {-1, 1}
        d3logpdf_dlink3 = np.where(y, 2./(inv_link_f**3), -2./((1.-inv_link_f)**3))
        np.seterr(**state)
        return d3logpdf_dlink3

        """

        """
        ns = np.ones_like(gp, dtype=int)
        Ysim = np.random.binomial(ns, self.gp_link.transf(gp))

    def exact_inference_gradients(self, dL_dKdiag,Y_metadata=None):
        pass

from ...util.linalg import mdot, jitchol, backsub_both_sides, tdot, dtrtrs, dtrtri, dpotri, dpotrs, symmetrify, DSYR

class EPDTC(LatentFunctionInference):
    def __init__(self, epsilon=1e-6, eta=1., delta=1., limit=1):

        pass

        """

        """
        if (N>=D):
            return Y
        else:

    def inference(self, kern, X, Z, likelihood, Y, Y_metadata=None):

        Kmm = kern.K(Z)
        Kmn = kern.K(Z,X)

            mu, Sigma, mu_tilde, tau_tilde, Z_hat = self._ep_approximation = self.expectation_propagation(Kmm, Kmn, Y, likelihood, Y_metadata)
        else:

            psi1 = Kmn.T#kern.psi1(Z, X)
            psi2 = kern.psi2(Z, X)
        else:
            psi1 = Kmn.T#kern.K(X, Z)
            psi2 = None

        #see whether we're using variational uncertain inputs

        #beta = 1./np.fmax(likelihood.gaussian_variance(Y_metadata), 1e-6)
        beta = tau_tilde
        VVT_factor = beta[:,None]*mu_tilde[:,None]
        trYYT = self.get_trYYT(mu_tilde[:,None])

        Lm = jitchol(Kmm)

        # The rather complex computations of A
                psi2_beta = psi2 * (beta.flatten().reshape(num_data, 1, 1)).sum(0)
            else:
                psi2_beta = psi2.sum(0) * beta
        else:
                tmp = psi1 * (np.sqrt(beta.reshape(num_data, 1)))
            else:

        # factor B

        delit = -0.5 * DBi_plus_BiPBi
        delit += -0.5 * B * output_dim
        delit += output_dim * np.eye(num_inducing)
        dL_dKmm = backsub_both_sides(Lm, delit)

            psi0, A, LB, trYYT, data_fit, VVT_factor)

            data_fit, num_data, output_dim, trYYT, mu_tilde[:,None])

        dL_dthetaL = 0#likelihood.exact_inference_gradients(dL_dR,Y_metadata)

        else:

        else:
            psi1V = np.dot(mu_tilde[:,None].T*beta, psi1).T

    def expectation_propagation(self, Kmm, Kmn, Y, likelihood, Y_metadata):

        KmnKnm = np.dot(Kmn,Kmn.T)
        Lmi = dtrtrs(Lm,np.eye(Lm.shape[0]))[0] #chol_inv(Lm)
        Kmmi = np.dot(Lmi.T,Lmi)
        KmmiKmn = np.dot(Kmmi,Kmn)
        Qnn_diag = np.sum(Kmn*KmmiKmn,-2)
        LLT0 = Kmm.copy()

        LLT = Kmm.copy() #Sigma = K.copy()
        Sigma_diag = Qnn_diag.copy()

        else:

                tau_cav = 1./Sigma_diag[i] - self.eta*tau_tilde[i]
                v_cav = mu[i]/Sigma_diag[i] - self.eta*v_tilde[i]
                delta_tau = self.delta/self.eta*(1./sigma2_hat[i] - 1./Sigma_diag[i])
                delta_v = self.delta/self.eta*(mu_hat[i]/sigma2_hat[i] - mu[i]/Sigma_diag[i])

                #DSYR(Sigma, Sigma[:,i].copy(), -delta_tau/(1.+ delta_tau*Sigma[i,i]))
                DSYR(LLT,Kmn[:,i].copy(),delta_tau)
                L = jitchol(LLT)

                V,info = dtrtrs(L,Kmn,lower=1)
                Sigma_diag = np.sum(V*V,-2)
                si = np.sum(V.T*V[:,i],-1)
                mu += (delta_v-delta_tau*mu[i])*si
                #mu = np.dot(Sigma, v_tilde)

            LLT = LLT0 + np.dot(Kmn*tau_tilde[None,:],Kmn.T)
            L = jitchol(LLT)
            V,info = dtrtrs(L,Kmn,lower=1)
            V2,info = dtrtrs(L.T,V,lower=0)
            #Sigma_diag = np.sum(V*V,-2)
            #Knmv_tilde = np.dot(Kmn,v_tilde)
            #mu = np.dot(V2.T,Knmv_tilde)
            Sigma = np.dot(V2.T,V2)

            tau_diff = 0
            v_diff = 0

    dL_dpsi0 = -0.5 * output_dim * (beta[:,None] * np.ones([num_data, 1])).flatten()
            dL_dpsi2 = beta[:, None, None] * dL_dpsi2_beta[None, :, :]
        else:
            dL_dpsi1 += 2.*np.dot(dL_dpsi2_beta, (psi1 * beta.reshape(num_data, 1)).T).T
    else:
            # repeat for each of the N psi_2 matrices
            dL_dpsi2 = np.repeat(dL_dpsi2[None, :, :], num_data, axis=0)
        else:

def _compute_dL_dR(likelihood, het_noise, uncertain_inputs, LB, _LBi_Lmi_psi1Vf, DBi_plus_BiPBi, Lm, A, psi0, psi1, beta, data_fit, num_data, output_dim, trYYT, Y):
        else:

            dL_dR = -0.5 * beta + 0.5 * (beta*Y)**2

    else:

def _compute_log_marginal_likelihood(likelihood, num_data, output_dim, beta, het_noise, psi0, A, LB, trYYT, data_fit,Y):
        lik_1 = -0.5 * num_data * output_dim * np.log(2. * np.pi) + 0.5 * np.sum(np.log(beta)) - 0.5 * np.sum(beta * np.square(Y).sum(axis=-1))
    else:

from gp import GP
from parameterization.param import Param
from ..inference.latent_function_inference import var_dtc

from GPy.inference.latent_function_inference.posterior import Posterior
from GPy.inference.optimization.stochastics import SparseGPStochastics,\
    SparseGPMissing
#no stochastics.py file added! from GPy.inference.optimization.stochastics import SparseGPStochastics,\
    #SparseGPMissing
logger = logging.getLogger("sparse gp")

class SparseGP(GP):
    """
    A general purpose Sparse GP model

    these.

    """

    def __init__(self, X, Y, Z, kernel, likelihood, inference_method=None,
                 name='sparse gp', Y_metadata=None, normalizer=False):
        #pick a sensible inference method
            if isinstance(likelihood, likelihoods.Gaussian):
                inference_method = var_dtc.VarDTC(limit=1 if not self.missing_data else Y.shape[1])
            else:
                #inference_method = ??
                raise NotImplementedError, "what to do what to do?"
            print "defaulting to ", inference_method, "for latent function inference"

        self.num_inducing = Z.shape[0]

        GP.__init__(self, X, Y, kernel, likelihood, inference_method=inference_method, name=name, Y_metadata=Y_metadata, normalizer=normalizer)

        logger.info("Adding Z as parameter")
        self.link_parameter(self.Z, index=0)

        return isinstance(self.X, VariationalPosterior)

        self.posterior, self._log_marginal_likelihood, self.grad_dict = self.inference_method.inference(self.kern, self.X, self.Z, self.likelihood, self.Y, self.Y_metadata)

            #gradients wrt kernel
            dL_dKmm = self.grad_dict['dL_dKmm']
            self.kern.update_gradients_full(dL_dKmm, self.Z, None)
            kerngrad = self.kern.gradient.copy()
            self.kern.update_gradients_expectations(variational_posterior=self.X,
                                                    Z=self.Z,
                                                    dL_dpsi0=self.grad_dict['dL_dpsi0'],
                                                    dL_dpsi1=self.grad_dict['dL_dpsi1'],
                                                    dL_dpsi2=self.grad_dict['dL_dpsi2'])
            self.kern.gradient += kerngrad

            #gradients wrt Z
            self.Z.gradient = self.kern.gradients_X(dL_dKmm, self.Z)
            self.Z.gradient += self.kern.gradients_Z_expectations(
                               self.grad_dict['dL_dpsi0'],
                               self.grad_dict['dL_dpsi1'],
                               self.grad_dict['dL_dpsi2'],
                               Z=self.Z,
                               variational_posterior=self.X)
        else:
            self.kern.update_gradients_diag(self.grad_dict['dL_dKdiag'], self.X)
            self.kern.update_gradients_full(self.grad_dict['dL_dKnm'], self.X, self.Z)
            kerngrad += self.kern.gradient
            self.kern.update_gradients_full(self.grad_dict['dL_dKmm'], self.Z, None)
            self.Z.gradient = self.kern.gradients_X(self.grad_dict['dL_dKmm'], self.Z)
            self.Z.gradient += self.kern.gradients_X(self.grad_dict['dL_dKnm'].T, self.Z, self.X)

    def _raw_predict(self, Xnew, full_cov=False, kern=None):
        """
        Make a prediction for the latent function values
        """

        if kern is None: kern = self.kern

        if not isinstance(Xnew, VariationalPosterior):
            Kx = kern.K(self.Z, Xnew)
            mu = np.dot(Kx.T, self.posterior.woodbury_vector)
            if full_cov:
                Kxx = kern.K(Xnew)
                if self.posterior.woodbury_inv.ndim == 2:
                    var = Kxx - np.dot(Kx.T, np.dot(self.posterior.woodbury_inv, Kx))
                elif self.posterior.woodbury_inv.ndim == 3:
                    var = Kxx[:,:,None] - np.tensordot(np.dot(np.atleast_3d(self.posterior.woodbury_inv).T, Kx).T, Kx, [1,0]).swapaxes(1,2)
                var = var
            else:
                Kxx = kern.Kdiag(Xnew)
                var = (Kxx - np.sum(np.dot(np.atleast_3d(self.posterior.woodbury_inv).T, Kx) * Kx[None,:,:], 1)).T
        else:
            Kx = kern.psi1(self.Z, Xnew)
            mu = np.dot(Kx, self.posterior.woodbury_vector)
                raise NotImplementedError, "TODO"
            else:
                Kxx = kern.psi0(self.Z, Xnew)
                psi2 = kern.psi2(self.Z, Xnew)
                var = Kxx - np.sum(np.sum(psi2 * Kmmi_LmiBLmi[None, :, :], 1), 1)

# #Copyright (c) 2012, GPy authors (see AUTHORS.txt).

try:
except:
    pass

def ax_default(fignum, ax):
        fig = pb.figure(fignum)
    else:
    return fig, ax

def meanplot(x, mu, color=Tango.colorsHex['darkBlue'], ax=None, fignum=None, linewidth=2,**kw):
    _, axes = ax_default(fignum, ax)
    return axes.plot(x,mu,color=color,linewidth=linewidth,**kw)

def gpplot(x, mu, lower, upper, edgecol=Tango.colorsHex['darkBlue'], fillcol=Tango.colorsHex['lightBlue'], ax=None, fignum=None, **kwargs):

    mu = mu.flatten()
    x = x.flatten()
    lower = lower.flatten()
    upper = upper.flatten()

    plots = []

    #here's the mean
    plots.append(meanplot(x, mu, edgecol, axes))

    #here's the box
    kwargs['linewidth']=0.5
    if not 'alpha' in kwargs.keys():
        kwargs['alpha'] = 0.3
    plots.append(axes.fill(np.hstack((x,x[::-1])),np.hstack((upper,lower[::-1])),color=fillcol,**kwargs))

    #this is the edge:
    plots.append(meanplot(x, upper,color=edgecol,linewidth=0.2,ax=axes))
    plots.append(meanplot(x, lower,color=edgecol,linewidth=0.2,ax=axes))

def removeRightTicks(ax=None):
    ax = ax or pb.gca()
    for i, line in enumerate(ax.get_yticklines()):
        if i%2 == 1:   # odd indices
            line.set_visible(False)

def removeUpperTicks(ax=None):
    for i, line in enumerate(ax.get_xticklines()):

def fewerXticks(ax=None,divideby=2):
    ax.set_xticks(ax.get_xticks()[::divideby])

def align_subplots(N,M,xlim=None, ylim=None):
    """make all of the subplots have the same limits, turn off unnecessary ticks"""
    #find sensible xlim,ylim
    if xlim is None:
        xlim = [np.inf,-np.inf]
        for i in range(N*M):
            pb.subplot(N,M,i+1)
            xlim[0] = min(xlim[0],pb.xlim()[0])
            xlim[1] = max(xlim[1],pb.xlim()[1])
    if ylim is None:
        ylim = [np.inf,-np.inf]
            ylim[0] = min(ylim[0],pb.ylim()[0])
            ylim[1] = max(ylim[1],pb.ylim()[1])

    for i in range(N*M):
        pb.subplot(N,M,i+1)
        pb.xlim(xlim)
        pb.ylim(ylim)
        if (i)%M:
            pb.yticks([])
        else:
            removeRightTicks()
        if i<(M*(N-1)):
            pb.xticks([])
        else:
            removeUpperTicks()

def align_subplot_array(axes,xlim=None, ylim=None):
    """
    Make all of the axes in the array hae the same limits, turn off unnecessary ticks
    use pb.subplots() to get an array of axes
    """
        for ax in axes.flatten():
            xlim[0] = min(xlim[0],ax.get_xlim()[0])
            xlim[1] = max(xlim[1],ax.get_xlim()[1])
            ylim[0] = min(ylim[0],ax.get_ylim()[0])
            ylim[1] = max(ylim[1],ax.get_ylim()[1])

    N,M = axes.shape
    for i,ax in enumerate(axes.flatten()):
        ax.set_xlim(xlim)
        ax.set_ylim(ylim)
        if (i)%M:
            ax.set_yticks([])
        else:
            removeRightTicks(ax)
            ax.set_xticks([])
        else:
            removeUpperTicks(ax)

def x_frame1D(X,plot_limits=None,resolution=None):
    """
    Internal helper function for making plots, returns a set of input values to plot as well as lower and upper limits
    """
    assert X.shape[1] ==1, "x_frame1D is defined for one-dimensional inputs"
        xmin,xmax = X.min(0),X.max(0)
        xmin, xmax = xmin-0.2*(xmax-xmin), xmax+0.2*(xmax-xmin)
    elif len(plot_limits)==2:
        xmin, xmax = plot_limits
    else:
        raise ValueError, "Bad limits for plotting"

    Xnew = np.linspace(xmin,xmax,resolution or 200)[:,None]
    return Xnew, xmin, xmax

def x_frame2D(X,plot_limits=None,resolution=None):
    """
    """
    assert X.shape[1] ==2, "x_frame2D is defined for two-dimensional inputs"
    else:

    resolution = resolution or 50
    xx,yy = np.mgrid[xmin[0]:xmax[0]:1j*resolution,xmin[1]:xmax[1]:1j*resolution]
    Xnew = np.vstack((xx.flatten(),yy.flatten())).T
    return Xnew, xx, yy, xmin, xmax

# One trigger in init
_trigger_start = -1

class ParamTestParent(Parameterized):
    parent_changed_count = _trigger_start
        self.parent_changed_count += 1

class ParameterizedTest(Parameterized):
    # One trigger after initialization
    params_changed_count = _trigger_start
        self.params_changed_count += 1

class Test(unittest.TestCase):

        self.parent = ParamTestParent('test parent')
        self.par = ParameterizedTest('test model')
        self.par2 = ParameterizedTest('test model 2')
        self.p = Param('test parameter', numpy.random.normal(1,2,(10,3)))

        self.par.link_parameter(self.p)
        self.par.link_parameter(Param('test1', numpy.random.normal(0,1,(1,))))
        self.par.link_parameter(Param('test2', numpy.random.normal(0,1,(1,))))

        self.par2.link_parameter(Param('par2 test1', numpy.random.normal(0,1,(1,))))
        self.par2.link_parameter(Param('par2 test2', numpy.random.normal(0,1,(1,))))

        self.parent.link_parameter(self.par)
        self.parent.link_parameter(self.par2)

        self._observer_triggered = None
        self._trigger_count = 0
        self._first = None
        self._second = None

    def _trigger(self, me, which):
        self._observer_triggered = which
        self._trigger_count += 1
        if self._first is not None:
            self._second = self._trigger
        else:
            self._first = self._trigger

    def _trigger_priority(self, me, which):
            self._second = self._trigger_priority
        else:
            self._first = self._trigger_priority

    def test_observable(self):
        self.par.add_observer(self, self._trigger, -1)
        self.assertEqual(self.par.params_changed_count, 0, 'no params changed yet')
        self.assertEqual(self.par.params_changed_count, self.parent.parent_changed_count, 'parent should be triggered as often as param')

        self.p[0,1] = 3 # trigger observers
        self.assertIs(self._observer_triggered, self.p, 'observer should have triggered')
        self.assertEqual(self._trigger_count, 1, 'observer should have triggered once')
        self.assertEqual(self.par.params_changed_count, 1, 'params changed once')

        self.par.remove_observer(self)
        self.p[0,1] = 4
        self.assertIs(self._observer_triggered, self.p, 'observer should not have triggered')
        self.assertEqual(self.par.params_changed_count, 2, 'params changed second')

        self.assertEqual(self._trigger_count, 2, 'observer should have triggered once')
        self.assertEqual(self.par.params_changed_count, 3, 'params changed second')

        self.par.remove_observer(self, self._trigger)
        self.p[0,1] = 3
        self.assertEqual(self.par.params_changed_count, 4, 'params changed second')

    def test_set_params(self):
        self.par.param_array[:] = 1
        self.par._trigger_params_changed()
        self.assertEqual(self.par.params_changed_count, 1, 'now params changed')
        self.assertEqual(self.parent.parent_changed_count, self.par.params_changed_count)

        self.par.param_array[:] = 2
        self.assertEqual(self.par.params_changed_count, 2, 'now params changed')

    def test_priority_notify(self):
        self.assertEqual(self.par.params_changed_count, 0)
        self.par.notify_observers(0, None)
        self.assertEqual(self.par.params_changed_count, 1)

        self.par.notify_observers(0, -numpy.inf)
        self.assertEqual(self.par.params_changed_count, 2)
        self.assertEqual(self.parent.parent_changed_count, 1)

    def test_priority(self):
        self.par.add_observer(self, self._trigger_priority, 0)
        self.par.notify_observers(0)
        self.assertEqual(self._first, self._trigger_priority, 'priority should be first')
        self.assertEqual(self._second, self._trigger, 'priority should be first')

        self._first = self._second = None

        self.par.add_observer(self, self._trigger, 1)
        self.assertEqual(self._first, self._trigger, 'priority should be second')
        self.assertEqual(self._second, self._trigger_priority, 'priority should be second')

    #import sys;sys.argv = ['', 'Test.testName']

class WarpingFunction(object):
    """
    abstract function for warping
    z = f(y)
    """

    def f(self,y,psi):
        """function transformation
        y is a list of values (GP training data) of shpape [N,1]
        """

    def fgrad_y(self,y,psi):
        """gradient of f w.r.t to y"""

    def fgrad_y_psi(self,y,psi):

    def f_inv(self,z,psi):
        """inverse function transformation"""

    def plot(self, psi, xmin, xmax):
        y = np.arange(xmin, xmax, 0.01)
        f_y = self.f(y, psi)
        plt.figure()
        plt.plot(y, f_y)
        plt.xlabel('y')
        plt.ylabel('f(y)')
        plt.title('warping function')

class TanhWarpingFunction(WarpingFunction):

    def __init__(self,n_terms=3):
        """n_terms specifies the number of tanh terms to be used"""
        self.n_terms = n_terms
        self.num_parameters = 3 * self.n_terms

        """
        transform y with f using parameter vector psi
        psi = [[a,b,c]]
        ::math::`f = \\sum_{terms} a * tanh(b*(y+c))`

        """

        #1. check that number of params is consistent
        assert psi.shape[0] == self.n_terms, 'inconsistent parameter dimensions'
        assert psi.shape[1] == 3, 'inconsistent parameter dimensions'

        #2. exponentiate the a and b (positive!)
        mpsi = psi.copy()

        #3. transform data
        z = y.copy()
        for i in range(len(mpsi)):
            a,b,c = mpsi[i]
            z += a*np.tanh(b*(y+c))
        return z

    def f_inv(self, y, psi, iterations = 10):
        """
        calculate the numerical inverse of f

        :param iterations: number of N.R. iterations

        """

        y = y.copy()
        z = np.ones_like(y)

        for i in range(iterations):
            z -= (self.f(z, psi) - y)/self.fgrad_y(z,psi)

        return z

    def fgrad_y(self, y, psi, return_precalc = False):
        """
        gradient of f w.r.t to y ([N x 1])
        returns: Nx1 vector of derivatives, unless return_precalc is true,
        then it also returns the precomputed stuff
        """

        # vectorized version

        # S = (mpsi[:,1]*(y + mpsi[:,2])).T
        S = (mpsi[:,1]*(y[:,:,None] + mpsi[:,2])).T
        R = np.tanh(S)
        D = 1-R**2

        # GRAD = (1+(mpsi[:,0:1]*mpsi[:,1:2]*D).sum(axis=0))[:,np.newaxis]
        GRAD = (1+(mpsi[:,0:1][:,:,None]*mpsi[:,1:2][:,:,None]*D).sum(axis=0)).T

        if return_precalc:
            # return GRAD,S.sum(axis=1),R.sum(axis=1),D.sum(axis=1)
            return GRAD, S, R, D

        return GRAD

    def fgrad_y_psi(self, y, psi, return_covar_chain = False):
        """
        gradient of f w.r.t to y and psi

        returns: NxIx3 tensor of partial derivatives

        """

        # 1. exponentiate the a and b (positive!)
        w, s, r, d = self.fgrad_y(y, psi, return_precalc = True)

        gradients = np.zeros((y.shape[0], y.shape[1], len(mpsi), 3))
            gradients[:,:,i,0] = (b*(1.0/np.cosh(s[i]))**2).T
            gradients[:,:,i,1] = a*(d[i] - 2.0*s[i]*r[i]*(1.0/np.cosh(s[i]))**2).T
            gradients[:,:,i,2] = (-2.0*a*(b**2)*r[i]*((1.0/np.cosh(s[i]))**2)).T

        if return_covar_chain:
            covar_grad_chain = np.zeros((y.shape[0], y.shape[1], len(mpsi), 3))

            for i in range(len(mpsi)):
                a,b,c = mpsi[i]
                covar_grad_chain[:, :, i, 0] = (r[i]).T
                covar_grad_chain[:, :, i, 1] = (a*(y + c) * ((1.0/np.cosh(s[i]))**2).T)
                covar_grad_chain[:, :, i, 2] = a*b*((1.0/np.cosh(s[i]))**2).T

            return gradients, covar_grad_chain

        return gradients

        variables = ['a', 'b', 'c']
        names = sum([['warp_tanh_%s_t%i' % (variables[n],q) for n in range(3)] for q in range(self.n_terms)],[])
        return names

class TanhWarpingFunction_d(WarpingFunction):

        self.num_parameters = 3 * self.n_terms + 1

        """
        Transform y with f using parameter vector psi

        :math:`f = \\sum_{terms} a * tanh(b*(y+c))`
        """

        # assert psi.shape[0] == self.n_terms, 'inconsistent parameter dimensions'
        # assert psi.shape[1] == 4, 'inconsistent parameter dimensions'
        d = psi[-1]
        mpsi = mpsi[:self.num_parameters-1].reshape(self.n_terms, 3)

        z = d*y.copy()
        return z

    def f_inv(self, z, psi, max_iterations=1000, y=None):
        """

        :param max_iterations: maximum number of N.R. iterations

        """

        z = z.copy()
        if y is None:
            y = np.ones_like(z)
            
        it = 0
        update = np.inf

        while it == 0 or (np.abs(update).sum() > 1e-10 and it < max_iterations):
            update = (self.f(y, psi) - z)/self.fgrad_y(y, psi)
            y -= update
            it += 1
        if it == max_iterations:
            print "WARNING!!! Maximum number of iterations reached in f_inv "

        return y

        """

        :returns: Nx1 vector of derivatives, unless return_precalc is true, then it also returns the precomputed stuff

        """

        d = psi[-1]

        D = 1-R**2

        GRAD = (d + (mpsi[:,0:1][:,:,None]*mpsi[:,1:2][:,:,None]*D).sum(axis=0)).T

        """

        :returns: NxIx4 tensor of partial derivatives

        """

        gradients = np.zeros((y.shape[0], y.shape[1], len(mpsi), 4))
            a,b,c  = mpsi[i]
        gradients[:,:,0,3] = 1.0

            covar_grad_chain = np.zeros((y.shape[0], y.shape[1], len(mpsi), 4))

            covar_grad_chain[:, :, 0, 3] = y

        variables = ['a', 'b', 'c', 'd']
        names.append('warp_tanh_d')

one, two, three = 'one', 'two', 'three'

        self.param_index = ParameterIndexOperations()
        self.param_index.add(one, [3,9])
        self.param_index.add(two, [0,5])
        self.param_index.add(three, [2,4,7,10])
        self.view = ParameterIndexOperationsView(self.param_index, 2, 6)

    def test_clear(self):
        self.param_index.clear()
        self.assertDictEqual(self.param_index._properties, {})

    def test_remove(self):
        removed = self.param_index.remove(three, np.r_[3:13])
        self.assertListEqual(removed.tolist(), [4,7,10])
        self.assertListEqual(self.param_index[three].tolist(), [2])
        removed = self.param_index.remove(one, [1])
        self.assertListEqual(removed.tolist(), [])
        self.assertListEqual(self.param_index[one].tolist(), [3,9])
        self.assertListEqual(self.param_index.remove('not in there', []).tolist(), [])
        removed = self.param_index.remove(one, [9])
        self.assertListEqual(removed.tolist(), [9])
        self.assertListEqual(self.param_index[one].tolist(), [3])
        self.assertListEqual(self.param_index.remove('not in there', [2,3,4]).tolist(), [])
        self.assertListEqual(self.view.remove('not in there', [2,3,4]).tolist(), [])

    def test_shift_left(self):
        self.view.shift_left(0, 2)
        self.assertListEqual(self.param_index[three].tolist(), [2,5,8])
        self.assertListEqual(self.param_index[two].tolist(), [0,3])
        self.assertListEqual(self.param_index[one].tolist(), [7])
        #          0    1    2    3    4    5    6    7    8    9    10
        #                                            one
        #         two            two
        #                   three          three          three
        # view:             [0    1    2    3    4    5    ]
        self.assertListEqual(self.view[three].tolist(), [0,3])
        self.assertListEqual(self.view[two].tolist(), [1])
        self.assertListEqual(self.view[one].tolist(), [5])
        self.param_index.shift_left(7, 1)
        #
        #         two            two
        #                   three          three     three
        self.assertListEqual(self.param_index[three].tolist(), [2,5,7])
        self.assertListEqual(self.param_index[one].tolist(), [])
        self.assertListEqual(self.view[three].tolist(), [0,3,5])
        self.assertListEqual(self.view[one].tolist(), [])

    def test_shift_right(self):
        self.view.shift_right(3, 2)
        self.assertListEqual(self.param_index[three].tolist(), [2,4,9,12])
        self.assertListEqual(self.param_index[two].tolist(), [0,7])
        self.assertListEqual(self.param_index[one].tolist(), [3,11])

    def test_index_view(self):
        #                        one                           one
        #         two                      two
        #                   three     three          three          three
        self.assertSetEqual(set(self.view.properties()), set([one, two, three]))
        for v,p in zip(self.view.properties_for(np.r_[:6]), self.param_index.properties_for(np.r_[2:2+6])):
            self.assertEqual(v, p)
        self.assertSetEqual(set(self.view[two]), set([3]))
        self.assertSetEqual(set(self.param_index[two]), set([0, 5]))
        self.view.add(two, np.array([0]))
        self.assertSetEqual(set(self.view[two]), set([0,3]))
        self.assertSetEqual(set(self.param_index[two]), set([0, 2, 5]))
        self.view.clear()
            self.assertEqual(v, [])
        param_index = ParameterIndexOperations()
        param_index.add(one, [3,9])
        param_index.add(two, [0,5])
        param_index.add(three, [2,4,7,10])
        view2 = ParameterIndexOperationsView(param_index, 2, 8)
        self.view.update(view2)
        for [i,v],[i2,v2] in zip(sorted(param_index.items()), sorted(self.param_index.items())):
            self.assertEqual(i, i2)
            np.testing.assert_equal(v, v2)

    def test_view_of_view(self):
        #                        one                           one
        #         two                      two
        # view2:                      [0    1    2    3    4    5    ]
        view2 = ParameterIndexOperationsView(self.view, 2, 6)
        view2.shift_right(0, 2)

    def test_indexview_remove(self):
        removed = self.view.remove(two, [3])
        self.assertListEqual(removed.tolist(), [3])
        removed = self.view.remove(three, np.r_[:5])
        self.assertListEqual(removed.tolist(), [0, 2])

    def test_misc(self):
        for k,v in self.param_index.copy()._properties.iteritems():
            self.assertListEqual(self.param_index[k].tolist(), v.tolist())
        self.assertEqual(self.param_index.size, 8)
        self.assertEqual(self.view.size, 5)

    def test_print(self):
        print self.param_index
        print self.view

    #import sys;sys.argv = ['', 'Test.test_index_view']

from ...util.linalg import jitchol, tdot, dtrtrs, dpotri, pdinv

class FITC(LatentFunctionInference):
    """

    """

        num_inducing, _ = Z.shape

        #make sure the noise is not hetero
        sigma_n = likelihood.gaussian_variance(Y_metadata)
        if sigma_n.size >1:
            raise NotImplementedError, "no hetero noise with this implementation of FITC"

        Knn = kern.Kdiag(X)
        Knm = kern.K(X, Z)
        U = Knm

        #factor Kmm
        Kmmi, L, Li, _ = pdinv(Kmm)

        #compute beta_star, the effective noise precision
        LiUT = np.dot(Li, U.T)
        sigma_star = Knn + sigma_n - np.sum(np.square(LiUT),0)
        beta_star = 1./sigma_star

        # Compute and factor A
        A = tdot(LiUT*np.sqrt(beta_star)) + np.eye(num_inducing)
        LA = jitchol(A)

        # back substutue to get b, P, v
        URiy = np.dot(U.T*beta_star,Y)
        tmp, _ = dtrtrs(L, URiy, lower=1)
        b, _ = dtrtrs(LA, tmp, lower=1)
        tmp, _ = dtrtrs(LA, b, lower=1, trans=1)
        v, _ = dtrtrs(L, tmp, lower=1, trans=1)
        tmp, _ = dtrtrs(LA, Li, lower=1, trans=0)
        P = tdot(tmp.T)

        #compute log marginal
        log_marginal = -0.5*num_data*output_dim*np.log(2*np.pi) + \
                       -np.sum(np.log(np.diag(LA)))*output_dim + \
                       0.5*output_dim*np.sum(np.log(beta_star)) + \
                       -0.5*np.sum(np.square(Y.T*np.sqrt(beta_star))) + \
                       0.5*np.sum(np.square(b))
        #compute dL_dR
        Uv = np.dot(U, v)
        dL_dR = 0.5*(np.sum(U*np.dot(U,P), 1) - 1./beta_star + np.sum(np.square(Y), 1) - 2.*np.sum(Uv*Y, 1) + np.sum(np.square(Uv), 1))*beta_star**2

        vvT_P = tdot(v.reshape(-1,1)) + P
        dL_dK = 0.5*(Kmmi - vvT_P)
        KiU = np.dot(Kmmi, U.T)
        dL_dK += np.dot(KiU*dL_dR, KiU.T)

        # Compute dL_dU
        vY = np.dot(v.reshape(-1,1),Y.T)
        dL_dU = vY - np.dot(vvT_P, U.T)
        dL_dU *= beta_star
        dL_dU -= 2.*KiU*dL_dR

        dL_dthetaL = likelihood.exact_inference_gradients(dL_dR)
        grad_dict = {'dL_dKmm': dL_dK, 'dL_dKdiag':dL_dR, 'dL_dKnm':dL_dU.T, 'dL_dthetaL':dL_dthetaL}

        post = Posterior(woodbury_inv=Kmmi-P, woodbury_vector=v, K=Kmm, mean=None, cov=None, K_chol=L)

"""
Gaussian Processes classification examples
"""
import GPy

def oil(num_inducing=50, max_iters=100, kernel=None, optimize=True, plot=True):
    """
    Run a Gaussian process classification on the three phase oil data. The demonstration calls the basic GP classification model and uses EP to approximate the likelihood.

    """
    try:import pods
    except ImportError:print 'pods unavailable, see https://github.com/sods/ods for example datasets'
    X = data['X']
    Xtest = data['Xtest']
    Y = data['Y'][:, 0:1]
    Ytest = data['Ytest'][:, 0:1]
    Y[Y.flatten()==-1] = 0
    Ytest[Ytest.flatten()==-1] = 0

    # Create GP model
    m = GPy.models.SparseGPClassification(X, Y, kernel=kernel, num_inducing=num_inducing)

    # Contrain all parameters to be positive
    #m.tie_params('.*len')
    m['.*len'] = 10.

    # Optimize
        for _ in range(5):
            m.optimize(max_iters=int(max_iters/5))
    print(m)

    #Test
    probs = m.predict(Xtest)[0]
    GPy.util.classification.conf_matrix(probs, Ytest)
    return m

def toy_linear_1d_classification(seed=default_seed, optimize=True, plot=True):
    """
    Simple 1D classification example using EP approximation

    :param seed: seed value for data generation (default is 4).

    """

    data = pods.datasets.toy_linear_1d_classification(seed=seed)
    Y[Y.flatten() == -1] = 0

    # Model definition
    m = GPy.models.GPClassification(data['X'], Y)

    # Optimize
        #m.update_likelihood_approximation()
        # Parameters optimization:
        #m.pseudo_EM()

    # Plot
    if plot:
        fig, axes = plt.subplots(2, 1)
        m.plot_f(ax=axes[0])
        m.plot(ax=axes[1])

    print m
    return m

def toy_linear_1d_classification_laplace(seed=default_seed, optimize=True, plot=True):
    """
    Simple 1D classification example using Laplace approximation

    """

    likelihood = GPy.likelihoods.Bernoulli()
    laplace_inf = GPy.inference.latent_function_inference.Laplace()
    kernel = GPy.kern.RBF(1)

    m = GPy.core.GP(data['X'], Y, kernel=kernel, likelihood=likelihood, inference_method=laplace_inf)

    # Optimize
        try:
            m.optimize('scg', messages=1)
            return m

    # Plot
    if plot:

    print m
    return m

def sparse_toy_linear_1d_classification(num_inducing=10, seed=default_seed, optimize=True, plot=True):
    """
    Sparse 1D classification example

    """

    m = GPy.models.SparseGPClassification(data['X'], Y, num_inducing=num_inducing)
    m['.*len'] = 4.

    # Optimize

    # Plot
    if plot:

    print m
    return m

def toy_heaviside(seed=default_seed, max_iters=100, optimize=True, plot=True):
    """
    Simple 1D classification example using a heavy side gp transformation

    """

    likelihood = GPy.likelihoods.Bernoulli(gp_link=GPy.likelihoods.link_functions.Heaviside())
    ep = GPy.inference.latent_function_inference.expectation_propagation.EP()
    m = GPy.core.GP(X=data['X'], Y=Y, kernel=kernel, likelihood=likelihood, inference_method=ep, name='gp_classification_heaviside')
    #m = GPy.models.GPClassification(data['X'], likelihood=likelihood)

    # Optimize
        print m

    # Plot
    if plot:

    print m
    return m

def crescent_data(model_type='Full', num_inducing=10, seed=default_seed, kernel=None, optimize=True, plot=True):
    """
    Run a Gaussian process classification on the crescent data. The demonstration calls the basic GP classification model and uses EP to approximate the likelihood.

    :param model_type: type of model to fit ['Full', 'FITC', 'DTC'].
    :param inducing: number of inducing variables (only used for 'FITC' or 'DTC').
    :type inducing: int
    :param seed: seed value for data generation.
    :param kernel: kernel to use in the model
    :type kernel: a GPy kernel
    """
    data = pods.datasets.crescent_data(seed=seed)

    if model_type == 'Full':
        m = GPy.models.GPClassification(data['X'], Y, kernel=kernel)

    elif model_type == 'DTC':
        m = GPy.models.SparseGPClassification(data['X'], Y, kernel=kernel, num_inducing=num_inducing)
        m['.*len'] = 10.

    elif model_type == 'FITC':
        m = GPy.models.FITCClassification(data['X'], Y, kernel=kernel, num_inducing=num_inducing)
        m['.*len'] = 3.

        m.pseudo_EM()

    if plot:
        m.plot()

    print m
    return m

"""
The module for psi-statistics for RBF kernel
"""

from GPy.util.caching import Cacher

def psicomputations(variance, lengthscale, Z, variational_posterior):
    """
    Z - MxQ
    mu - NxQ
    S - NxQ
    gamma - NxQ
    """
    # _psi1                NxM

    psi0 = np.empty(mu.shape[0])
    psi0[:] = variance
    psi1 = _psi1computations(variance, lengthscale, Z, mu, S)
    psi2 = _psi2computations(variance, lengthscale, Z, mu, S).sum(axis=0)

def __psi1computations(variance, lengthscale, Z, mu, S):
    """
    Z - MxQ
    mu - NxQ
    S - NxQ
    gamma - NxQ
    """
    # here are the "statistics" for psi1
    # _psi1                NxM

    lengthscale2 = np.square(lengthscale)

    # psi1
    _psi1_logdenom = np.log(S/lengthscale2+1.).sum(axis=-1) # N
    _psi1_log = (_psi1_logdenom[:,None]+np.einsum('nmq,nq->nm',np.square(mu[:,None,:]-Z[None,:,:]),1./(S+lengthscale2)))/(-2.)
    _psi1 = variance*np.exp(_psi1_log)

    return _psi1

def __psi2computations(variance, lengthscale, Z, mu, S):
    """
    Z - MxQ
    mu - NxQ
    S - NxQ
    gamma - NxQ
    """
    # here are the "statistics" for psi2
    # _psi2                MxM

    _psi2_logdenom = np.log(2.*S/lengthscale2+1.).sum(axis=-1)/(-2.) # N
    _psi2_exp1 = (np.square(Z[:,None,:]-Z[None,:,:])/lengthscale2).sum(axis=-1)/(-4.) #MxM
    Z_hat = (Z[:,None,:]+Z[None,:,:])/2. #MxMxQ
    denom = 1./(2.*S+lengthscale2)
    _psi2_exp2 = -(np.square(mu)*denom).sum(axis=-1)[:,None,None]+2.*np.einsum('nq,moq,nq->nmo',mu,Z_hat,denom)-np.einsum('moq,nq->nmo',np.square(Z_hat),denom)
    _psi2 = variance*variance*np.exp(_psi2_logdenom[:,None,None]+_psi2_exp1[None,:,:]+_psi2_exp2)

    return _psi2

def psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior):
    ARD = (len(lengthscale)!=1)

    dvar_psi1, dl_psi1, dZ_psi1, dmu_psi1, dS_psi1 = _psi1compDer(dL_dpsi1, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance)
    dvar_psi2, dl_psi2, dZ_psi2, dmu_psi2, dS_psi2 = _psi2compDer(dL_dpsi2, variance, lengthscale, Z, variational_posterior.mean, variational_posterior.variance)

    dL_dvar = np.sum(dL_dpsi0) + dvar_psi1 + dvar_psi2

    dL_dlengscale = dl_psi1 + dl_psi2
    if not ARD:
        dL_dlengscale = dL_dlengscale.sum()

    dL_dmu = dmu_psi1 + dmu_psi2
    dL_dS = dS_psi1 + dS_psi2
    dL_dZ = dZ_psi1 + dZ_psi2

    return dL_dvar, dL_dlengscale, dL_dZ, dL_dmu, dL_dS

def _psi1compDer(dL_dpsi1, variance, lengthscale, Z, mu, S):
    """
    dL_dpsi1 - NxM
    Z - MxQ
    mu - NxQ
    S - NxQ
    gamma - NxQ
    """
    # Produced intermediate results: dL_dparams w.r.t. psi1
    # _dL_dvariance     1
    # _dL_dlengthscale  Q
    # _dL_dZ            MxQ
    # _dL_dgamma        NxQ
    # _dL_dmu           NxQ
    # _dL_dS            NxQ

    _psi1 = _psi1computations(variance, lengthscale, Z, mu, S)
    Lpsi1 = dL_dpsi1*_psi1
    Zmu = Z[None,:,:]-mu[:,None,:] # NxMxQ
    denom = 1./(S+lengthscale2)
    Zmu2_denom = np.square(Zmu)*denom[:,None,:] #NxMxQ
    _dL_dvar = Lpsi1.sum()/variance
    _dL_dmu = np.einsum('nm,nmq,nq->nq',Lpsi1,Zmu,denom)
    _dL_dS = np.einsum('nm,nmq,nq->nq',Lpsi1,(Zmu2_denom-1.),denom)/2.
    _dL_dZ = -np.einsum('nm,nmq,nq->mq',Lpsi1,Zmu,denom)
    _dL_dl = np.einsum('nm,nmq,nq->q',Lpsi1,(Zmu2_denom+(S/lengthscale2)[:,None,:]),denom*lengthscale)

    return _dL_dvar, _dL_dl, _dL_dZ, _dL_dmu, _dL_dS

def _psi2compDer(dL_dpsi2, variance, lengthscale, Z, mu, S):
    """
    Z - MxQ
    mu - NxQ
    S - NxQ
    gamma - NxQ
    dL_dpsi2 - MxM
    """
    # Produced the derivatives w.r.t. psi2:
    # _dL_dvariance      1
    # _dL_dlengthscale   Q
    # _dL_dZ             MxQ
    # _dL_dgamma         NxQ
    # _dL_dmu            NxQ
    # _dL_dS             NxQ

    denom = 1./(2*S+lengthscale2)
    denom2 = np.square(denom)

    _psi2 = _psi2computations(variance, lengthscale, Z, mu, S) # NxMxM
    Lpsi2 = dL_dpsi2*_psi2 # dL_dpsi2 is MxM, using broadcast to multiply N out
    Lpsi2sum = np.einsum('nmo->n',Lpsi2) #N
    Lpsi2Z = np.einsum('nmo,oq->nq',Lpsi2,Z) #NxQ
    Lpsi2Z2 = np.einsum('nmo,oq,oq->nq',Lpsi2,Z,Z) #NxQ
    Lpsi2Z2p = np.einsum('nmo,mq,oq->nq',Lpsi2,Z,Z) #NxQ
    Lpsi2Zhat = Lpsi2Z
    Lpsi2Zhat2 = (Lpsi2Z2+Lpsi2Z2p)/2

    _dL_dvar = Lpsi2sum.sum()*2/variance
    _dL_dmu = (-2*denom) * (mu*Lpsi2sum[:,None]-Lpsi2Zhat)
    _dL_dS = (2*np.square(denom))*(np.square(mu)*Lpsi2sum[:,None]-2*mu*Lpsi2Zhat+Lpsi2Zhat2) - denom*Lpsi2sum[:,None]
    _dL_dZ = -np.einsum('nmo,oq->oq',Lpsi2,Z)/lengthscale2+np.einsum('nmo,oq->mq',Lpsi2,Z)/lengthscale2+ \
             2*np.einsum('nmo,nq,nq->mq',Lpsi2,mu,denom) - np.einsum('nmo,nq,mq->mq',Lpsi2,denom,Z) - np.einsum('nmo,oq,nq->mq',Lpsi2,Z,denom)
    _dL_dl = 2*lengthscale* ((S/lengthscale2*denom+np.square(mu*denom))*Lpsi2sum[:,None]+(Lpsi2Z2-Lpsi2Z2p)/(2*np.square(lengthscale2))-
                             (2*mu*denom2)*Lpsi2Zhat+denom2*Lpsi2Zhat2).sum(axis=0)

_psi1computations = Cacher(__psi1computations, limit=1)
_psi2computations = Cacher(__psi2computations, limit=1)

class BayesianGPLVM(SparseGP_MPI):
    """

    """
                 name='bayesian gplvm', mpi_comm=None, normalizer=None,

        else:

            if mpi_comm is not None:
            else:
                from ..inference.latent_function_inference.var_dtc import VarDTC
                self.logger.debug("creating inference_method var_dtc")
                inference_method = VarDTC(limit=1 if not missing_data else Y.shape[1])
        if isinstance(inference_method,VarDTC_minibatch):
            inference_method.mpi_comm = mpi_comm

        super(BayesianGPLVM,self).__init__(X, Y, Z, kernel, likelihood=likelihood,
                                           normalizer=normalizer, mpi_comm=mpi_comm,
                                           variational_prior=self.variational_prior,
                                           )

        super(BayesianGPLVM,self).parameters_changed()
            return        

        kl_fctr = 1.
        self._log_marginal_likelihood -= kl_fctr*self.variational_prior.KL_divergence(self.X)

        self.X.mean.gradient, self.X.variance.gradient = self.kern.gradients_qX_expectations(
                                            variational_posterior=self.X,
                                            Z=self.Z,
                                            dL_dpsi0=self.grad_dict['dL_dpsi0'],
                                            dL_dpsi1=self.grad_dict['dL_dpsi1'],
                                            dL_dpsi2=self.grad_dict['dL_dpsi2'])

#

        import sys

        """

        Notes:
        """

        return X

        """
        """

        """
        """

        """
        """
        import sys

    """
    """

from numpy.lib.function_base import vectorize
from lists_and_dicts import IntArrayDict

def extract_properties_to_index(index, props):
    prop_index = dict()
    for i, cl in enumerate(props):
        for c in cl:
            ind = prop_index.get(c, list())
            ind.append(index[i])
            prop_index[c] = ind

    for c, i in prop_index.items():
        prop_index[c] = numpy.array(i, dtype=int)

    return prop_index

class ParameterIndexOperations(object):
    """
    This object wraps a dictionary, whos keys are _operations_ that we'd like
    to apply to a parameter array, and whose values are np integer arrays which
    index the parameter array appropriately.

    A model instance will contain one instance of this class for each thing
    that needs indexing (i.e. constraints, ties and priors). Parameters within
    the model constain instances of the ParameterIndexOperationsView class,
    which can map from a 'local' index (starting 0) to this global index.

    Here's an illustration:

    #=======================================================================
    model : 0 1 2 3 4 5 6 7 8 9
    key1: 4 5
    key2: 7 8

    param1: 0 1 2 3 4 5
    key1: 2 3
    key2: 5

    param2: 0 1 2 3 4
    key1: 0
    key2: 2 3

    The views of this global index have a subset of the keys in this global
    (model) index.

    Adding a new key (e.g. a constraint) to a view will cause the view to pass
    the new key to the global index, along with the local index and an offset.
    This global index then stores the key and the appropriate global index
    (which can be seen by the view).

    See also:

    """
    _offset = 0
    def __init__(self, constraints=None):
        self._properties = IntArrayDict()
        if constraints is not None:
            for t, i in constraints.iteritems():
                self.add(t, i)

    def iteritems(self):
        return self._properties.iteritems()

    def items(self):
        return self._properties.items()

    def properties(self):
        return self._properties.keys()

    def iterproperties(self):
        return self._properties.iterkeys()

    def shift_right(self, start, size):
        for ind in self.iterindices():
            toshift = ind>=start
            ind[toshift] += size

    def shift_left(self, start, size):
        for v, ind in self.items():
            todelete = (ind>=start) * (ind<start+size)
            if todelete.size != 0:
                ind = ind[~todelete]
            if toshift.size != 0:
                ind[toshift] -= size
            if ind.size != 0: self._properties[v] = ind
            else: del self._properties[v]

        self._properties.clear()

    @property
    def size(self):
        return reduce(lambda a,b: a+b.size, self.iterindices(), 0)

    def iterindices(self):
        return self._properties.itervalues()

    def indices(self):
        return self._properties.values()

    def properties_for(self, index):
        """
        Returns a list of properties, such that each entry in the list corresponds
        to the element of the index given.

        Example:
        let properties: 'one':[1,2,3,4], 'two':[3,5,6]

        >>> properties_for([2,3,5])
        [['one'], ['one', 'two'], ['two']]
        """
        return vectorize(lambda i: [prop for prop in self.iterproperties() if i in self[prop]], otypes=[list])(index)

    def properties_to_index_dict(self, index):
        """
        Return a dictionary, containing properties as keys and indices as index
        Thus, the indices for each constraint, which is contained will be collected as
        one dictionary

        Example:

        >>> properties_to_index_dict([2,3,5])
        {'one':[2,3], 'two':[3,5]}
        """
        props = self.properties_for(index)
        prop_index = extract_properties_to_index(index, props)
        return prop_index

    def add(self, prop, indices):
        self._properties[prop] = combine_indices(self._properties[prop], indices)

    def remove(self, prop, indices):
        if prop in self._properties:
            diff = remove_indices(self[prop], indices)
            removed = numpy.intersect1d(self[prop], indices, True)
            if not index_empty(diff):
                self._properties[prop] = diff
            else:
                del self._properties[prop]
            return removed.astype(int)
        return numpy.array([]).astype(int)

    def update(self, parameter_index_view, offset=0):
        for i, v in parameter_index_view.iteritems():
            self.add(i, v+offset)

    def copy(self):
        return self.__deepcopy__(None)

        return ParameterIndexOperations(dict(self.iteritems()))

    def __getitem__(self, prop):
        return self._properties[prop]

    def __delitem__(self, prop):
        del self._properties[prop]

    def __str__(self, *args, **kwargs):
        import pprint
        return pprint.pformat(dict(self._properties))

def combine_indices(arr1, arr2):
    return numpy.union1d(arr1, arr2)

def remove_indices(arr, to_remove):
    return numpy.setdiff1d(arr, to_remove, True)

def index_empty(index):
    return numpy.size(index) == 0

class ParameterIndexOperationsView(object):
    def __init__(self, param_index_operations, offset, size):
        self._param_index_ops = param_index_operations
        self._offset = offset
        self._size = size

        return [self._param_index_ops, self._offset, self._size]

        self._param_index_ops = state[0]
        self._offset = state[1]
        self._size = state[2]

    def _filter_index(self, ind):
        return ind[(ind >= self._offset) * (ind < (self._offset + self._size))] - self._offset

        for i, ind in self._param_index_ops.iteritems():
            ind2 = self._filter_index(ind)
            if ind2.size > 0:
                yield i, ind2

        return [[i,v] for i,v in self.iteritems()]

        return [i for i in self.iterproperties()]

        for i, _ in self.iteritems():
            yield i

        self._param_index_ops.shift_right(start+self._offset, size)

        self._param_index_ops.shift_left(start+self._offset, size)

        for i, ind in self.items():
            self._param_index_ops.remove(i, ind+self._offset)

    @property

        for _, ind in self.iteritems():
            yield ind

        return [ind for ind in self.iterindices()]

        """

        Example:

        """

        """

        Example:

        """
        return extract_properties_to_index(index, self.properties_for(index))

        self._param_index_ops.add(prop, indices+self._offset)

        removed = self._param_index_ops.remove(prop, numpy.array(indices)+self._offset)
        if removed.size > 0:
            return removed-self._offset
        return removed

        ind = self._filter_index(self._param_index_ops[prop])
        return ind

        self.remove(prop, self[prop])

        return pprint.pformat(dict(self.iteritems()))

    pass

# tdot function courtesy of Ian Murray:
# Iain Murray, April 2013. iain contactable via iainmurray.net
# http://homepages.inf.ed.ac.uk/imurray2/code/tdot/tdot.py

from scipy import linalg, weave
import ctypes
from ctypes import byref, c_char, c_int, c_double # TODO
import scipy
import os
from config import config

_scipyversion = np.float64((scipy.__version__).split('.')[:2])
_fix_dpotri_scipy_bug = True
if np.all(_scipyversion >= np.array([0, 14])):
    from scipy.linalg import lapack
    _fix_dpotri_scipy_bug = False
elif np.all(_scipyversion >= np.array([0, 12])):
    #import scipy.linalg.lapack.clapack as lapack
else:
    from scipy.linalg.lapack import flapack as lapack

if config.getboolean('anaconda', 'installed') and config.getboolean('anaconda', 'MKL'):
    try:
        anaconda_path = str(config.get('anaconda', 'location'))
        mkl_rt = ctypes.cdll.LoadLibrary(os.path.join(anaconda_path, 'DLLs', 'mkl_rt.dll'))
        dsyrk = mkl_rt.dsyrk
        dsyr = mkl_rt.dsyr
        _blas_available = True
        print 'anaconda installed and mkl is loaded'
    except:
        _blas_available = False
else:
    try:
        _blaslib = ctypes.cdll.LoadLibrary(np.core._dotblas.__file__) # @UndefinedVariable
        dsyrk = _blaslib.dsyrk_
        dsyr = _blaslib.dsyr_
    except AttributeError as e:
        warnings.warn("warning: caught this exception:" + str(e))

def force_F_ordered_symmetric(A):
    """
    return a F ordered version of A, assuming A is symmetric
    """
    if A.flags['F_CONTIGUOUS']:
        return A
    if A.flags['C_CONTIGUOUS']:
        return A.T
    else:
        return np.asfortranarray(A)

def force_F_ordered(A):
    """
    return a F ordered version of A, assuming A is triangular
    """
        return A
    print "why are your arrays not F order?"
    return np.asfortranarray(A)

# def jitchol(A, maxtries=5):
#     A = force_F_ordered_symmetric(A)
#     L, info = lapack.dpotrf(A, lower=1)
#     if info == 0:
#         return L
#     else:
#         if maxtries==0:
#             raise linalg.LinAlgError, "not positive definite, even with jitter."
#         diagA = np.diag(A)
#         if np.any(diagA <= 0.):
#             raise linalg.LinAlgError, "not pd: non-positive diagonal elements"
#         jitter = diagA.mean() * 1e-6

#         return jitchol(A+np.eye(A.shape[0])*jitter, maxtries-1)

def jitchol(A, maxtries=5):
    A = np.ascontiguousarray(A)
    L, info = lapack.dpotrf(A, lower=1)
    if info == 0:
        return L
    else:
        diagA = np.diag(A)
        if np.any(diagA <= 0.):
            raise linalg.LinAlgError, "not pd: non-positive diagonal elements"
        jitter = diagA.mean() * 1e-6
        while maxtries > 0 and np.isfinite(jitter):
            try:
                L = linalg.cholesky(A + np.eye(A.shape[0]) * jitter, lower=True)
            except:
                jitter *= 10
            finally:
                maxtries -= 1
        raise linalg.LinAlgError, "not positive definite, even with jitter."
    import traceback
    try: raise
    except:
        logging.warning('\n'.join(['Added jitter of {:.10e}'.format(jitter),
            '  in '+traceback.format_list(traceback.extract_stack(limit=2)[-2:-1])[0][2:]]))
    import ipdb;ipdb.set_trace()
    return L

# def dtrtri(L, lower=1):
#     """
#     Wrapper for lapack dtrtri function
#     Inverse of L
#
#     :param L: Triangular Matrix L
#     :param lower: is matrix lower (true) or upper (false)
#     :returns: Li, info
#     """
#     L = force_F_ordered(L)
#     return lapack.dtrtri(L, lower=lower)

def dtrtrs(A, B, lower=1, trans=0, unitdiag=0):
    """
    Wrapper for lapack dtrtrs function

    DTRTRS solves a triangular system of the form

        A * X = B  or  A**T * X = B,

    where A is a triangular matrix of order N, and B is an N-by-NRHS
    matrix.  A check is made to verify that A is nonsingular.

    :param A: Matrix A(triangular)
    :param B: Matrix B
    :param lower: is matrix lower (true) or upper (false)
    :returns: Solution to A * X = B or A**T * X = B

    """
    A = np.asfortranarray(A)
    #Note: B does not seem to need to be F ordered!
    return lapack.dtrtrs(A, B, lower=lower, trans=trans, unitdiag=unitdiag)

def dpotrs(A, B, lower=1):
    """
    Wrapper for lapack dpotrs function
    :param A: Matrix A
    :returns:
    """
    A = force_F_ordered(A)
    return lapack.dpotrs(A, B, lower=lower)

def dpotri(A, lower=1):
    """
    Wrapper for lapack dpotri function

    DPOTRI - compute the inverse of a real symmetric positive
      definite matrix A using the Cholesky factorization A =
      U**T*U or A = L*L**T computed by DPOTRF

    :returns: A inverse

    """
    if _fix_dpotri_scipy_bug:
        assert lower==1, "scipy linalg behaviour is very weird. please use lower, fortran ordered arrays"
        lower = 0

    R, info = lapack.dpotri(A, lower=lower) #needs to be zero here, seems to be a scipy bug

    symmetrify(R)
    return R, info

def pddet(A):
    """
    Determinant of a positive definite matrix, only symmetric matricies though
    """
    L = jitchol(A)
    logdetA = 2*sum(np.log(np.diag(L)))
    return logdetA

def trace_dot(a, b):
    """
    Efficiently compute the trace of the matrix product of a and b
    """
    return np.sum(a * b)

def mdot(*args):
    """
    Multiply all the arguments using matrix product rules.
    The output is equivalent to multiplying the arguments one by one
    from left to right using dot().
    Precedence can be controlled by creating tuples of arguments,
    for instance mdot(a,((b,c),d)) multiplies a (a*((b*c)*d)).
    Note that this means the output of dot(a,b) and mdot(a,b) will differ if
    a or b is a pure tuple of numbers.

    """
    if len(args) == 1:
        return args[0]
    elif len(args) == 2:
        return _mdot_r(args[0], args[1])
    else:
        return _mdot_r(args[:-1], args[-1])

def _mdot_r(a, b):
    """Recursive helper for mdot"""
    if type(a) == types.TupleType:
        if len(a) > 1:
            a = mdot(*a)
        else:
            a = a[0]
    if type(b) == types.TupleType:
        if len(b) > 1:
            b = mdot(*b)
        else:
            b = b[0]
    return np.dot(a, b)

def pdinv(A, *args):
    """
    :param A: A DxD pd numpy array

    :rval Ai: the inverse of A
    :rtype Ai: np.ndarray
    :rval L: the Cholesky decomposition of A
    :rtype L: np.ndarray
    :rval Li: the Cholesky decomposition of Ai
    :rtype Li: np.ndarray
    :rval logdet: the log of the determinant of A
    :rtype logdet: float64

    """
    L = jitchol(A, *args)
    logdet = 2.*np.sum(np.log(np.diag(L)))
    Li = dtrtri(L)
    Ai, _ = dpotri(L, lower=1)
    # Ai = np.tril(Ai) + np.tril(Ai,-1).T
    symmetrify(Ai)

    return Ai, L, Li, logdet

def dtrtri(L):
    """
    Inverts a Cholesky lower triangular matrix

    :param L: lower triangular matrix
    :rtype: inverse of L

    """

    L = force_F_ordered(L)
    return lapack.dtrtri(L, lower=1)[0]

def multiple_pdinv(A):
    """
    :param A: A DxDxN numpy array (each A[:,:,i] is pd)

    :rval invs: the inverses of A
    :rtype invs: np.ndarray
    :rval hld: 0.5* the log of the determinants of A
    :rtype hld: np.array

    """
    N = A.shape[-1]
    chols = [jitchol(A[:, :, i]) for i in range(N)]
    halflogdets = [np.sum(np.log(np.diag(L[0]))) for L in chols]
    invs = [dpotri(L[0], True)[0] for L in chols]
    invs = [np.triu(I) + np.triu(I, 1).T for I in invs]
    return np.dstack(invs), np.array(halflogdets)

def pca(Y, input_dim):
    """
    Principal component analysis: maximum likelihood solution by SVD

    :param Y: NxD np.array of data
    :param input_dim: int, dimension of projection

    :rval X: - Nxinput_dim np.array of dimensionality reduced data
    :rval W: - input_dimxD mapping from X to Y

    """
    if not np.allclose(Y.mean(axis=0), 0.0):
        print "Y is not zero mean, centering it locally (GPy.util.linalg.pca)"

        # Y -= Y.mean(axis=0)

    Z = linalg.svd(Y - Y.mean(axis=0), full_matrices=False)
    [X, W] = [Z[0][:, 0:input_dim], np.dot(np.diag(Z[1]), Z[2]).T[:, 0:input_dim]]
    v = X.std(axis=0)
    X /= v;
    W *= v;
    return X, W.T

def ppca(Y, Q, iterations=100):
    """
    EM implementation for probabilistic pca.

    :param array-like Y: Observed Data
    :param int Q: Dimensionality for reduced array
    :param int iterations: number of iterations for EM
    """
    from numpy.ma import dot as madot
    N, D = Y.shape
    # Initialise W randomly
    W = np.random.randn(D, Q) * 1e-3
    Y = np.ma.masked_invalid(Y, copy=0)
    mu = Y.mean(0)
    Ycentered = Y - mu
    try:
        for _ in range(iterations):
            exp_x = np.asarray_chkfinite(np.linalg.solve(W.T.dot(W), madot(W.T, Ycentered.T))).T
            W = np.asarray_chkfinite(np.linalg.solve(exp_x.T.dot(exp_x), madot(exp_x.T, Ycentered))).T
    except np.linalg.linalg.LinAlgError:
        #"converged"
        pass
    return np.asarray_chkfinite(exp_x), np.asarray_chkfinite(W)

def tdot_numpy(mat, out=None):
    return np.dot(mat, mat.T, out)

def tdot_blas(mat, out=None):
    """returns np.dot(mat, mat.T), but faster for large 2D arrays of doubles."""
    if (mat.dtype != 'float64') or (len(mat.shape) != 2):
        return np.dot(mat, mat.T)
    nn = mat.shape[0]
    if out is None:
        out = np.zeros((nn, nn))
    else:
        assert(out.dtype == 'float64')
        assert(out.shape == (nn, nn))
        # FIXME: should allow non-contiguous out, and copy output into it:
        assert(8 in out.strides)
        # zeroing needed because of dumb way I copy across triangular answer
        out[:] = 0.0

    # # Call to DSYRK from BLAS
    # If already in Fortran order (rare), and has the right sorts of strides I
    # could avoid the copy. I also thought swapping to cblas API would allow use
    # of C order. However, I tried that and had errors with large matrices:
    # http://homepages.inf.ed.ac.uk/imurray2/code/tdot/tdot_broken.py
    mat = np.asfortranarray(mat)
    TRANS = c_char('n')
    N = c_int(mat.shape[0])
    K = c_int(mat.shape[1])
    LDA = c_int(mat.shape[0])
    UPLO = c_char('l')
    ALPHA = c_double(1.0)
    A = mat.ctypes.data_as(ctypes.c_void_p)
    BETA = c_double(0.0)
    C = out.ctypes.data_as(ctypes.c_void_p)
    LDC = c_int(np.max(out.strides) / 8)
    dsyrk(byref(UPLO), byref(TRANS), byref(N), byref(K),
            byref(ALPHA), A, byref(LDA), byref(BETA), C, byref(LDC))

    symmetrify(out, upper=True)

    return np.ascontiguousarray(out)

def tdot(*args, **kwargs):
    if _blas_available:
        return tdot_blas(*args, **kwargs)
    else:
        return tdot_numpy(*args, **kwargs)

def DSYR_blas(A, x, alpha=1.):
    """
    Performs a symmetric rank-1 update operation:
    A <- A + alpha * np.dot(x,x.T)

    :param A: Symmetric NxN np.array
    :param x: Nx1 np.array
    :param alpha: scalar

    """
    N = c_int(A.shape[0])
    LDA = c_int(A.shape[0])
    ALPHA = c_double(alpha)
    A_ = A.ctypes.data_as(ctypes.c_void_p)
    x_ = x.ctypes.data_as(ctypes.c_void_p)
    INCX = c_int(1)
    dsyr(byref(UPLO), byref(N), byref(ALPHA),
            x_, byref(INCX), A_, byref(LDA))
    symmetrify(A, upper=True)

def DSYR_numpy(A, x, alpha=1.):
    """

    """
    A += alpha * np.dot(x[:, None], x[None, :])

def DSYR(*args, **kwargs):
        return DSYR_blas(*args, **kwargs)
    else:
        return DSYR_numpy(*args, **kwargs)

def symmetrify(A, upper=False):
    """
    Take the square matrix A and make it symmetrical by copting elements from the lower half to the upper

    works IN PLACE.

    note: tries to use weave, falls back to a slower numpy version
    """
    if config.getboolean('weave', 'working'):
        try:
            symmetrify_weave(A, upper)
        except:
            print "\n Weave compilation failed. Falling back to (slower) numpy implementation\n"
            config.set('weave', 'working', 'False')
            symmetrify_numpy(A, upper)
    else:
        symmetrify_numpy(A, upper)

def symmetrify_weave(A, upper=False):
    """

    """
    N, M = A.shape
    assert N == M

    c_contig_code = """
    int iN;
    for (int i=1; i<N; i++){
      iN = i*N;
      for (int j=0; j<i; j++){
        A[i+j*N] = A[iN+j];
      }
    }
    """
    f_contig_code = """
    int iN;
      iN = i*N;
        A[iN+j] = A[i+j*N];
      }
    }
    """

    N = int(N) # for safe type casting
    if A.flags['C_CONTIGUOUS'] and upper:
        weave.inline(f_contig_code, ['A', 'N'], extra_compile_args=['-O3'])
    elif A.flags['C_CONTIGUOUS'] and not upper:
        weave.inline(c_contig_code, ['A', 'N'], extra_compile_args=['-O3'])
    elif A.flags['F_CONTIGUOUS'] and upper:
    elif A.flags['F_CONTIGUOUS'] and not upper:
    else:
        if upper:
            tmp = np.tril(A.T)
        else:
            tmp = np.tril(A)
        A[:] = 0.0
        A += tmp
        A += np.tril(tmp, -1).T

def symmetrify_numpy(A, upper=False):
    """
    Force a matrix to be symmetric
    """
    triu = np.triu_indices_from(A,k=1)
    if upper:
        A.T[triu] = A[triu]
    else:
        A[triu] = A.T[triu]

def cholupdate(L, x):
    """
    update the LOWER cholesky factor of a pd matrix IN PLACE

    if L is the lower chol. of K, then this function computes L\_
    where L\_ is the lower chol of K + x*x^T

    """
    support_code = """
    #include <math.h>
    """
    code = """
    double r,c,s;
    int j,i;
    for(j=0; j<N; j++){
      r = sqrt(L(j,j)*L(j,j) + x(j)*x(j));
      c = r / L(j,j);
      s = x(j) / L(j,j);
      L(j,j) = r;
      for (i=j+1; i<N; i++){
        L(i,j) = (L(i,j) + s*x(i))/c;
        x(i) = c*x(i) - s*L(i,j);
      }
    }
    """
    x = x.copy()
    N = x.size
    weave.inline(code, support_code=support_code, arg_names=['N', 'L', 'x'], type_converters=weave.converters.blitz)

def backsub_both_sides(L, X, transpose='left'):
    """ Return L^-T * X * L^-1, assumuing X is symmetrical and L is lower cholesky"""
    if transpose == 'left':
        tmp, _ = dtrtrs(L, X, lower=1, trans=1)
        return dtrtrs(L, tmp.T, lower=1, trans=1)[0].T
    else:
        tmp, _ = dtrtrs(L, X, lower=1, trans=0)
        return dtrtrs(L, tmp.T, lower=1, trans=0)[0].T

from ..util.warping_functions import *
from GPy.util.warping_functions import TanhWarpingFunction_d
from GPy import kern

class WarpedGP(GP):
    def __init__(self, X, Y, kernel=None, warping_function=None, warping_terms=3, normalize_X=False, normalize_Y=False):

            kernel = kern.rbf(X.shape[1])

        if warping_function == None:
            self.warping_function = TanhWarpingFunction_d(warping_terms)
            self.warping_params = (np.random.randn(self.warping_function.n_terms * 3 + 1,) * 1)

        self.scale_data = False
        if self.scale_data:
            Y = self._scale_data(Y)
        self.has_uncertain_inputs = False
        self.Y_untransformed = Y.copy()
        self.predict_in_warped_space = False
        likelihood = likelihoods.Gaussian(self.transform_data(), normalize=normalize_Y)

        GP.__init__(self, X, likelihood, kernel, normalize_X=normalize_X)
        self._set_params(self._get_params())

    def _scale_data(self, Y):
        self._Ymax = Y.max()
        self._Ymin = Y.min()
        return (Y - self._Ymin) / (self._Ymax - self._Ymin) - 0.5

    def _unscale_data(self, Y):
        return (Y + 0.5) * (self._Ymax - self._Ymin) + self._Ymin

        self.warping_params = x[:self.warping_function.num_parameters]
        Y = self.transform_data()
        self.likelihood.set_data(Y)
        GP._set_params(self, x[self.warping_function.num_parameters:].copy())

        return np.hstack((self.warping_params.flatten().copy(), GP._get_params(self).copy()))

        warping_names = self.warping_function._get_param_names()
        param_names = GP._get_param_names(self)
        return warping_names + param_names

    def transform_data(self):
        Y = self.warping_function.f(self.Y_untransformed.copy(), self.warping_params).copy()
        return Y

        ll = GP.log_likelihood(self)
        jacobian = self.warping_function.fgrad_y(self.Y_untransformed, self.warping_params)
        return ll + np.log(jacobian).sum()

        ll_grads = GP._log_likelihood_gradients(self)
        alpha = np.dot(self.Ki, self.likelihood.Y.flatten())
        warping_grads = self.warping_function_gradients(alpha)

        warping_grads = np.append(warping_grads[:, :-1].flatten(), warping_grads[0, -1])
        return np.hstack((warping_grads.flatten(), ll_grads.flatten()))

    def warping_function_gradients(self, Kiy):
        grad_y = self.warping_function.fgrad_y(self.Y_untransformed, self.warping_params)
        grad_y_psi, grad_psi = self.warping_function.fgrad_y_psi(self.Y_untransformed, self.warping_params,
                                                                 return_covar_chain=True)
        djac_dpsi = ((1.0 / grad_y[:, :, None, None]) * grad_y_psi).sum(axis=0).sum(axis=0)
        dquad_dpsi = (Kiy[:, None, None, None] * grad_psi).sum(axis=0).sum(axis=0)

        return -dquad_dpsi + djac_dpsi

    def plot_warping(self):
        self.warping_function.plot(self.warping_params, self.Y_untransformed.min(), self.Y_untransformed.max())

    def predict(self, Xnew, which_parts='all', full_cov=False, pred_init=None):
        # normalize X values
        Xnew = (Xnew.copy() - self._Xoffset) / self._Xscale
        mu, var = GP._raw_predict(self, Xnew, full_cov=full_cov, which_parts=which_parts)

        mean, var, _025pm, _975pm = self.likelihood.predictive_values(mu, var, full_cov)

        if self.predict_in_warped_space:
            mean = self.warping_function.f_inv(mean, self.warping_params, y=pred_init)
            var = self.warping_function.f_inv(var, self.warping_params)

            mean = self._unscale_data(mean)
        
        return mean, var, _025pm, _975pm

from ... import util
from scipy import integrate, weave

class Stationary(Kern):
    """
    Stationary kernels (covariance functions).

    Stationary covariance fucntion depend only on r, where r is defined as

      r = \sqrt{ \sum_{q=1}^Q (x_q - x'_q)^2 }

    The covariance function k(x, x' can then be written k(r).

    In this implementation, r is scaled by the lengthscales parameter(s):

      r = \sqrt{ \sum_{q=1}^Q \frac{(x_q - x'_q)^2}{\ell_q^2} }.

    By default, there's only one lengthscale: seaprate lengthscales for each
    dimension can be enables by setting ARD=True.

    To implement a stationary covariance function using this class, one need
    only define the covariance function k(r), and it derivative.

      ...
      def K_of_r(self, r):
          return foo
      def dK_dr(self, r):
          return bar

    The lengthscale(s) and variance parameters are added to the structure automatically.

    """

    def __init__(self, input_dim, variance, lengthscale, ARD, active_dims, name, useGPU=False):
        super(Stationary, self).__init__(input_dim, active_dims, name,useGPU=useGPU)
        if not ARD:
            if lengthscale is None:
                lengthscale = np.ones(1)
            else:
                lengthscale = np.asarray(lengthscale)
                assert lengthscale.size == 1, "Only 1 lengthscale needed for non-ARD kernel"
        else:
            if lengthscale is not None:
                assert lengthscale.size in [1, input_dim], "Bad number of lengthscales"
                if lengthscale.size != input_dim:
                    lengthscale = np.ones(input_dim)*lengthscale
            else:
                lengthscale = np.ones(self.input_dim)
        self.lengthscale = Param('lengthscale', lengthscale, Logexp())
        assert self.variance.size==1
        self.link_parameters(self.variance, self.lengthscale)

        raise NotImplementedError, "implement the covariance function as a fn of r to use this class"

        raise NotImplementedError, "implement derivative of the covariance function wrt r to use this class"

    @Cache_this(limit=5, ignore_args=())
        """
        Kernel function applied on inputs X and X2.
        In the stationary case there is an inner function depending on the
        distances from X to X2, called r.

        K(X, X2) = K_of_r((X-X2)**2)
        """
        r = self._scaled_dist(X, X2)
        return self.K_of_r(r)

    @Cache_this(limit=3, ignore_args=())
    def dK_dr_via_X(self, X, X2):
        #a convenience function, so we can cache dK_dr
        return self.dK_dr(self._scaled_dist(X, X2))

    def _unscaled_dist(self, X, X2=None):
        """
        Compute the Euclidean distance between each row of X and X2, or between
        each pair of rows of X if X2 is None.
        """
        #X, = self._slice_X(X)
            Xsq = np.sum(np.square(X),1)
            r2 = -2.*tdot(X) + (Xsq[:,None] + Xsq[None,:])
            util.diag.view(r2)[:,]= 0. # force diagnoal to be zero: sometime numerically a little negative
            r2 = np.clip(r2, 0, np.inf)
            return np.sqrt(r2)
        else:
            #X2, = self._slice_X(X2)
            X1sq = np.sum(np.square(X),1)
            X2sq = np.sum(np.square(X2),1)
            r2 = -2.*np.dot(X, X2.T) + X1sq[:,None] + X2sq[None,:]

    def _scaled_dist(self, X, X2=None):
        """
        Efficiently compute the scaled distance, r.

        r = \sqrt( \sum_{q=1}^Q (x_q - x'q)^2/l_q^2 )

        Note that if thre is only one lengthscale, l comes outside the sum. In
        this case we compute the unscaled distance first (in a separate
        function for caching) and divide by lengthscale afterwards

        """
            if X2 is not None:
                X2 = X2 / self.lengthscale
            return self._unscaled_dist(X/self.lengthscale, X2)
        else:
            return self._unscaled_dist(X, X2)/self.lengthscale

        ret = np.empty(X.shape[0])
        ret[:] = self.variance
        return ret

        """
        Given the derivative of the objective with respect to the diagonal of
        the covariance matrix, compute the derivative wrt the parameters of
        this kernel and stor in the <parameter>.gradient field.

        See also update_gradients_full
        """
        self.variance.gradient = np.sum(dL_dKdiag)
        self.lengthscale.gradient = 0.

        """
        Given the derivative of the objective wrt the covariance matrix
        (dL_dK), compute the gradient wrt the parameters of this kernel,
        and store in the parameters object as e.g. self.variance.gradient
        """
        self.variance.gradient = np.einsum('ij,ij,i', self.K(X, X2), dL_dK, 1./self.variance)

        #now the lengthscale gradient(s)
        dL_dr = self.dK_dr_via_X(X, X2) * dL_dK
            #rinv = self._inv_dis# this is rather high memory? Should we loop instead?t(X, X2)
            #d =  X[:, None, :] - X2[None, :, :]
            #x_xl3 = np.square(d)
            #self.lengthscale.gradient = -((dL_dr*rinv)[:,:,None]*x_xl3).sum(0).sum(0)/self.lengthscale**3
            tmp = dL_dr*self._inv_dist(X, X2)
            if X2 is None: X2 = X
            
            if config.getboolean('weave', 'working'):
                try:
                    self.lengthscale.gradient = self.weave_lengthscale_grads(tmp, X, X2)
                except:
                    print "\n Weave compilation failed. Falling back to (slower) numpy implementation\n"
                    config.set('weave', 'working', 'False')
                    self.lengthscale.gradient = np.array([np.einsum('ij,ij,...', tmp, np.square(X[:,q:q+1] - X2[:,q:q+1].T), -1./self.lengthscale[q]**3) for q in xrange(self.input_dim)])
            else:
                self.lengthscale.gradient = np.array([np.einsum('ij,ij,...', tmp, np.square(X[:,q:q+1] - X2[:,q:q+1].T), -1./self.lengthscale[q]**3) for q in xrange(self.input_dim)])
        else:
            r = self._scaled_dist(X, X2)
            self.lengthscale.gradient = -np.sum(dL_dr*r)/self.lengthscale

    def _inv_dist(self, X, X2=None):
        """
        Compute the elementwise inverse of the distance matrix, expecpt on the
        diagonal, where we return zero (the distance on the diagonal is zero).
        This term appears in derviatives.
        """
        dist = self._scaled_dist(X, X2).copy()
        return 1./np.where(dist != 0., dist, np.inf)

    def weave_lengthscale_grads(self, tmp, X, X2):
        """Use scipy.weave to compute derivatives wrt the lengthscales"""
        N,M = tmp.shape
        Q = X.shape[1]
        if hasattr(X, 'values'):X = X.values
        if hasattr(X2, 'values'):X2 = X2.values
        grads = np.zeros(self.input_dim)
        code = """
        double gradq;
        for(int q=0; q<Q; q++){
          gradq = 0;
          for(int n=0; n<N; n++){
            for(int m=0; m<M; m++){
              gradq += tmp(n,m)*(X(n,q)-X2(m,q))*(X(n,q)-X2(m,q));
            }
          }
          grads(q) = gradq;
        }
        """
        weave.inline(code, ['tmp', 'X', 'X2', 'grads', 'N', 'M', 'Q'], type_converters=weave.converters.blitz, support_code="#include <math.h>")
        return -grads/self.lengthscale**3

        """
        Given the derivative of the objective wrt K (dL_dK), compute the derivative wrt X
        """
            try:
                return self.gradients_X_weave(dL_dK, X, X2)
            except:
                return self.gradients_X_(dL_dK, X, X2)
        else:
            return self.gradients_X_(dL_dK, X, X2)

    def gradients_X_(self, dL_dK, X, X2=None):
        invdist = self._inv_dist(X, X2)
        tmp = invdist*dL_dr
            tmp = tmp + tmp.T
            X2 = X

        #The high-memory numpy way:
        #d =  X[:, None, :] - X2[None, :, :]
        #ret = np.sum(tmp[:,:,None]*d,1)/self.lengthscale**2

        #the lower memory way with a loop
        ret = np.empty(X.shape, dtype=np.float64)
        for q in xrange(self.input_dim):
            np.sum(tmp*(X[:,q][:,None]-X2[:,q][None,:]), axis=1, out=ret[:,q])
        ret /= self.lengthscale**2

        return ret

    def gradients_X_weave(self, dL_dK, X, X2=None):
            X2 = X

        code = """
        int n,m,d;
        double retnd;
        #pragma omp parallel for private(n,d, retnd, m)
        for(d=0;d<D;d++){
          for(n=0;n<N;n++){
            retnd = 0.0;
            for(m=0;m<M;m++){
              retnd += tmp(n,m)*(X(n,d)-X2(m,d));
            }
            ret(n,d) = retnd;
          }
        }
 
        """
        if hasattr(X, 'values'):X = X.values #remove the GPy wrapping to make passing into weave safe
        ret = np.zeros(X.shape)
        N,D = X.shape
        from scipy import weave
        #include <omp.h>
        #include <stdio.h>
        """
        weave_options = {'headers'           : ['<omp.h>'],
                         'extra_compile_args': ['-fopenmp -O3'], # -march=native'],
                         'extra_link_args'   : ['-lgomp']}
        weave.inline(code, ['ret', 'N', 'D', 'M', 'tmp', 'X', 'X2'], type_converters=weave.converters.blitz, support_code=support_code, **weave_options)
        return ret/self.lengthscale**2
    
        return np.ones(self.input_dim)/self.lengthscale**2

class Exponential(Stationary):
    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=False, active_dims=None, name='Exponential'):
        super(Exponential, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name)

        return self.variance * np.exp(-0.5 * r)

        return -0.5*self.K_of_r(r)

class OU(Stationary):
    """
    OU kernel:

    .. math::

       k(r) = \\sigma^2 \exp(- r) \\ \\ \\ \\  \\text{ where  } r = \sqrt{\sum_{i=1}^input_dim \\frac{(x_i-y_i)^2}{\ell_i^2} }

    """

    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=False, active_dims=None, name='OU'):
        super(OU, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name)

        return self.variance * np.exp(-r)

    def dK_dr(self,r):
        return -1.*self.variance*np.exp(-r)

class Matern32(Stationary):
    """
    Matern 3/2 kernel:

    .. math::

       k(r) = \\sigma^2 (1 + \\sqrt{3} r) \exp(- \sqrt{3} r) \\ \\ \\ \\  \\text{ where  } r = \sqrt{\sum_{i=1}^input_dim \\frac{(x_i-y_i)^2}{\ell_i^2} }

    """

    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=False, active_dims=None, name='Mat32'):
        super(Matern32, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name)

        return self.variance * (1. + np.sqrt(3.) * r) * np.exp(-np.sqrt(3.) * r)

        return -3.*self.variance*r*np.exp(-np.sqrt(3.)*r)

    def Gram_matrix(self, F, F1, F2, lower, upper):
        """
        Return the Gram matrix of the vector of functions F with respect to the
        RKHS norm. The use of this function is limited to input_dim=1.

        :param F: vector of functions
        :type F: np.array
        :param F1: vector of derivatives of F
        :type F1: np.array
        :param F2: vector of second derivatives of F
        :type F2: np.array
        :param lower,upper: boundaries of the input domain
        :type lower,upper: floats
        """
        assert self.input_dim == 1
        def L(x, i):
            return(3. / self.lengthscale ** 2 * F[i](x) + 2 * np.sqrt(3) / self.lengthscale * F1[i](x) + F2[i](x))
        n = F.shape[0]
        G = np.zeros((n, n))
        for i in range(n):
            for j in range(i, n):
                G[i, j] = G[j, i] = integrate.quad(lambda x : L(x, i) * L(x, j), lower, upper)[0]
        Flower = np.array([f(lower) for f in F])[:, None]
        F1lower = np.array([f(lower) for f in F1])[:, None]
        return(self.lengthscale ** 3 / (12.*np.sqrt(3) * self.variance) * G + 1. / self.variance * np.dot(Flower, Flower.T) + self.lengthscale ** 2 / (3.*self.variance) * np.dot(F1lower, F1lower.T))

class Matern52(Stationary):
    """
    Matern 5/2 kernel:

    .. math::

       k(r) = \sigma^2 (1 + \sqrt{5} r + \\frac53 r^2) \exp(- \sqrt{5} r)
       """
    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=False, active_dims=None, name='Mat52'):
        super(Matern52, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name)

        return self.variance*(1+np.sqrt(5.)*r+5./3*r**2)*np.exp(-np.sqrt(5.)*r)

        return self.variance*(10./3*r -5.*r -5.*np.sqrt(5.)/3*r**2)*np.exp(-np.sqrt(5.)*r)

    def Gram_matrix(self, F, F1, F2, F3, lower, upper):
        """
        Return the Gram matrix of the vector of functions F with respect to the RKHS norm. The use of this function is limited to input_dim=1.

        :param F3: vector of third derivatives of F
        :type F3: np.array
        """
        def L(x,i):
            return(5*np.sqrt(5)/self.lengthscale**3*F[i](x) + 15./self.lengthscale**2*F1[i](x)+ 3*np.sqrt(5)/self.lengthscale*F2[i](x) + F3[i](x))
        G = np.zeros((n,n))
            for j in range(i,n):
                G[i,j] = G[j,i] = integrate.quad(lambda x : L(x,i)*L(x,j),lower,upper)[0]
        G_coef = 3.*self.lengthscale**5/(400*np.sqrt(5))
        Flower = np.array([f(lower) for f in F])[:,None]
        F1lower = np.array([f(lower) for f in F1])[:,None]
        F2lower = np.array([f(lower) for f in F2])[:,None]
        orig = 9./8*np.dot(Flower,Flower.T) + 9.*self.lengthscale**4/200*np.dot(F2lower,F2lower.T)
        orig2 = 3./5*self.lengthscale**2 * ( np.dot(F1lower,F1lower.T) + 1./8*np.dot(Flower,F2lower.T) + 1./8*np.dot(F2lower,Flower.T))
        return(1./self.variance* (G_coef*G + orig + orig2))

class ExpQuad(Stationary):
    """
    The Exponentiated quadratic covariance function.

    .. math::

    notes::
     - Yes, this is exactly the same as the RBF covariance function, but the
       RBF implementation also has some features for doing variational kernels
       (the psi-statistics).

    """
    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=False, active_dims=None, name='ExpQuad'):
        super(ExpQuad, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name)

class Cosine(Stationary):
    def __init__(self, input_dim, variance=1., lengthscale=None, ARD=False, active_dims=None, name='Cosine'):
        super(Cosine, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name)

        return self.variance * np.cos(r)

        return -self.variance * np.sin(r)

class RatQuad(Stationary):
    """
    Rational Quadratic Kernel

    .. math::

       k(r) = \sigma^2 \\bigg( 1 + \\frac{r^2}{2} \\bigg)^{- \\alpha}

    """

    def __init__(self, input_dim, variance=1., lengthscale=None, power=2., ARD=False, active_dims=None, name='RatQuad'):
        super(RatQuad, self).__init__(input_dim, variance, lengthscale, ARD, active_dims, name)
        self.power = Param('power', power, Logexp())
        self.link_parameters(self.power)

        r2 = np.power(r, 2.)
        return self.variance*np.power(1. + r2/2., -self.power)

        return -self.variance*self.power*r*np.power(1. + r2/2., - self.power - 1.)

        super(RatQuad, self).update_gradients_full(dL_dK, X, X2)
        dK_dpow = -self.variance * np.power(2., self.power) * np.power(r2 + 2., -self.power) * np.log(0.5*(r2+2.))
        grad = np.sum(dL_dK*dK_dpow)
        self.power.gradient = grad

        super(RatQuad, self).update_gradients_diag(dL_dKdiag, X)
        self.power.gradient = 0.

# Copyright (c) 2013, the GPy Authors (see AUTHORS.txt)

from ..inference.latent_function_inference.expectation_propagation import EP

class GPClassification(GP):
    """
    Gaussian Process classification

    :param Y: observed values, can be None if likelihood is not None

    """

    def __init__(self, X, Y, kernel=None,Y_metadata=None):

        likelihood = likelihoods.Bernoulli()

        GP.__init__(self, X=X, Y=Y,  kernel=kernel, likelihood=likelihood, inference_method=EP(), name='gp_classification')

# Copyright (c) 2012 - 2014, GPy authors (see AUTHORS.txt).

class Gamma(Likelihood):
    """
    Gamma likelihood

    .. math::
        p(y_{i}|\\lambda(f_{i})) = \\frac{\\beta^{\\alpha_{i}}}{\\Gamma(\\alpha_{i})}y_{i}^{\\alpha_{i}-1}e^{-\\beta y_{i}}\\\\
        \\alpha_{i} = \\beta y_{i}

    """
    def __init__(self,gp_link=None,beta=1.):
        super(Gamma, self).__init__(gp_link, 'Gamma')

        self.beta = Param('beta', beta)
        self.beta.fix()#TODO: gradients!

        """

        .. math::
            p(y_{i}|\\lambda(f_{i})) = \\frac{\\beta^{\\alpha_{i}}}{\\Gamma(\\alpha_{i})}y_{i}^{\\alpha_{i}-1}e^{-\\beta y_{i}}\\\\
            \\alpha_{i} = \\beta y_{i}

        """
        #return stats.gamma.pdf(obs,a = self.gp_link.transf(gp)/self.variance,scale=self.variance)
        alpha = link_f*self.beta
        objective = (y**(alpha - 1.) * np.exp(-self.beta*y) * self.beta**alpha)/ special.gamma(alpha)
        return np.exp(np.sum(np.log(objective)))

        """

        .. math::
            \\ln p(y_{i}|\lambda(f_{i})) = \\alpha_{i}\\log \\beta - \\log \\Gamma(\\alpha_{i}) + (\\alpha_{i} - 1)\\log y_{i} - \\beta y_{i}\\\\

        """
        #alpha = self.gp_link.transf(gp)*self.beta
        #return (1. - alpha)*np.log(obs) + self.beta*obs - alpha * np.log(self.beta) + np.log(special.gamma(alpha))
        log_objective = alpha*np.log(self.beta) - np.log(special.gamma(alpha)) + (alpha - 1)*np.log(y) - self.beta*y

        """

        .. math::
            \\frac{d \\ln p(y_{i}|\\lambda(f_{i}))}{d\\lambda(f)} = \\beta (\\log \\beta y_{i}) - \\Psi(\\alpha_{i})\\beta\\\\

        :param Y_metadata: Y_metadata which is not used in gamma distribution

        """
        grad = self.beta*np.log(self.beta*y) - special.psi(self.beta*link_f)*self.beta
        #old
        #return -self.gp_link.dtransf_df(gp)*self.beta*np.log(obs) + special.psi(self.gp_link.transf(gp)*self.beta) * self.gp_link.dtransf_df(gp)*self.beta

        """

        .. math::
            \\frac{d^{2} \\ln p(y_{i}|\lambda(f_{i}))}{d^{2}\\lambda(f)} = -\\beta^{2}\\frac{d\\Psi(\\alpha_{i})}{d\\alpha_{i}}\\\\

        .. Note::
        """
        hess = -special.polygamma(1, self.beta*link_f)*(self.beta**2)
        #old
        #return -self.gp_link.d2transf_df2(gp)*self.beta*np.log(obs) + special.polygamma(1,self.gp_link.transf(gp)*self.beta)*(self.gp_link.dtransf_df(gp)*self.beta)**2 + special.psi(self.gp_link.transf(gp)*self.beta)*self.gp_link.d2transf_df2(gp)*self.beta

        """

        .. math::
            \\frac{d^{3} \\ln p(y_{i}|\lambda(f_{i}))}{d^{3}\\lambda(f)} = -\\beta^{3}\\frac{d^{2}\\Psi(\\alpha_{i})}{d\\alpha_{i}}\\\\

        """
        d3lik_dlink3 = -special.polygamma(2, self.beta*link_f)*(self.beta**3)

"""
The module of tools for parallelization (MPI)
"""
try:
    def get_id_within_node(comm=MPI.COMM_WORLD):
        rank = comm.rank
        nodename =  MPI.Get_processor_name()
        nodelist = comm.allgather(nodename)
        return len([i for i in nodelist[:rank] if i==nodename])

    numpy_to_MPI_typemap = {
        np.dtype(np.float64) : MPI.DOUBLE,
        np.dtype(np.float32) : MPI.FLOAT,
        np.dtype(np.int)     : MPI.INT,
        np.dtype(np.int8)    : MPI.CHAR,
        np.dtype(np.uint8)   : MPI.UNSIGNED_CHAR,
        np.dtype(np.int32)   : MPI.INT,
        np.dtype(np.uint32)  : MPI.UNSIGNED_INT,
    }
except:
    pass

def divide_data(datanum, rank, size):
    assert rank<size and datanum>0
    
    residue = (datanum)%size
    datanum_list = np.empty((size),dtype=np.int32)
    for i in xrange(size):
        if i<residue:
            datanum_list[i] = int(datanum/size)+1
        else:
            datanum_list[i] = int(datanum/size)
    if rank<residue:
        size = datanum/size+1
        offset = size*rank
    else:
        size = datanum/size
        offset = size*rank+residue
    return offset, offset+size, datanum_list

from scipy import linalg, optimize
import sys
import re
import numdifftools as ndt
import pdb
import cPickle

class Metropolis_Hastings:
    def __init__(self,model,cov=None):
        """Metropolis Hastings, with tunings according to Gelman et al. """
        current = self.model._get_params_transformed()
        self.D = current.size
        self.chains = []
        if cov is None:
            self.cov = model.Laplace_covariance()
        else:
            self.cov = cov
        self.scale = 2.4/np.sqrt(self.D)
        self.new_chain(current)

    def new_chain(self, start=None):
        self.chains.append([])
        if start is None:
            self.model.randomize()
        else:
            self.model._set_params_transformed(start)

    def sample(self, Ntotal, Nburn, Nthin, tune=True, tune_throughout=False, tune_interval=400):
        fcurrent = self.model.log_likelihood() + self.model.log_prior()
        accepted = np.zeros(Ntotal,dtype=np.bool)
        for it in range(Ntotal):
            print "sample %d of %d\r"%(it,Ntotal),
            prop = np.random.multivariate_normal(current, self.cov*self.scale*self.scale)
            self.model._set_params_transformed(prop)
            fprop = self.model.log_likelihood() + self.model.log_prior()

            if fprop>fcurrent:#sample accepted, going 'uphill'
                accepted[it] = True
                current = prop
                fcurrent = fprop
            else:
                u = np.random.rand()
                if np.exp(fprop-fcurrent)>u:#sample accepted downhill
                    accepted[it] = True
                    current = prop
                    fcurrent = fprop

            #store current value
            if (it > Nburn) & ((it%Nthin)==0):
                self.chains[-1].append(current)

            #tuning!
            if it & ((it%tune_interval)==0) & tune & ((it<Nburn) | (tune_throughout)):
                pc = np.mean(accepted[it-tune_interval:it])
                self.cov = np.cov(np.vstack(self.chains[-1][-tune_interval:]).T)
                if pc > .25:
                    self.scale *= 1.1
                if pc < .15:
                    self.scale /= 1.1

    def predict(self,function,args):
        """Make a prediction for the function, to which we will pass the additional arguments"""
        param = self.model._get_params()
        fs = []
        for p in self.chain:
            self.model._set_params(p)
            fs.append(function(*args))
        self.model._set_params(param)# reset model to starting state
        return fs

"""
"""

try:
except:
    pass    

        } else {
        }
    }
    
        int s;
            }
        }
        }
    }

    __global__ void compDenom(double *log_denom1, double *log_denom2, double *l, double *S, int N, int Q)
    {
        
            int n=i/Q;
            int q=i%Q;

        }
    }

    __global__ void psi1computations(double *psi1, double *log_denom1, double var, double *l, double *Z, double *mu, double *S, int N, int M, int Q)
    {
        
                    double muZ = mu[IDX_NQ(n,q)]-Z[IDX_MQ(m,q)];
                    log_psi1 += (muZ*muZ/(Snq+lq)+log_denom1[IDX_NQ(n,q)])/(-2.);
                }
            }
        }
    }
    
    __global__ void psi2computations(double *psi2, double *psi2n, double *log_denom2, double var, double *l, double *Z, double *mu, double *S, int N, int M, int Q)
    {
        
                    double dZ = Z[IDX_MQ(m1,q)] - Z[IDX_MQ(m2,q)];
                    double muZhat = mu[IDX_NQ(n,q)]- (Z[IDX_MQ(m1,q)]+Z[IDX_MQ(m2,q)])/2.;
                    log_psi2_n += dZ*dZ/(-4.*lq)-muZhat*muZhat/(2.*Snq+lq) + log_denom2[IDX_NQ(n,q)]/(-2.);
                }
            }
            }
        }
    }
    
    __global__ void psi1compDer(double *dvar, double *dl, double *dZ, double *dmu, double *dS, double *dL_dpsi1, double *psi1, double var, double *l, double *Z, double *mu, double *S, int N, int M, int Q)
    {

                double Snq,mu_nq;
                if(n<N) {Snq = S[IDX_NQ(n,q)]; mu_nq=mu[IDX_NQ(n,q)];}
                    if(n<N) {
                        
                        double Zmu = Z[IDX_MQ(m,q)] - mu_nq;
                        
                        dmu_local += lpsi1*Zmu/denom;
                        dS_local += lpsi1*(Zmu2_denom-1.)/denom;
                        dl_local += lpsi1*(Zmu2_denom+Snq/lq)/denom;
                        g_local[threadIdx.x] = -lpsi1*Zmu/denom;
                    }
                }
                if(n<N) {
                }
            }
            g_local[threadIdx.x] = dl_local*lq_sqrt;
        }
        if(threadIdx.x==0) {dvar[blockIdx.x] += g_local[0]/var;}        
    }
    
    __global__ void psi2compDer(double *dvar, double *dl, double *dZ, double *dmu, double *dS, double *dL_dpsi2, double *psi2n, double var, double *l, double *Z, double *mu, double *S, int N, int M, int Q)
    {

                        if(n<N) {
                            
                            double dZ = Z[IDX_MQ(m1,q)] - Z[IDX_MQ(m2,q)];
                            double muZhat =  mu_nq - (Z[IDX_MQ(m1,q)] + Z[IDX_MQ(m2,q)])/2.;
                            
                            dmu_local += lpsi2*muZhat/denom;
                            dS_local += lpsi2*(2.*muZhat2_denom-1.)/denom;
                            dl_local += lpsi2*((Snq/lq+muZhat2_denom)/denom+dZ*dZ/(4.*lq*lq));
                            g_local[threadIdx.x] += 2.*lpsi2*(muZhat/denom-dZ/(2*lq));
                        }
                    }
                }
                if(n<N) {
                }
            }
        }
    }
    """

class PSICOMP_RBF_GPU(PSICOMP_RBF):

        self.g_psi1computations.prepare('PPdPPPPiii')
        self.g_psi2computations.prepare('PPPdPPPPiii')
        self.g_psi1compDer.prepare('PPPPPPPdPPPPiii')
        self.g_psi2compDer.prepare('PPPPPPPdPPPPiii')
        self.g_compDenom.prepare('PPPPii')
        
        s = PSICOMP_RBF_GPU(threadnum=self.threadnum, blocknum=self.blocknum, GPU_direct=self.GPU_direct)
        return s
    
                             # grad
                             }
        else:
    
    def sync_params(self, lengthscale, Z, mu, S):
        else:
        # t=self.g_compDenom(self.gpuCache['log_denom1_gpu'],self.gpuCache['log_denom2_gpu'],self.gpuCache['l_gpu'],self.gpuCache['S_gpu'], np.int32(N), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True)
        # print 'g_compDenom '+str(t)
        self.g_compDenom.prepared_call((self.blocknum,1),(self.threadnum,1,1), self.gpuCache['log_denom1_gpu'].gpudata,self.gpuCache['log_denom2_gpu'].gpudata,self.gpuCache['l_gpu'].gpudata,self.gpuCache['S_gpu'].gpudata, np.int32(N), np.int32(Q))
        
        """
        Z - MxQ
        mu - NxQ
        S - NxQ
        """
        self.sync_params(lengthscale, Z, variational_posterior.mean, variational_posterior.variance)
        
        self.g_psi1computations.prepared_call((self.blocknum,1),(self.threadnum,1,1),psi1_gpu.gpudata, log_denom1_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata, np.int32(N), np.int32(M), np.int32(Q))
        self.g_psi2computations.prepared_call((self.blocknum,1),(self.threadnum,1,1),psi2_gpu.gpudata, psi2n_gpu.gpudata, log_denom2_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata, np.int32(N), np.int32(M), np.int32(Q))
        # t = self.g_psi1computations(psi1_gpu, log_denom1_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True)
        # print 'g_psi1computations '+str(t)
        # t = self.g_psi2computations(psi2_gpu, psi2n_gpu, log_denom2_gpu, np.float64(variance),l_gpu,Z_gpu,mu_gpu,S_gpu, np.int32(N), np.int32(M), np.int32(Q), block=(self.threadnum,1,1), grid=(self.blocknum,1),time_kernel=True)
        # print 'g_psi2computations '+str(t)
         
        else:

            dL_dpsi0_sum = dL_dpsi0.get().sum() #gpuarray.sum(dL_dpsi0).get()
        else:

        self.g_psi1compDer.prepared_call((self.blocknum,1),(self.threadnum,1,1),dvar_gpu.gpudata,dl_gpu.gpudata,dZ_gpu.gpudata,dmu_gpu.gpudata,dS_gpu.gpudata,dL_dpsi1_gpu.gpudata,psi1_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata, np.int32(N), np.int32(M), np.int32(Q))
        self.g_psi2compDer.prepared_call((self.blocknum,1),(self.threadnum,1,1),dvar_gpu.gpudata,dl_gpu.gpudata,dZ_gpu.gpudata,dmu_gpu.gpudata,dS_gpu.gpudata,dL_dpsi2_gpu.gpudata,psi2n_gpu.gpudata, np.float64(variance),l_gpu.gpudata,Z_gpu.gpudata,mu_gpu.gpudata,S_gpu.gpudata, np.int32(N), np.int32(M), np.int32(Q))

        dL_dvar = dL_dpsi0_sum + dvar_gpu.get().sum()#gpuarray.sum(dvar_gpu).get()
        if ARD:
        else:
            dL_dlengscale = dl_gpu.get().sum() #gpuarray.sum(dl_gpu).get()
            
        return dL_dvar, dL_dlengscale, dL_dZ, dL_dmu, dL_dS
    
import GPy

class Kernel(Mapping):
    """
    Mapping based on a kernel/covariance function.

    .. math::

       f(\mathbf{x}*) = \mathbf{A}\mathbf{k}(\mathbf{X}, \mathbf{x}^*) + \mathbf{b}

    :param X: input observations containing :math:`\mathbf{X}`
    :param kernel: a GPy kernel, defaults to GPy.kern.RBF
    :type kernel: GPy.kern.kern

    """

    def __init__(self, X, output_dim=1, kernel=None):
        Mapping.__init__(self, input_dim=X.shape[1], output_dim=output_dim)
            kernel = GPy.kern.RBF(self.input_dim)
        self.X = X
        self.num_data = X.shape[0]
        self.num_params = self.output_dim*(self.num_data + 1)
        self.A = np.array((self.num_data, self.output_dim))
        self.bias = np.array(self.output_dim)
        self.name = 'kernel'
        return sum([['A_%i_%i' % (n, d) for d in range(self.output_dim)] for n in range(self.num_data)], []) + ['bias_%i' % d for d in range(self.output_dim)]

        return np.hstack((self.A.flatten(), self.bias))

        self.A = x[:self.num_data * self.output_dim].reshape(self.num_data, self.output_dim).copy()
        self.bias = x[self.num_data*self.output_dim:].copy()

        self.A = np.random.randn(self.num_data, self.output_dim)/np.sqrt(self.num_data+1)
        self.bias = np.random.randn(self.output_dim)/np.sqrt(self.num_data+1)

        return np.dot(self.kern.K(X, self.X),self.A) + self.bias

        return self.kern.gradients_X((dL_df[:, None, :]*self.A[None, :, :]).sum(2), X, self.X)

def conf_matrix(p,labels,names=['1','0'],threshold=.5,show=True):
    """
    Returns error rate and true/false positives in a binary classification problem
    - Actual classes are displayed by column.
    - Predicted classes are displayed by row.

    :param p: array of class '1' probabilities.
    :param labels: array of actual classes.
    :param names: list of class names, defaults to ['1','0'].
    :param threshold: probability value used to decide the class.
    :param show: whether the matrix should be shown or not
    :type show: False|True
    """
    assert p.size == labels.size, "Arrays p and labels have different dimensions."
    decision = np.ones((labels.size,1))
    decision[p<threshold] = 0
    diff = decision - labels
    false_0 = diff[diff == -1].size
    false_1 = diff[diff == 1].size
    true_1 = np.sum(decision[diff ==0])
    true_0 = labels.size - true_1 - false_0 - false_1
    error = (false_1 + false_0)/np.float(labels.size)
    if show:
        print 100. - error * 100,'% instances correctly classified'
        print '%-10s|  %-10s|  %-10s| ' % ('',names[0],names[1])
        print '----------|------------|------------|'
        print '%-10s|  %-10s|  %-10s| ' % (names[0],true_1,false_0)
        print '%-10s|  %-10s|  %-10s| ' % (names[1],false_1,true_0)
    return error,true_1, false_1, true_0, false_0

# Copyright (c) 2014, Zhenwen Dai

from ...core import Model
from ...core.parameterization import variational

def infer_newX(model, Y_new, optimize=True, init='L2'):
    """
    Infer the distribution of X for the new observed data *Y_new*.

    :param model: the GPy model used in inference
    :param Y_new: the new observed data for inference
    :type Y_new: numpy.ndarray
    :param optimize: whether to optimize the location of new X (True by default)
    :type optimize: boolean
    :return: a tuple containing the estimated posterior distribution of X and the model that optimize X
    :rtype: (GPy.core.parameterization.variational.VariationalPosterior, GPy.core.Model)
    """
    infr_m = InferenceX(model, Y_new, init=init)

        infr_m.optimize()

    return infr_m.X, infr_m

class InferenceX(Model):
    """
    The class for inference of new X with given new Y. (do_test_latent)

    :param Y: the new observed data for inference
    :type Y: numpy.ndarray
    """
    def __init__(self, model, Y, name='inferenceX', init='L2'):
        if np.isnan(Y).any() or getattr(model, 'missing_data', False):
            assert Y.shape[0]==1, "The current implementation of inference X only support one data point at a time with missing data!"
            self.missing_data = True
            self.valid_dim = np.logical_not(np.isnan(Y[0]))
            self.ninan = getattr(model, 'ninan', None)
        else:
            self.missing_data = False
        super(InferenceX, self).__init__(name)
        self.likelihood = model.likelihood.copy()
        self.kern = model.kern.copy()
        if model.kern.useGPU:
            from ...models import SSGPLVM
            if isinstance(model, SSGPLVM):
                self.kern.GPU_SSRBF(True)
            else:
                self.kern.GPU(True)
        from copy import deepcopy
        self.posterior = deepcopy(model.posterior)
        if hasattr(model, 'variational_prior'):
            self.uncertain_input = True
            self.variational_prior = model.variational_prior.copy()
        else:
            self.uncertain_input = False
        if hasattr(model, 'inducing_inputs'):
            self.sparse_gp = True
            self.Z = model.Z.copy()
        else:
            self.sparse_gp = False
            self.Z = model.X.copy()
        self.Y = Y
        self.X = self._init_X(model, Y, init=init)
        self.compute_dL()

    def _init_X(self, model, Y_new, init='L2'):
        # Initialize the new X by finding the nearest point in Y space.

        Y = model.Y
            Y = Y[:,self.valid_dim]
            Y_new = Y_new[:,self.valid_dim]
            dist = -2.*Y_new.dot(Y.T) + np.square(Y_new).sum(axis=1)[:,None]+ np.square(Y).sum(axis=1)[None,:]
        else:
            if init=='L2':
                dist = -2.*Y_new.dot(Y.T) + np.square(Y_new).sum(axis=1)[:,None]+ np.square(Y).sum(axis=1)[None,:]
            elif init=='NCC':
                dist = Y_new.dot(Y.T)
            elif init=='rand':
                dist = np.random.rand(Y_new.shape[0],Y.shape[0])
        idx = dist.argmin(axis=1)

        from ...models import SSGPLVM
        from ...util.misc import param_to_array
        if isinstance(model, SSGPLVM):
            X = variational.SpikeAndSlabPosterior(param_to_array(model.X.mean[idx]), param_to_array(model.X.variance[idx]), param_to_array(model.X.gamma[idx]))
            if model.group_spike:
                X.gamma.fix()
        else:
            if self.uncertain_input and self.sparse_gp:
                X = variational.NormalPosterior(param_to_array(model.X.mean[idx]), param_to_array(model.X.variance[idx]))
            else:
                from ...core import Param
                X = Param('latent mean',param_to_array(model.X[idx]).copy())

        return X

    def compute_dL(self):
        # Common computation
        beta = 1./np.fmax(self.likelihood.variance, 1e-6)
        output_dim = self.Y.shape[-1]
        wv = self.posterior.woodbury_vector
            wv = wv[:,self.valid_dim]
            output_dim = self.valid_dim.sum()
            if self.ninan is not None:
                self.dL_dpsi2 = beta/2.*(self.posterior.woodbury_inv[:,:,self.valid_dim] - np.einsum('md,od->mo',wv, wv)[:, :, None]).sum(-1)
            else:
                self.dL_dpsi2 = beta/2.*(output_dim*self.posterior.woodbury_inv - np.einsum('md,od->mo',wv, wv))
            self.dL_dpsi1 = beta*np.dot(self.Y[:,self.valid_dim], wv.T)
            self.dL_dpsi0 = - beta/2.* np.ones(self.Y.shape[0])
        else:
            self.dL_dpsi2 = beta*(output_dim*self.posterior.woodbury_inv - np.einsum('md,od->mo',wv, wv))/2.
            self.dL_dpsi1 = beta*np.dot(self.Y, wv.T)
            self.dL_dpsi0 = -beta/2.*output_dim* np.ones(self.Y.shape[0])

        if self.uncertain_input:
            psi0 = self.kern.psi0(self.Z, self.X)
            psi1 = self.kern.psi1(self.Z, self.X)
            psi2 = self.kern.psi2(self.Z, self.X)
        else:
            psi0 = self.kern.Kdiag(self.X)
            psi1 = self.kern.K(self.X, self.Z)
            psi2 = np.dot(psi1.T,psi1)

        self._log_marginal_likelihood = (self.dL_dpsi2*psi2).sum()+(self.dL_dpsi1*psi1).sum()+(self.dL_dpsi0*psi0).sum()

            X_grad = self.kern.gradients_qX_expectations(variational_posterior=self.X, Z=self.Z, dL_dpsi0=self.dL_dpsi0, dL_dpsi1=self.dL_dpsi1, dL_dpsi2=self.dL_dpsi2)
            self.X.set_gradients(X_grad)
        else:
            dL_dpsi1 = self.dL_dpsi1 + 2.*np.dot(psi1,self.dL_dpsi2)
            X_grad = self.kern.gradients_X_diag(self.dL_dpsi0, self.X)
            X_grad += self.kern.gradients_X(dL_dpsi1, self.X, self.Z)
            self.X.gradient = X_grad

            from ...core.parameterization.variational import SpikeAndSlabPrior
            if isinstance(self.variational_prior, SpikeAndSlabPrior):
                # Update Log-likelihood
                KL_div = self.variational_prior.KL_divergence(self.X, N=self.Y.shape[0])
                # update for the KL divergence
                self.variational_prior.update_gradients_KL(self.X, N=self.Y.shape[0])
            else:
                KL_div = self.variational_prior.KL_divergence(self.X)
                self.variational_prior.update_gradients_KL(self.X)
            self._log_marginal_likelihood += -KL_div

#TODO
"""
A lot of this code assumes that the link function is the identity.

I think laplace code is okay, but I'm quite sure that the EP moments will only work if the link is identity.

Furthermore, exact Guassian inference can only be done for the identity link, so we should be asserting so for all calls which relate to that.

James 11/12/13
"""

class Gaussian(Likelihood):
    """
    Gaussian likelihood

    .. math::
        \\ln p(y_{i}|\\lambda(f_{i})) = -\\frac{N \\ln 2\\pi}{2} - \\frac{\\ln |K|}{2} - \\frac{(y_{i} - \\lambda(f_{i}))^{T}\\sigma^{-2}(y_{i} - \\lambda(f_{i}))}{2}

    :param variance: variance value of the Gaussian distribution
    :param N: Number of data points
    :type N: int
    """
    def __init__(self, gp_link=None, variance=1., name='Gaussian_noise'):

        assert isinstance(gp_link, link_functions.Identity), "the likelihood only implemented for the identity link"

        super(Gaussian, self).__init__(gp_link, name=name)

        if isinstance(gp_link, link_functions.Identity):

    def betaY(self,Y,Y_metadata=None):
        #TODO: ~Ricardo this does not live here
        return Y/self.gaussian_variance(Y_metadata)

    def gaussian_variance(self, Y_metadata=None):
        return self.variance

    def update_gradients(self, grad):
        self.variance.gradient = grad

        return dL_dKdiag.sum()

        """
        """
        return Y

    def _moments_match_ep(self, data_i, tau_i, v_i):
        """

        """
        sigma2_hat = 1./(1./self.variance + tau_i)
        mu_hat = sigma2_hat*(data_i/self.variance + v_i)
        sum_var = self.variance + 1./tau_i
        Z_hat = 1./np.sqrt(2.*np.pi*sum_var)*np.exp(-.5*(data_i - v_i/tau_i)**2./sum_var)

            if var.ndim == 2:
                var += np.eye(var.shape[0])*self.variance
            if var.ndim == 3:
                var += np.atleast_3d(np.eye(var.shape[0])*self.variance)
        else:
            var += self.variance

    def predictive_mean(self, mu, sigma):
        return mu

    def predictive_variance(self, mu, sigma, predictive_mean=None):
        return self.variance + sigma**2

        return  [stats.norm.ppf(q/100.)*np.sqrt(var + self.variance) + mu for q in quantiles]

        """

        .. math::
            \\ln p(y_{i}|\\lambda(f_{i})) = -\\frac{N \\ln 2\\pi}{2} - \\frac{\\ln |K|}{2} - \\frac{(y_{i} - \\lambda(f_{i}))^{T}\\sigma^{-2}(y_{i} - \\lambda(f_{i}))}{2}

        :param Y_metadata: Y_metadata not used in gaussian
        """
        #Assumes no covariance, exp, sum, log for numerical stability
        return np.exp(np.sum(np.log(stats.norm.pdf(y, link_f, np.sqrt(self.variance)))))

        """
        Log likelihood function given link(f)

        .. math::

        """
        assert np.asarray(link_f).shape == np.asarray(y).shape
        N = y.shape[0]
        ln_det_cov = N*np.log(self.variance)

        return -0.5*(np.sum((y-link_f)**2/self.variance) + ln_det_cov + N*np.log(2.*np.pi))

        """
        Gradient of the pdf at y, given link(f) w.r.t link(f)

        .. math::
            \\frac{d \\ln p(y_{i}|\\lambda(f_{i}))}{d\\lambda(f)} = \\frac{1}{\\sigma^{2}}(y_{i} - \\lambda(f_{i}))

        :returns: gradient of log likelihood evaluated at points link(f)
        """
        s2_i = (1.0/self.variance)
        grad = s2_i*y - s2_i*link_f

        """
        Hessian at y, given link_f, w.r.t link_f.
        i.e. second derivative logpdf at y given link(f_i) link(f_j)  w.r.t link(f_i) and link(f_j)

        .. math::
            \\frac{d^{2} \\ln p(y_{i}|\\lambda(f_{i}))}{d^{2}f} = -\\frac{1}{\\sigma^{2}}

        :returns: Diagonal of log hessian matrix (second derivative of log likelihood evaluated at points link(f))

        .. Note::
        """
        hess = -(1.0/self.variance)*np.ones((N, 1))

        """

        .. math::
            \\frac{d^{3} \\ln p(y_{i}|\\lambda(f_{i}))}{d^{3}\\lambda(f)} = 0

        :returns: third derivative of log likelihood evaluated at points link(f)
        """
        d3logpdf_dlink3 = np.zeros((N,1))

    def dlogpdf_link_dvar(self, link_f, y, Y_metadata=None):
        """
        Gradient of the log-likelihood function at y given link(f), w.r.t variance parameter (noise_variance)

        .. math::
            \\frac{d \\ln p(y_{i}|\\lambda(f_{i}))}{d\\sigma^{2}} = -\\frac{N}{2\\sigma^{2}} + \\frac{(y_{i} - \\lambda(f_{i}))^{2}}{2\\sigma^{4}}

        :returns: derivative of log likelihood evaluated at points link(f) w.r.t variance parameter
        """
        e = y - link_f
        s_4 = 1.0/(self.variance**2)
        dlik_dsigma = -0.5*N/self.variance + 0.5*s_4*np.sum(np.square(e))
        return np.sum(dlik_dsigma) # Sure about this sum?

    def dlogpdf_dlink_dvar(self, link_f, y, Y_metadata=None):
        """
        Derivative of the dlogpdf_dlink w.r.t variance parameter (noise_variance)

        .. math::
            \\frac{d}{d\\sigma^{2}}(\\frac{d \\ln p(y_{i}|\\lambda(f_{i}))}{d\\lambda(f)}) = \\frac{1}{\\sigma^{4}}(-y_{i} + \\lambda(f_{i}))

        """
        dlik_grad_dsigma = -s_4*y + s_4*link_f
        return dlik_grad_dsigma

    def d2logpdf_dlink2_dvar(self, link_f, y, Y_metadata=None):
        """
        Gradient of the hessian (d2logpdf_dlink2) w.r.t variance parameter (noise_variance)

        .. math::
            \\frac{d}{d\\sigma^{2}}(\\frac{d^{2} \\ln p(y_{i}|\\lambda(f_{i}))}{d^{2}\\lambda(f)}) = \\frac{1}{\\sigma^{4}}

        :returns: derivative of log hessian evaluated at points link(f_i) and link(f_j) w.r.t variance parameter
        """
        d2logpdf_dlink2_dvar = np.ones((N,1))*s_4

        return np.asarray([[dlogpdf_dvar]])

    def _mean(self, gp):
        """
        Expected value of y under the Mass (or density) function p(y|f)

        .. math::
            E_{p(y|f)}[y]
        """

    def _variance(self, gp):
        """
        Variance of y under the Mass (or density) function p(y|f)

        .. math::
            Var_{p(y|f)}[y]
        """

        """

        """
        #orig_shape = gp.shape
        Ysim = np.array([np.random.normal(self.gp_link.transf(gpj), scale=np.sqrt(self.variance), size=1) for gpj in gp])

        """
        assumes independence
        """
        v = var_star + self.variance
        return -0.5*np.log(2*np.pi) -0.5*np.log(v) - 0.5*np.square(y_test - mu_star)/v

# Copyright (c) 2013,2014, GPy authors (see AUTHORS.txt).

import sys

class Mapping(Parameterized):
    """
    Base model for shared behavior between models that can act like a mapping.
    """

    def __init__(self, input_dim, output_dim, name='mapping'):
        super(Mapping, self).__init__(name=name)

        """Evaluate derivatives of mapping outputs with respect to inputs.

        :param dL_df: gradient of the objective with respect to the function.
        :type dL_df: ndarray (num_data x output_dim)
        :param X: the input locations where derivatives are to be evaluated.
        :type X: ndarray (num_data x input_dim)
        :returns: matrix containing gradients of the function with respect to the inputs.
        """

        """The gradient of the outputs of the mapping with respect to each of the parameters.

        :param X: input locations where the function is evaluated.
        :returns: Matrix containing gradients with respect to parameters of each output for each input data.
        :rtype: ndarray (num_params length)
        """

        """
        Plots the mapping associated with the model.
          - In one dimension, the function is plotted.
          - In two dimensions, a contour-plot shows the function
          - In higher dimensions, we've not implemented this yet !TODO!

        using which_data and which_functions

        This is a convenience function: arguments are passed to
        GPy.plotting.matplot_dep.models_plots.plot_mapping
        """

        if "matplotlib" in sys.modules:
            from ..plotting.matplot_dep import models_plots
            mapping_plots.plot_mapping(self,*args)
        else:
            raise NameError, "matplotlib package has not been imported."

class Bijective_mapping(Mapping):
    """
    This is a mapping that is bijective, i.e. you can go from X to f and
    also back from f to X. The inverse mapping is called g().
    """
    def __init__(self, input_dim, output_dim, name='bijective_mapping'):
        super(Bijective_apping, self).__init__(name=name)

    def g(self, f):
        """Inverse mapping from output domain of the function to the inputs."""

class Mapping_check_model(Model):
    """
    gradients of a given mapping are implemented correctly. It enables
    checkgradient() to be called independently on each mapping.
    """
    def __init__(self, mapping=None, dL_df=None, X=None):
        num_samples = 20
        if mapping==None:
            mapping = GPy.mapping.linear(1, 1)
        if X==None:
            X = np.random.randn(num_samples, mapping.input_dim)
        if dL_df==None:
            dL_df = np.ones((num_samples, mapping.output_dim))

        self.mapping=mapping
        self.X = X
        self.dL_df = dL_df
        self.num_params = self.mapping.num_params
        Model.__init__(self)

        return self.mapping._get_params()

        return self.mapping._get_param_names()

        self.mapping._set_params(x)

        return (self.dL_df*self.mapping.f(self.X)).sum()

        raise NotImplementedError, "This needs to be implemented to use the Mapping_check_model class."

class Mapping_check_df_dtheta(Mapping_check_model):
    """This class allows gradient checks for the gradient of a mapping with respect to parameters. """
        Mapping_check_model.__init__(self,mapping=mapping,dL_df=dL_df, X=X)

        return self.mapping.df_dtheta(self.dL_df, self.X)

class Mapping_check_df_dX(Mapping_check_model):
    """This class allows gradient checks for the gradient of a mapping with respect to X. """

            dL_df = np.ones((self.X.shape[0],self.mapping.output_dim))
        self.num_params = self.X.shape[0]*self.mapping.input_dim

        return self.mapping.df_dX(self.dL_df, self.X).flatten()

        return ['X_'  +str(i) + ','+str(j) for j in range(self.X.shape[1]) for i in range(self.X.shape[0])]

        return self.X.flatten()

        self.X=x.reshape(self.X.shape)

"""
The package for the Psi statistics computation of the linear kernel for Bayesian GPLVM
"""

    """
    """
    # psi0    N
    # psi1    NxM
    # psi2    MxM

    psi0 = (variance*(np.square(mu)+S)).sum(axis=1)
    psi1 = np.dot(mu,(variance*Z).T)
    psi2 = np.dot(S.sum(axis=0)*np.square(variance)*Z,Z.T)+ tdot(psi1.T)

    dL_dvar, dL_dmu, dL_dS, dL_dZ = _psi2computations(dL_dpsi2, variance, Z, mu, S)

    dL_dpsi0_var = dL_dpsi0[:,None]*variance[None,:]
    dL_dpsi1_mu = np.dot(dL_dpsi1.T,mu)
    dL_dvar += (dL_dpsi0[:,None]*mu2S).sum(axis=0)+ (dL_dpsi1_mu*Z).sum(axis=0)
    dL_dmu += 2.*dL_dpsi0_var*mu+np.dot(dL_dpsi1,Z)*variance
    dL_dS += dL_dpsi0_var
    dL_dZ += dL_dpsi1_mu*variance
    
    return dL_dvar, dL_dZ, dL_dmu, dL_dS

def _psi2computations(dL_dpsi2, variance, Z, mu, S):
    """
    Z - MxQ
    mu - NxQ
    S - NxQ
    gamma - NxQ
    """
    
    common_sum = np.dot(mu,(variance*Z).T)
    Z_expect = (np.dot(dL_dpsi2,Z)*Z).sum(axis=0)
    common_expect = np.dot(common_sum,np.dot(dL_dpsi2T,Z))
    Z2_expect = np.inner(common_sum,dL_dpsi2T)
    Z1_expect = np.dot(dL_dpsi2T,Z)

    dL_dvar = 2.*S.sum(axis=0)*variance*Z_expect+(common_expect*mu).sum(axis=0)
            
    dL_dmu = common_expect*variance
    
    dL_dS = np.empty(S.shape)
    dL_dS[:] = Z_expect*variance2
    
    dL_dZ = variance2*S.sum(axis=0)*Z1_expect+np.dot(Z2_expect.T,variance*mu)

    return dL_dvar, dL_dmu, dL_dS, dL_dZ

import GPy

def get_slices(input_list):
    num_outputs = len(input_list)
    _s = [0] + [ _x.shape[0] for _x in input_list ]
    _s = np.cumsum(_s)
    slices = [slice(a,b) for a,b in zip(_s[:-1],_s[1:])]
    return slices

def build_XY(input_list,output_list=None,index=None):
    if output_list is not None:
        assert num_outputs == len(output_list)
        Y = np.vstack(output_list)
    else:
        Y = None

    if index is not None:
        assert len(index) == num_outputs
        I = np.hstack( [np.repeat(j,_x.shape[0]) for _x,j in zip(input_list,index)] )
    else:
        I = np.hstack( [np.repeat(j,_x.shape[0]) for _x,j in zip(input_list,range(num_outputs))] )

    X = np.vstack(input_list)
    X = np.hstack([X,I[:,None]])

    return X,Y,I[:,None]#slices

def build_likelihood(Y_list,noise_index,likelihoods_list=None):
    Ny = len(Y_list)
    if likelihoods_list is None:
       likelihoods_list = [GPy.likelihoods.Gaussian(name="Gaussian_noise_%s" %j) for y,j in zip(Y_list,range(Ny))]
    else:
        assert len(likelihoods_list) == Ny
    #likelihood = GPy.likelihoods.mixed_noise.MixedNoise(likelihoods_list=likelihoods_list, noise_index=noise_index)
    likelihood = GPy.likelihoods.mixed_noise.MixedNoise(likelihoods_list=likelihoods_list)
    return likelihood

def ICM(input_dim, num_outputs, kernel, W_rank=1,W=None,kappa=None,name='ICM'):
    """
    Builds a kernel for an Intrinsic Coregionalization Model

    :input_dim: Input dimensionality (does not include dimension of indices)
    :num_outputs: Number of outputs
    :param kernel: kernel that will be multiplied by the coregionalize kernel (matrix B).
    :param W_rank: number tuples of the corregionalization parameters 'W'
    """
    if kernel.input_dim <> input_dim:
        kernel.input_dim = input_dim
        warnings.warn("kernel's input dimension overwritten to fit input_dim parameter.")

    K = kernel.prod(GPy.kern.Coregionalize(1, num_outputs, active_dims=[input_dim], rank=W_rank,W=W,kappa=kappa,name='B'),name=name)
    return K

def LCM(input_dim, num_outputs, kernels_list, W_rank=1,name='ICM'):
    """
    Builds a kernel for an Linear Coregionalization Model

    """
    Nk = len(kernels_list)
    K = ICM(input_dim,num_outputs,kernels_list[0],W_rank,name='%s%s' %(name,0))
    j = 1
    for kernel in kernels_list[1:]:
        K += ICM(input_dim,num_outputs,kernel,W_rank,name='%s%s' %(name,j))
        j += 1
    return K

def Private(input_dim, num_outputs, kernel, output, kappa=None,name='X'):
    """

    :input_dim: Input dimensionality
    """
    K = ICM(input_dim,num_outputs,kernel,W_rank=1,kappa=kappa,name=name)
    K.B.W.fix(0)
    _range = range(num_outputs)
    _range.pop(output)
    for j in _range:
        K.B.kappa[j] = 0
        K.B.kappa[j].fix()
    return K

import datetime as dt
from scipy import optimize
from warnings import warn

try:
    import rasmussens_minimize as rasm
    rasm_available = True
    rasm_available = False
from scg import SCG

class Optimizer():
    """
    Superclass for all the optimizers.

    :param x_init: initial set of parameters
    :param f_fp: function that returns the function AND the gradients at the same time
    :param f: function to optimize
    :param fp: gradients
    :param messages: print messages from the optimizer?
    :type messages: (True | False)
    :param max_f_eval: maximum number of function evaluations

    :rtype: optimizer object.

    """
    def __init__(self, x_init, messages=False, model=None, max_f_eval=1e4, max_iters=1e3,
                 ftol=None, gtol=None, xtol=None, bfgs_factor=None):
        self.opt_name = None
        self.x_init = x_init
        self.f_opt = None
        self.x_opt = None
        self.funct_eval = None
        self.status = None
        self.max_f_eval = int(max_f_eval)
        self.max_iters = int(max_iters)
        self.bfgs_factor = bfgs_factor
        self.trace = None
        self.time = "Not available"
        self.xtol = xtol
        self.ftol = ftol

    def run(self, **kwargs):
        start = dt.datetime.now()
        self.opt(**kwargs)
        end = dt.datetime.now()
        self.time = str(end - start)

    def opt(self, f_fp=None, f=None, fp=None):
        raise NotImplementedError, "this needs to be implemented to use the optimizer class"

        """
        See GPy.plotting.matplot_dep.inference_plots
        """
        import sys
        from ...plotting.matplot_dep import inference_plots
        inference_plots.plot_optimizer(self)

        diagnostics = "Optimizer: \t\t\t\t %s\n" % self.opt_name
        diagnostics += "f(x_opt): \t\t\t\t %.3f\n" % self.f_opt
        diagnostics += "Number of function evaluations: \t %d\n" % self.funct_eval
        diagnostics += "Optimization status: \t\t\t %s\n" % self.status
        diagnostics += "Time elapsed: \t\t\t\t %s\n" % self.time
        return diagnostics

class opt_tnc(Optimizer):
    def __init__(self, *args, **kwargs):
        Optimizer.__init__(self, *args, **kwargs)
        self.opt_name = "TNC (Scipy implementation)"

        """
        Run the TNC optimizer

        """
        tnc_rcstrings = ['Local minimum', 'Converged', 'XConverged', 'Maximum number of f evaluations reached',
             'Line search failed', 'Function is constant']

        assert f_fp != None, "TNC requires f_fp"

        opt_dict = {}
        if self.xtol is not None:
            opt_dict['xtol'] = self.xtol
        if self.ftol is not None:
            opt_dict['ftol'] = self.ftol
        if self.gtol is not None:
            opt_dict['pgtol'] = self.gtol

        opt_result = optimize.fmin_tnc(f_fp, self.x_init, messages=self.messages,
                       maxfun=self.max_f_eval, **opt_dict)
        self.x_opt = opt_result[0]
        self.f_opt = f_fp(self.x_opt)[0]
        self.funct_eval = opt_result[1]
        self.status = tnc_rcstrings[opt_result[2]]

class opt_lbfgsb(Optimizer):
        self.opt_name = "L-BFGS-B (Scipy implementation)"

        """
        Run the optimizer

        """
        rcstrings = ['Converged', 'Maximum number of f evaluations reached', 'Error']

        assert f_fp != None, "BFGS requires f_fp"

            iprint = 1
        else:
            iprint = -1

            print "WARNING: l-bfgs-b doesn't have an xtol arg, so I'm going to ignore it"
            print "WARNING: l-bfgs-b doesn't have an ftol arg, so I'm going to ignore it"
        if self.bfgs_factor is not None:
            opt_dict['factr'] = self.bfgs_factor

        opt_result = optimize.fmin_l_bfgs_b(f_fp, self.x_init, iprint=iprint,
                                            maxfun=self.max_iters, **opt_dict)
        self.funct_eval = opt_result[2]['funcalls']
        self.status = rcstrings[opt_result[2]['warnflag']]

class opt_simplex(Optimizer):
        self.opt_name = "Nelder-Mead simplex routine (via Scipy)"

        """
        The simplex optimizer does not require gradients.
        """

        statuses = ['Converged', 'Maximum number of function evaluations made', 'Maximum number of iterations reached']

            print "WARNING: simplex doesn't have an gtol arg, so I'm going to ignore it"

        opt_result = optimize.fmin(f, self.x_init, (), disp=self.messages,
                   maxfun=self.max_f_eval, full_output=True, **opt_dict)

        self.f_opt = opt_result[1]
        self.funct_eval = opt_result[3]
        self.status = statuses[opt_result[4]]

class opt_rasm(Optimizer):
        self.opt_name = "Rasmussen's Conjugate Gradient"

        """
        Run Rasmussen's Conjugate Gradient optimizer
        """

        assert f_fp != None, "Rasmussen's minimizer requires f_fp"
        statuses = ['Converged', 'Line search failed', 'Maximum number of f evaluations reached',
                'NaNs in optimization']

            print "WARNING: minimize doesn't have an xtol arg, so I'm going to ignore it"
            print "WARNING: minimize doesn't have an ftol arg, so I'm going to ignore it"
            print "WARNING: minimize doesn't have an gtol arg, so I'm going to ignore it"

        opt_result = rasm.minimize(self.x_init, f_fp, (), messages=self.messages,
                                   maxnumfuneval=self.max_f_eval)
        self.f_opt = opt_result[1][-1]
        self.funct_eval = opt_result[2]
        self.status = statuses[opt_result[3]]

        self.trace = opt_result[1]

class opt_SCG(Optimizer):
        if 'max_f_eval' in kwargs:
            warn("max_f_eval deprecated for SCG optimizer: use max_iters instead!\nIgnoring max_f_eval!", FutureWarning)

        self.opt_name = "Scaled Conjugate Gradients"

        assert not f is None
        assert not fp is None

        opt_result = SCG(f, fp, self.x_init, display=self.messages,
                         maxiters=self.max_iters,
                         max_f_eval=self.max_f_eval,
                         xtol=self.xtol, ftol=self.ftol,
                         gtol=self.gtol)

        self.f_opt = self.trace[-1]
        self.status = opt_result[3]

def get_optimizer(f_min):

    optimizers = {'fmin_tnc': opt_tnc,
          'simplex': opt_simplex,
          'lbfgsb': opt_lbfgsb,
          'scg': opt_SCG}

    if rasm_available:
        optimizers['rasmussen'] = opt_rasm

    for opt_name in optimizers.keys():
        if opt_name.lower().find(f_min.lower()) != -1:
            return optimizers[opt_name]

    raise KeyError('No optimizer was found matching the name: %s' % f_min)

from ...util.linalg import pdinv, dpotrs, dpotri, symmetrify, jitchol

class Posterior(object):
    """
    An object to represent a Gaussian posterior over latent function values, p(f|D).
    This may be computed exactly for Gaussian likelihoods, or approximated for
    non-Gaussian likelihoods.

    The purpose of this class is to serve as an interface between the inference
    schemes and the model classes.  the model class can make predictions for
    the function at any new point x_* by integrating over this posterior.

    """
    def __init__(self, woodbury_chol=None, woodbury_vector=None, K=None, mean=None, cov=None, K_chol=None, woodbury_inv=None):
        """
        woodbury_chol : a lower triangular matrix L that satisfies posterior_covariance = K - K L^{-T} L^{-1} K
        woodbury_vector : a matrix (or vector, as Nx1 matrix) M which satisfies posterior_mean = K M
        K : the proir covariance (required for lazy computation of various quantities)
        mean : the posterior mean
        cov : the posterior covariance

        Not all of the above need to be supplied! You *must* supply:

          K (for lazy computation)
          or
          K_chol (for lazy computation)

       You may supply either:

          woodbury_chol
          woodbury_vector

        Or:

          mean
          cov

        Of course, you can supply more than that, but this class will lazily
        compute all other quantites on demand.

        """
        #obligatory
        self._K = K

        if ((woodbury_chol is not None) and (woodbury_vector is not None))\
                or ((woodbury_inv is not None) and (woodbury_vector is not None))\
                or ((woodbury_inv is not None) and (mean is not None))\
                or ((mean is not None) and (cov is not None)):
            pass # we have sufficient to compute the posterior
        else:
            raise ValueError, "insufficient information to compute the posterior"

        self._K_chol = K_chol
        self._K = K
        #option 1:
        self._woodbury_chol = woodbury_chol
        self._woodbury_vector = woodbury_vector

        #option 2.
        self._woodbury_inv = woodbury_inv
        #and woodbury vector

        #option 2:
        self._mean = mean
        self._covariance = cov

        #compute this lazily
        self._precision = None

    @property
    def mean(self):
        """
        Posterior mean
        $$
        K_{xx}v
        v := \texttt{Woodbury vector}
        $$
        """
        if self._mean is None:
            self._mean = np.dot(self._K, self.woodbury_vector)
        return self._mean

    @property
    def covariance(self):
        """
        Posterior covariance
        $$
        K_{xx} - K_{xx}W_{xx}^{-1}K_{xx}
        W_{xx} := \texttt{Woodbury inv}
        $$
        """
        if self._covariance is None:
            #LiK, _ = dtrtrs(self.woodbury_chol, self._K, lower=1)
            self._covariance = (np.atleast_3d(self._K) - np.tensordot(np.dot(np.atleast_3d(self.woodbury_inv).T, self._K), self._K, [1,0]).T).squeeze()
            #self._covariance = self._K - self._K.dot(self.woodbury_inv).dot(self._K)
        return self._covariance

    @property
    def precision(self):
        """
        Inverse of posterior covariance
        """
        if self._precision is None:
            cov = np.atleast_3d(self.covariance)
            self._precision = np.zeros(cov.shape) # if one covariance per dimension
            for p in xrange(cov.shape[-1]):
                self._precision[:,:,p] = pdinv(cov[:,:,p])[0]
        return self._precision

    @property
    def woodbury_chol(self):
        """
        return $L_{W}$ where L is the lower triangular Cholesky decomposition of the Woodbury matrix
        $$
        L_{W}L_{W}^{\top} = W^{-1}
        W^{-1} := \texttt{Woodbury inv}
        $$
        """
        if self._woodbury_chol is None:
            #compute woodbury chol from
            if self._woodbury_inv is not None:
                winv = np.atleast_3d(self._woodbury_inv)
                self._woodbury_chol = np.zeros(winv.shape)
                for p in xrange(winv.shape[-1]):
                    self._woodbury_chol[:,:,p] = pdinv(winv[:,:,p])[2]
                #Li = jitchol(self._woodbury_inv)
                #self._woodbury_chol, _ = dtrtri(Li)
                #W, _, _, _, = pdinv(self._woodbury_inv)
                #symmetrify(W)
                #self._woodbury_chol = jitchol(W)
            #try computing woodbury chol from cov
            elif self._covariance is not None:
                raise NotImplementedError, "TODO: check code here"
                B = self._K - self._covariance
                tmp, _ = dpotrs(self.K_chol, B)
                self._woodbury_inv, _ = dpotrs(self.K_chol, tmp.T)
                _, _, self._woodbury_chol, _ = pdinv(self._woodbury_inv)
            else:
                raise ValueError, "insufficient information to compute posterior"
        return self._woodbury_chol

    @property
    def woodbury_inv(self):
        """
        The inverse of the woodbury matrix, in the gaussian likelihood case it is defined as
        $$
        (K_{xx} + \Sigma_{xx})^{-1}
        \Sigma_{xx} := \texttt{Likelihood.variance / Approximate likelihood covariance}
        $$
        """
        if self._woodbury_inv is None:
            if self._woodbury_chol is not None:
                self._woodbury_inv, _ = dpotri(self._woodbury_chol, lower=1)
                #self._woodbury_inv, _ = dpotrs(self.woodbury_chol, np.eye(self.woodbury_chol.shape[0]), lower=1)
                symmetrify(self._woodbury_inv)
                self._woodbury_inv, _ = dpotrs(self.K_chol, tmp.T)                
        return self._woodbury_inv

    @property
    def woodbury_vector(self):
        """
        Woodbury vector in the gaussian likelihood case only is defined as
        $$
        (K_{xx} + \Sigma)^{-1}Y
        \Sigma := \texttt{Likelihood.variance / Approximate likelihood covariance}
        $$
        """
        if self._woodbury_vector is None:
            self._woodbury_vector, _ = dpotrs(self.K_chol, self.mean)
        return self._woodbury_vector

    @property
    def K_chol(self):
        """
        Cholesky of the prior covariance K
        """
        if self._K_chol is None:
            self._K_chol = jitchol(self._K)
        return self._K_chol

from kern import CombinationKernel

class Add(CombinationKernel):
    """
    Add given list of kernels together.
    propagates gradients through.

    This kernel will take over the active dims of it's subkernels passed in.
    """
    def __init__(self, subkerns, name='add'):
        for i, kern in enumerate(subkerns[:]):
            if isinstance(kern, Add):
                del subkerns[i]
                for part in kern.parts[::-1]:
                    kern.unlink_parameter(part)
                    subkerns.insert(i, part)

        super(Add, self).__init__(subkerns, name)

    @Cache_this(limit=2, force_kwargs=['which_parts'])
    def K(self, X, X2=None, which_parts=None):
        """
        Add all kernels together.
        If a list of parts (of this kernel!) `which_parts` is given, only
        the parts of the list are taken to compute the covariance.
        """
        if which_parts is None:
            which_parts = self.parts
        elif not isinstance(which_parts, (list, tuple)):
            # if only one part is given
            which_parts = [which_parts]
        return reduce(np.add, (p.K(X, X2) for p in which_parts))

    def Kdiag(self, X, which_parts=None):
        return reduce(np.add, (p.Kdiag(X) for p in which_parts))

        [p.update_gradients_full(dL_dK, X, X2) for p in self.parts if not p.is_fixed]

    def update_gradients_diag(self, dL_dK, X):
        [p.update_gradients_diag(dL_dK, X) for p in self.parts]

        """Compute the gradient of the objective function with respect to X.

        :param dL_dK: An array of gradients of the objective function with respect to the covariance function.
        :type dL_dK: np.ndarray (num_samples x num_inducing)
        :param X: Observed data inputs
        :type X: np.ndarray (num_samples x input_dim)
        :param X2: Observed data inputs (optional, defaults to X)
        :type X2: np.ndarray (num_inducing x input_dim)"""

        [target.__iadd__(p.gradients_X(dL_dK, X, X2)) for p in self.parts]

        [target.__iadd__(p.gradients_X_diag(dL_dKdiag, X)) for p in self.parts]
    
        return reduce(np.add, (p.psi0(Z, variational_posterior) for p in self.parts))
    
        return reduce(np.add, (p.psi1(Z, variational_posterior) for p in self.parts))

        psi2 = reduce(np.add, (p.psi2(Z, variational_posterior) for p in self.parts))
        #return psi2
        # compute the "cross" terms
        from static import White, Bias
        from rbf import RBF
        #from rbf_inv import RBFInv
        from linear import Linear
        #ffrom fixed import Fixed

        for p1, p2 in itertools.combinations(self.parts, 2):
            # i1, i2 = p1.active_dims, p2.active_dims
            # white doesn;t combine with anything
            if isinstance(p1, White) or isinstance(p2, White):
                pass
            # rbf X bias
            #elif isinstance(p1, (Bias, Fixed)) and isinstance(p2, (RBF, RBFInv)):
            elif isinstance(p1,  Bias) and isinstance(p2, (RBF, Linear)):
                tmp = p2.psi1(Z, variational_posterior).sum(axis=0)
                psi2 += p1.variance * (tmp[:,None]+tmp[None,:]) #(tmp[:, :, None] + tmp[:, None, :])
            #elif isinstance(p2, (Bias, Fixed)) and isinstance(p1, (RBF, RBFInv)):
            elif isinstance(p2, Bias) and isinstance(p1, (RBF, Linear)):
                tmp = p1.psi1(Z, variational_posterior).sum(axis=0)
                psi2 += p2.variance * (tmp[:,None]+tmp[None,:]) #(tmp[:, :, None] + tmp[:, None, :])
            elif isinstance(p2, (RBF, Linear)) and isinstance(p1, (RBF, Linear)):
                assert np.intersect1d(p1.active_dims, p2.active_dims).size == 0, "only non overlapping kernel dimensions allowed so far"
                tmp1 = p1.psi1(Z, variational_posterior)
                tmp2 = p2.psi1(Z, variational_posterior)
                psi2 += np.einsum('nm,no->mo',tmp1,tmp2)+np.einsum('nm,no->mo',tmp2,tmp1)
                #(tmp1[:, :, None] * tmp2[:, None, :]) + (tmp2[:, :, None] * tmp1[:, None, :])
            else:
                raise NotImplementedError, "psi2 cannot be computed for this kernel"
        return psi2

        for p1 in self.parts:
            #compute the effective dL_dpsi1. Extra terms appear becaue of the cross terms in psi2!
            eff_dL_dpsi1 = dL_dpsi1.copy()
            for p2 in self.parts:
                if p2 is p1:
                    continue
                if isinstance(p2, White):
                    continue
                elif isinstance(p2, Bias):
                    eff_dL_dpsi1 += dL_dpsi2.sum(0) * p2.variance * 2.
                else:# np.setdiff1d(p1.active_dims, ar2, assume_unique): # TODO: Careful, not correct for overlapping active_dims
                    eff_dL_dpsi1 += dL_dpsi2.sum(0) * p2.psi1(Z, variational_posterior) * 2.
            p1.update_gradients_expectations(dL_dpsi0, eff_dL_dpsi1, dL_dpsi2, Z, variational_posterior)

    def gradients_Z_expectations(self, dL_psi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
        target = np.zeros(Z.shape)
            #compute the effective dL_dpsi1. extra terms appear becaue of the cross terms in psi2!
                if p2 is p1:
                    continue
                    continue
                else:
            target += p1.gradients_Z_expectations(dL_psi0, eff_dL_dpsi1, dL_dpsi2, Z, variational_posterior)

        target_grads = [np.zeros(v.shape) for v in variational_posterior.parameters]
        for p1 in self.parameters:
            for p2 in self.parameters:
                if p2 is p1:
                    continue
                    continue
                else:
            grads = p1.gradients_qX_expectations(dL_dpsi0, eff_dL_dpsi1, dL_dpsi2, Z, variational_posterior)
            [np.add(target_grads[i],grads[i],target_grads[i]) for i in xrange(len(grads))]
        return target_grads

    def add(self, other):
        if isinstance(other, Add):
            other_params = other.parameters[:]
            for p in other_params:
                other.unlink_parameter(p)
            self.link_parameters(*other_params)
        else:
            self.link_parameter(other)
        self.input_dim, self.active_dims = self.get_input_dim_active_dims(self.parts)

        if summarize:
            return reduce(np.add, [k.input_sensitivity(summarize) for k in self.parts])
        else:
            i_s = np.zeros((len(self.parts), self.input_dim))
            from operator import setitem
            [setitem(i_s, (i, Ellipsis), k.input_sensitivity(summarize)) for i, k in enumerate(self.parts)]
            return i_s

from ..models import GPLVM
from ..mappings import *

class BCGPLVM(GPLVM):
    """
    Back constrained Gaussian Process Latent Variable Model

    :param Y: observed data
    :type Y: np.ndarray
    :param mapping: mapping for back constraint
    :type mapping: GPy.core.Mapping object

    """
    def __init__(self, Y, input_dim, init='PCA', X=None, kernel=None, normalize_Y=False, mapping=None):
        
        if mapping is None:
            mapping = Kernel(X=Y, output_dim=input_dim)
        self.mapping = mapping
        GPLVM.__init__(self, Y, input_dim, init, X, kernel, normalize_Y)
        self.X = self.mapping.f(self.likelihood.Y)

        return self.mapping._get_param_names() + GP._get_param_names(self)

        return np.hstack((self.mapping._get_params(), GP._get_params(self)))

        self.mapping._set_params(x[:self.mapping.num_params])
        GP._set_params(self, x[self.mapping.num_params:])

        dL_df = self.kern.gradients_X(self.dL_dK, self.X)
        dL_dtheta = self.mapping.df_dtheta(dL_df, self.likelihood.Y)
        return np.hstack((dL_dtheta.flatten(), GP._log_likelihood_gradients(self)))

"""
Core module for parameterization.
This module implements all parameterization techniques, split up in modular bits.

HierarchyError:
raised when an error with the hierarchy occurs (circles etc.)

Observable:
Observable Pattern for patameterization

"""

from transformations import Transformation,Logexp, NegativeLogexp, Logistic, __fixed__, FIXED, UNFIXED
import re
from updateable import Updateable

class HierarchyError(Exception):
    """
    Gets thrown when something is wrong with the parameter hierarchy.
    """

def adjust_name_for_printing(name):
    """
    Make sure a name can be printed, alongside used as a variable name.
    """
    if name is not None:
        name2 = name
        name = name.replace(" ", "_").replace(".", "_").replace("-", "_m_")
        name = name.replace("+", "_p_").replace("!", "_I_")
        name = name.replace("**", "_xx_").replace("*", "_x_")
        name = name.replace("/", "_l_").replace("@", '_at_')
        name = name.replace("(", "_of_").replace(")", "")
        if re.match(r'^[a-zA-Z_][a-zA-Z0-9-_]*$', name) is None:
            raise NameError, "name {} converted to {} cannot be further converted to valid python variable name!".format(name2, name)
        return name
    return ''

class Parentable(object):
    """
    Enable an Object to have a parent.

    Additionally this adds the parent_index, which is the index for the parent
    to look for in its parameter list.
    """
    _parent_ = None
    _parent_index_ = None
        super(Parentable, self).__init__()

    def has_parent(self):
        """
        Return whether this parentable object currently has a parent.
        """
        return self._parent_ is not None

    def _parent_changed(self):
        """
        Gets called, when the parent changed, so we can adjust our
        inner attributes according to the new parent.
        """
        raise NotImplementedError, "shouldnt happen, Parentable objects need to be able to change their parent"

    def _disconnect_parent(self, *args, **kw):
        """
        Disconnect this object from its parent
        """
        raise NotImplementedError, "Abstract superclass"

    @property
    def _highest_parent_(self):
        """
        Gets the highest parent by traversing up to the root node of the hierarchy.
        """
        if self._parent_ is None:
            return self
        return self._parent_._highest_parent_

    def _notify_parent_change(self):
        """
        Dont do anything if in leaf node
        """
        pass

class Pickleable(object):
    """
    Make an object pickleable (See python doc 'pickling').

    This class allows for pickling support by Memento pattern.
    _getstate returns a memento of the class, which gets pickled.
    _setstate(<memento>) (re-)sets the state of the class to the memento
    """
    def __init__(self, *a, **kw):
        super(Pickleable, self).__init__()

    # Pickling operations
    def pickle(self, f, protocol=-1):
        """
        :param f: either filename or open file object to write to.
                  if it is an open buffer, you have to make sure to close
                  it properly.
        :param protocol: pickling protocol to use, python-pickle for details.
        """
        import cPickle as pickle
        if isinstance(f, str):
            with open(f, 'wb') as f:
                pickle.dump(self, f, protocol)
        else:
            pickle.dump(self, f, protocol)

    # copy and pickling
    def copy(self, memo=None, which=None):
        """
        Returns a (deep) copy of the current parameter handle.

        All connections to parents of the copy will be cut.

        :param dict memo: memo for deepcopy
        :param Parameterized which: parameterized object which started the copy process [default: self]
        """
        #raise NotImplementedError, "Copy is not yet implemented, TODO: Observable hierarchy"
        if memo is None:
            memo = {}
        import copy
        # the next part makes sure that we do not include parents in any form:
        parents = []
        if which is None:
            which = self
        which.traverse_parents(parents.append) # collect parents
        for p in parents:
            if not memo.has_key(id(p)):memo[id(p)] = None # set all parents to be None, so they will not be copied
        if not memo.has_key(id(self.gradient)):memo[id(self.gradient)] = None # reset the gradient
        if not memo.has_key(id(self._fixes_)):memo[id(self._fixes_)] = None # fixes have to be reset, as this is now highest parent
        copy = copy.deepcopy(self, memo) # and start the copy
        copy._parent_index_ = None
        copy._trigger_params_changed()
        return copy

        s = self.__new__(self.__class__) # fresh instance
        memo[id(self)] = s # be sure to break all cycles --> self is already done
        s.__setstate__(copy.deepcopy(self.__getstate__(), memo)) # standard copy
        return s

        ignore_list = ['_param_array_', # parameters get set from bottom to top
                       '_gradient_array_', # as well as gradients
                       '_optimizer_copy_',
                       'logger',
                       'observers',
                       '_fixes_', # and fixes
                       '_Cacher_wrap__cachers', # never pickle cachers
                       ]
        dc = dict()
        for k,v in self.__dict__.iteritems():
            if k not in ignore_list:
                dc[k] = v
        return dc

        self.__dict__.update(state)
        from lists_and_dicts import ObserverList
        self.observers = ObserverList()
        self._setup_observers()
        self._optimizer_copy_transformed = False

class Gradcheckable(Pickleable, Parentable):
    """
    Adds the functionality for an object to be gradcheckable.
    It is just a thin wrapper of a call to the highest parent for now.
    TODO: Can be done better, by only changing parameters of the current parameter handle,
    such that object hierarchy only has to change for those.
    """
        super(Gradcheckable, self).__init__(*a, **kw)

    def checkgrad(self, verbose=0, step=1e-6, tolerance=1e-3, df_tolerance=1e-12):
        """
        Check the gradient of this parameter with respect to the highest parent's
        objective function.
        This is a three point estimate of the gradient, wiggling at the parameters
        with a stepsize step.
        The check passes if either the ratio or the difference between numerical and
        analytical gradient is smaller then tolerance.

        :param bool verbose: whether each parameter shall be checked individually.
        :param float step: the stepsize for the numerical three point gradient estimate.
        :param float tolerance: the tolerance for the gradient ratio or difference.
        :param float df_tolerance: the tolerance for df_tolerance

        Note:-
           The *dF_ratio* indicates the limit of accuracy of numerical gradients.
           If it is too small, e.g., smaller than 1e-12, the numerical gradients
           are usually not accurate enough for the tests (shown with blue).
        """
        if self.has_parent():
            return self._highest_parent_._checkgrad(self, verbose=verbose, step=step, tolerance=tolerance, df_tolerance=df_tolerance)
        return self._checkgrad(self, verbose=verbose, step=step, tolerance=tolerance, df_tolerance=df_tolerance)

    def _checkgrad(self, param, verbose=0, step=1e-6, tolerance=1e-3):
        """
        Perform the checkgrad on the model.
        TODO: this can be done more efficiently, when doing it inside here
        """
        raise HierarchyError, "This parameter is not in a model with a likelihood, and, therefore, cannot be gradient checked!"

class Nameable(Gradcheckable):
    """
    Make an object nameable inside the hierarchy.
    """
    def __init__(self, name, *a, **kw):
        super(Nameable, self).__init__(*a, **kw)
        self._name = name or self.__class__.__name__

    @property
    def name(self):
        """
        The name of this object
        """
        return self._name
    @name.setter
    def name(self, name):
        """
        Set the name of this object.
        Tell the parent if the name has changed.
        """
        from_name = self.name
        assert isinstance(name, str)
        self._name = name
            self._parent_._name_changed(self, from_name)
    def hierarchy_name(self, adjust_for_printing=True):
        """
        return the name for this object with the parents names attached by dots.

        :param bool adjust_for_printing: whether to call :func:`~adjust_for_printing()`
        on the names, recursively
        """
        if adjust_for_printing: adjust = lambda x: adjust_name_for_printing(x)
        else: adjust = lambda x: x
            return self._parent_.hierarchy_name() + "." + adjust(self.name)
        return adjust(self.name)

class Indexable(Nameable, Updateable):
    """
    Make an object constrainable with Priors and Transformations.
    TODO: Mappings!!
    Adding a constraint to a Parameter means to tell the highest parent that
    the constraint was added and making sure that all parameters covered
    by this object are indeed conforming to the constraint.

    :func:`constrain()` and :func:`unconstrain()` are main methods here
    """
    def __init__(self, name, default_constraint=None, *a, **kw):
        super(Indexable, self).__init__(name=name, *a, **kw)
        self._default_constraint_ = default_constraint
        from index_operations import ParameterIndexOperations
        self.constraints = ParameterIndexOperations()
        self.priors = ParameterIndexOperations()
        if self._default_constraint_ is not None:
            self.constrain(self._default_constraint_)

    def _disconnect_parent(self, constr=None, *args, **kw):
        """
        From Parentable:
        disconnect the parent and set the new constraints to constr
        """
        if constr is None:
            constr = self.constraints.copy()
        self.constraints.clear()
        self.constraints = constr
        self._parent_ = None
        self._parent_index_ = None
        self._connect_fixes()
        self._notify_parent_change()

    # Indexable
    def _offset_for(self, param):
        """
        Return the offset of the param inside this parameterized object.
        This does not need to account for shaped parameters, as it
        basically just sums up the parameter sizes which come before param.
        """
        if param.has_parent():
            p = param._parent_._get_original(param)
            if p in self.parameters:
                return reduce(lambda a,b: a + b.size, self.parameters[:p._parent_index_], 0)
            return self._offset_for(param._parent_) + param._parent_._offset_for(param)
        return 0

    def _raveled_index_for(self, param):
        """
        get the raveled index for a param
        that is an int array, containing the indexes for the flattened
        param inside this parameterized logic.
        """
        from param import ParamConcatenation
        if isinstance(param, ParamConcatenation):
            return np.hstack((self._raveled_index_for(p) for p in param.params))
        return param._raveled_index() + self._offset_for(param)

        """
        Flattened array of ints, specifying the index of this object.
        This has to account for shaped parameters!
        """
        return np.r_[:self.size]

    # Fixing Parameters:
    def constrain_fixed(self, value=None, warning=True, trigger_parent=True):
        """
        Constrain this parameter to be fixed to the current value it carries.

        :param warning: print a warning for overwriting constraints.
        """
        if value is not None:
            self[:] = value

        index = self.unconstrain()
        index = self._add_to_index_operations(self.constraints, index, __fixed__, warning)
        self._highest_parent_._set_fixed(self, index)
        self.notify_observers(self, None if trigger_parent else -np.inf)
    fix = constrain_fixed

    def unconstrain_fixed(self):
        """
        This parameter will no longer be fixed.
        """
        unconstrained = self.unconstrain(__fixed__)
        self._highest_parent_._set_unfixed(self, unconstrained)
        return unconstrained
    unfix = unconstrain_fixed

    def _ensure_fixes(self):
        # Ensure that the fixes array is set:
        # Parameterized: ones(self.size)
        # Param: ones(self._realsize_
        if not self._has_fixes(): self._fixes_ = np.ones(self.size, dtype=bool)

    def _set_fixed(self, param, index):
        self._ensure_fixes()
        offset = self._offset_for(param)
        self._fixes_[index+offset] = FIXED
        if np.all(self._fixes_): self._fixes_ = None  # ==UNFIXED

    def _set_unfixed(self, param, index):
        self._fixes_[index+offset] = UNFIXED

    def _connect_fixes(self):
        fixed_indices = self.constraints[__fixed__]
        if fixed_indices.size > 0:
            self._ensure_fixes()
            self._fixes_[fixed_indices] = FIXED
        else:
            self._fixes_ = None
            del self.constraints[__fixed__]

    # Convenience for fixed
    def _has_fixes(self):
        return hasattr(self, "_fixes_") and self._fixes_ is not None and self._fixes_.size == self.size

    @property
    def is_fixed(self):
            if not p.is_fixed: return False

    def _get_original(self, param):
        # if advanced indexing is activated it happens that the array is a copy
        # you can retrieve the original param through this method, by passing
        # the copy here
        return self.parameters[param._parent_index_]

    # Prior Operations
    def set_prior(self, prior, warning=True):
        """
        Set the prior for this object to prior.
        :param :class:`~GPy.priors.Prior` prior: a prior to set for this parameter
        :param bool warning: whether to warn if another prior was set for this parameter
        """
        repriorized = self.unset_priors()
        self._add_to_index_operations(self.priors, repriorized, prior, warning)

        from domains import _REAL, _POSITIVE, _NEGATIVE
        if prior.domain is _POSITIVE:
            self.constrain_positive(warning)
        elif prior.domain is _NEGATIVE:
            self.constrain_negative(warning)
        elif prior.domain is _REAL:
            rav_i = self._raveled_index()
            assert all(all(False if c is __fixed__ else c.domain is _REAL for c in con) for con in self.constraints.properties_for(rav_i)), 'Domain of prior and constraint have to match, please unconstrain if you REALLY wish to use this prior'

    def unset_priors(self, *priors):
        """
        Un-set all priors given (in *priors) from this parameter handle.
        """
        return self._remove_from_index_operations(self.priors, priors)

    def log_prior(self):
        """evaluate the prior"""
        if self.priors.size > 0:
            x = self.param_array
            return reduce(lambda a, b: a + b, (p.lnpdf(x[ind]).sum() for p, ind in self.priors.iteritems()), 0)
        return 0.

    def _log_prior_gradients(self):
        """evaluate the gradients of the priors"""
            ret = np.zeros(x.size)
            [np.put(ret, ind, p.lnpdf_grad(x[ind])) for p, ind in self.priors.iteritems()]
            return ret
        return 0.

    # Tie parameters together

    def _has_ties(self):
        if self._highest_parent_.tie.tied_param is None:
            return self._highest_parent_.tie.label_buf[self._highest_parent_._raveled_index_for(self)].sum()>0

    def tie_together(self):
        self._highest_parent_.tie.add_tied_parameter(self)
        self._highest_parent_._set_fixed(self,self._raveled_index())
        self._trigger_params_changed()

    # Constrain operations -> done

    def constrain(self, transform, warning=True, trigger_parent=True):
        """
        :param transform: the :py:class:`GPy.core.transformations.Transformation`
                          to constrain the this parameter to.
        :param warning: print a warning if re-constraining parameters.

        Constrain the parameter to the given
        :py:class:`GPy.core.transformations.Transformation`.
        """
        if isinstance(transform, Transformation):
            self.param_array[...] = transform.initialize(self.param_array)
        reconstrained = self.unconstrain()
        added = self._add_to_index_operations(self.constraints, reconstrained, transform, warning)
        return added

    def unconstrain(self, *transforms):
        """
        :param transforms: The transformations to unconstrain from.

        remove all :py:class:`GPy.core.transformations.Transformation`
        transformats of this parameter object.
        """
        return self._remove_from_index_operations(self.constraints, transforms)

    def constrain_positive(self, warning=True, trigger_parent=True):
        """

        Constrain this parameter to the default positive constraint.
        """
        self.constrain(Logexp(), warning=warning, trigger_parent=trigger_parent)

    def constrain_negative(self, warning=True, trigger_parent=True):
        """

        Constrain this parameter to the default negative constraint.
        """
        self.constrain(NegativeLogexp(), warning=warning, trigger_parent=trigger_parent)

    def constrain_bounded(self, lower, upper, warning=True, trigger_parent=True):
        """
        :param lower, upper: the limits to bound this parameter to

        Constrain this parameter to lie within the given range.
        """
        self.constrain(Logistic(lower, upper), warning=warning, trigger_parent=trigger_parent)

    def unconstrain_positive(self):
        """
        Remove positive constraint of this parameter.
        """
        self.unconstrain(Logexp())

    def unconstrain_negative(self):
        """
        Remove negative constraint of this parameter.
        """
        self.unconstrain(NegativeLogexp())

    def unconstrain_bounded(self, lower, upper):
        """
        :param lower, upper: the limits to unbound this parameter from

        Remove (lower, upper) bounded constrain from this parameter/
        """
        self.unconstrain(Logistic(lower, upper))

    def _parent_changed(self, parent):
        """
        Called when the parent changed

        update the constraints and priors view, so that
        constraining is automized for the parent.
        """
        from index_operations import ParameterIndexOperationsView
        #if getattr(self, "_in_init_"):
            #self.constraints.update(param.constraints, start)
            #self.priors.update(param.priors, start)
        offset = parent._offset_for(self)
        self.constraints = ParameterIndexOperationsView(parent.constraints, offset, self.size)
        self.priors = ParameterIndexOperationsView(parent.priors, offset, self.size)
        self._fixes_ = None
            p._parent_changed(parent)

    def _add_to_index_operations(self, which, reconstrained, what, warning):
        """
        Helper preventing copy code.
        This adds the given what (transformation, prior etc) to parameter index operations which.
        reconstrained are reconstrained indices.
        warn when reconstraining parameters if warning is True.
        TODO: find out which parameters have changed specifically
        """
        if warning and reconstrained.size > 0:
            # TODO: figure out which parameters have changed and only print those
            print "WARNING: reconstraining parameters {}".format(self.hierarchy_name() or self.name)
        index = self._raveled_index()
        which.add(what, index)

    def _remove_from_index_operations(self, which, transforms):
        """
        Remove given what (transform prior etc) from which param index ops.
        """
        if len(transforms) == 0:
            transforms = which.properties()
        removed = np.empty((0,), dtype=int)
        for t in transforms:
            unconstrained = which.remove(t, self._raveled_index())
            removed = np.union1d(removed, unconstrained)
            if t is __fixed__:
                self._highest_parent_._set_unfixed(self, unconstrained)

class OptimizationHandlable(Indexable):
    """
    This enables optimization handles on an Object as done in GPy 0.4.

    `..._optimizer_copy_transformed`: make sure the transformations and constraints etc are handled
    """
        super(OptimizationHandlable, self).__init__(name, default_constraint=default_constraint, *a, **kw)
        self._optimizer_copy_ = None

    # Optimizer copy
    @property
    def optimizer_array(self):
        """
        Array for the optimizer to work on.
        This array always lives in the space for the optimizer.
        Thus, it is untransformed, going from Transformations.

        Setting this array, will make sure the transformed parameters for this model
        will be set accordingly. It has to be set with an array, retrieved from
        this method, as e.g. fixing will resize the array.

        The optimizer should only interfere with this array, such that transformations
        are secured.
        """
        if self.__dict__.get('_optimizer_copy_', None) is None or self.size != self._optimizer_copy_.size:
            self._optimizer_copy_ = np.empty(self.size)

        if not self._optimizer_copy_transformed:
            self._optimizer_copy_.flat = self.param_array.flat
            [np.put(self._optimizer_copy_, ind, c.finv(self.param_array[ind])) for c, ind in self.constraints.iteritems() if c != __fixed__]
            if self.has_parent() and (self.constraints[__fixed__].size != 0 or self._has_ties()):
                fixes = np.ones(self.size).astype(bool)
                fixes[self.constraints[__fixed__]] = FIXED
                return self._optimizer_copy_[np.logical_and(fixes, self._highest_parent_.tie.getTieFlag(self))]
            elif self._has_fixes():
                return self._optimizer_copy_[self._fixes_]

            self._optimizer_copy_transformed = True

        return self._optimizer_copy_

    @optimizer_array.setter
        """
        Make sure the optimizer copy does not get touched, thus, we only want to
        set the values *inside* not the array itself.

        Also we want to update param_array in here.
        """
        f = None
        if self.has_parent() and self.constraints[__fixed__].size != 0:
            f = np.ones(self.size).astype(bool)
            f[self.constraints[__fixed__]] = FIXED
        elif self._has_fixes():
            f = self._fixes_
        if f is None:
            self.param_array.flat = p
            [np.put(self.param_array, ind, c.f(self.param_array.flat[ind]))
             for c, ind in self.constraints.iteritems() if c != __fixed__]
        else:
            self.param_array.flat[f] = p
            [np.put(self.param_array, ind[f[ind]], c.f(self.param_array.flat[ind[f[ind]]]))
        #self._highest_parent_.tie.propagate_val()

        self.trigger_update()

    def _get_params_transformed(self):
        raise DeprecationWarning, "_get|set_params{_optimizer_copy_transformed} is deprecated, use self.optimizer array insetad!"
#
    def _set_params_transformed(self, p):

    def _trigger_params_changed(self, trigger_parent=True):
        """
        First tell all children to update,
        then update yourself.

        If trigger_parent is True, we will tell the parent, otherwise not.
        """
        [p._trigger_params_changed(trigger_parent=False) for p in self.parameters if not p.is_fixed]
        self.notify_observers(None, None if trigger_parent else -np.inf)

    def _size_transformed(self):
        """
        As fixes are not passed to the optimiser, the size of the model for the optimiser
        is the size of all parameters minus the size of the fixes.
        """
        return self.size - self.constraints[__fixed__].size

    def _transform_gradients(self, g):
        """
        Transform the gradients by multiplying the gradient factor for each
        constraint to it.
        """
        self._highest_parent_.tie.collate_gradient()
        [np.put(g, i, c.gradfactor(self.param_array[i], g[i])) for c, i in self.constraints.iteritems() if c != __fixed__]
        if self._has_fixes(): return g[self._fixes_]
        return g

    @property
    def num_params(self):
        """
        Return the number of parameters of this parameter_handle.
        Param objects will always return 0.
        """
        raise NotImplemented, "Abstract, please implement in respective classes"

    def parameter_names(self, add_self=False, adjust_for_printing=False, recursive=True):
        """
        Get the names of all parameters of this model.

        :param bool add_self: whether to add the own name in front of names
        :param bool adjust_for_printing: whether to call `adjust_name_for_printing` on names
        :param bool recursive: whether to traverse through hierarchy and append leaf node names
        """
        if recursive: names = [xi for x in self.parameters for xi in x.parameter_names(add_self=True, adjust_for_printing=adjust_for_printing)]
        else: names = [adjust(x.name) for x in self.parameters]
        if add_self: names = map(lambda x: adjust(self.name) + "." + x, names)

        n = np.array([p.hierarchy_name() + '[' + str(i) + ']' for p in self.flattened_parameters for i in p._indices()])
        return n

    def _get_param_names_transformed(self):
        n = self._get_param_names()
        if self._has_fixes():
            return n[self._fixes_]
        return n

    # Randomizeable
    def randomize(self, rand_gen=None, *args, **kwargs):
        """
        Randomize the model.
        Make this draw from the prior if one exists, else draw from given random generator

        :param rand_gen: np random number generator which takes args and kwargs
        :param flaot loc: loc parameter for random number generator
        :param float scale: scale parameter for random number generator
        :param args, kwargs: will be passed through to random number generator
        """
        if rand_gen is None:
            rand_gen = np.random.normal
        # first take care of all parameters (from N(0,1))
        x = rand_gen(size=self._size_transformed(), *args, **kwargs)
        updates = self.update_model()
        self.update_model(False) # Switch off the updates
        self.optimizer_array = x  # makes sure all of the tied parameters get the same init (since there's only one prior object...)
        # now draw from prior where possible
        x = self.param_array.copy()
        [np.put(x, ind, p.rvs(ind.size)) for p, ind in self.priors.iteritems() if not p is None]
        unfixlist = np.ones((self.size,),dtype=np.bool)
        unfixlist[self.constraints[__fixed__]] = False
        self.param_array.flat[unfixlist] = x.view(np.ndarray).ravel()[unfixlist]
        self.update_model(updates)

    # For shared memory arrays. This does nothing in Param, but sets the memory
    # for all parameterized objects
    @property
    def gradient_full(self):
        """
        Note to users:
        This does not return the gradient in the right shape! Use self.gradient
        for the right gradient array.

        To work on the gradient array, use this as the gradient handle.
        This method exists for in memory use of parameters.
        When trying to access the true gradient array, use this.
        """
        self.gradient # <<< ensure _gradient_array_
        return self._gradient_array_

    def _propagate_param_grad(self, parray, garray):
        """
        For propagating the param_array and gradient_array.
        This ensures the in memory view of each subsequent array.

        1.) connect param_array of children to self.param_array
        2.) tell all children to propagate further
        """
        if self.param_array.size != self.size:
            self._param_array_ = np.empty(self.size, dtype=np.float64)
        if self.gradient.size != self.size:
            self._gradient_array_ = np.empty(self.size, dtype=np.float64)

        pi_old_size = 0
        for pi in self.parameters:
            pislice = slice(pi_old_size, pi_old_size + pi.size)

            self.param_array[pislice] = pi.param_array.flat  # , requirements=['C', 'W']).flat
            self.gradient_full[pislice] = pi.gradient_full.flat  # , requirements=['C', 'W']).flat

            pi.param_array.data = parray[pislice].data
            pi.gradient_full.data = garray[pislice].data

            pi._propagate_param_grad(parray[pislice], garray[pislice])
            pi_old_size += pi.size

    def _connect_parameters(self):
        pass

_name_digit = re.compile("(?P<name>.*)_(?P<digit>\d+)$")
class Parameterizable(OptimizationHandlable):
    """
    A parameterisable class.

    This class provides the parameters list (ArrayList) and standard parameter handling,
    such as {add|remove}_parameter(), traverse hierarchy and param_array, gradient_array
    and the empty parameters_changed().

    This class is abstract and should not be instantiated.
    Use GPy.core.Parameterized() as node (or leaf) in the parameterized hierarchy.
    Use GPy.core.Param() for a leaf in the parameterized hierarchy.
    """
        super(Parameterizable, self).__init__(*args, **kwargs)
        from GPy.core.parameterization.lists_and_dicts import ArrayList
        self.parameters = ArrayList()
        self._param_array_ = None
        self._added_names_ = set()
        self.__visited = False # for traversing in reverse order we need to know if we were here already

    @property
    def param_array(self):
        """
        Array representing the parameters of this class.
        There is only one copy of all parameters in memory, two during optimization.

        !WARNING!: setting the parameter array MUST always be done in memory:
        m.param_array[:] = m_copy.param_array
        """
        if (self.__dict__.get('_param_array_', None) is None) or (self._param_array_.size != self.size):
        return self._param_array_

    @property
    def unfixed_param_array(self):
        """

        """
        if self.__dict__.get('_param_array_', None) is None:

        if self.constraints[__fixed__].size !=0:
            fixes = np.ones(self.size).astype(bool)
            fixes[self.constraints[__fixed__]] = FIXED
            return self._param_array_[fixes]
        else:
            return self._param_array_

    @param_array.setter
    def param_array(self, arr):
        self._param_array_ = arr

    def traverse(self, visit, *args, **kwargs):
        """
        Traverse the hierarchy performing visit(self, *args, **kwargs)
        at every node passed by downwards. This function includes self!

        See "visitor pattern" in literature. This is implemented in pre-order fashion.

        Example:
        Collect all children:

        children = []
        self.traverse(children.append)
        print children
        """
        if not self.__visited:
            visit(self, *args, **kwargs)
            self.__visited = True
            for c in self.parameters:
                c.traverse(visit, *args, **kwargs)
            self.__visited = False

    def traverse_parents(self, visit, *args, **kwargs):
        """
        Traverse the hierarchy upwards, visiting all parents and their children except self.

        Example:

        self.traverse_parents(parents.append)
        print parents
        """
            self._parent_._traverse_parents(visit, *args, **kwargs)

    def _traverse_parents(self, visit, *args, **kwargs):
            if self.has_parent():
                self._parent_._traverse_parents(visit, *args, **kwargs)
                self._parent_.traverse(visit, *args, **kwargs)

    #=========================================================================
    # Gradient handling
    @property
    def gradient(self):
        if (self.__dict__.get('_gradient_array_', None) is None) or self._gradient_array_.size != self.size:

    @gradient.setter
    def gradient(self, val):
        self._gradient_array_[:] = val

    @property
        return len(self.parameters)

    def _add_parameter_name(self, param, ignore_added_names=False):
        pname = adjust_name_for_printing(param.name)
        if ignore_added_names:
            self.__dict__[pname] = param
            return

        def warn_and_retry(param, match=None):
            #===================================================================
            # print """
            # WARNING: added a parameter with formatted name {},
            # which is already assigned to {}.
            # Trying to change the parameter name to
            #
            # {}.{}
            # """.format(pname, self.hierarchy_name(), self.hierarchy_name(), param.name + "_")
            if match is None:
                param.name += "_1"
            else:
                param.name = match.group('name') + "_" + str(int(match.group('digit'))+1)
            self._add_parameter_name(param, ignore_added_names)
        # and makes sure to not delete programmatically added parameters
        for other in self.parameters[::-1]:
            if other is not param and other.name.startswith(param.name):
                warn_and_retry(param, _name_digit.match(other.name))
                return
        if pname not in dir(self):
            self._added_names_.add(pname)
        elif pname in self.__dict__:
            if pname in self._added_names_:
                other = self.__dict__[pname]
                if not (param is other):
                    del self.__dict__[pname]
                    self._added_names_.remove(pname)
                    warn_and_retry(other)
                    warn_and_retry(param, _name_digit.match(other.name))
            return

    def _remove_parameter_name(self, param=None, pname=None):
        assert param is None or pname is None, "can only delete either param by name, or the name of a param"
        pname = adjust_name_for_printing(pname) or adjust_name_for_printing(param.name)
        if pname in self._added_names_:
            del self.__dict__[pname]
            self._added_names_.remove(pname)
        self._connect_parameters()

    def _name_changed(self, param, old_name):
        self._remove_parameter_name(None, old_name)
        self._add_parameter_name(param)

        super(Parameterizable, self).__setstate__(state)

    # notification system
    def _parameters_changed_notification(self, me, which=None):
        """
        In parameterizable we just need to make sure, that the next call to optimizer_array
        will update the optimizer_array to the latest parameters
        """
        self._optimizer_copy_transformed = False # tells the optimizer array to update on next request
    def _pass_through_notify_observers(self, me, which=None):
        self.notify_observers(which=which)
    def _setup_observers(self):
        """
        Setup the default observers

        1: parameters_changed_notify
        2: pass through to parent, if present
        """
        self.add_observer(self, self._parameters_changed_notification, -100)
            self.add_observer(self._parent_, self._parent_._pass_through_notify_observers, -np.inf)
    # From being parentable, we have to define the parent_change notification
        """
        Notify all parameters that the parent has changed
        """
            p._parent_changed(self)

        """
        This method gets called when parameters have changed.
        Another way of listening to param changes is to
        add self as a listener to the param, such that
        updates get passed through. See :py:function:``GPy.core.param.Observable.add_observer``
        """
        pass

    def save(self, filename, ftype='HDF5'):
        """
        Save all the model parameters into a file (HDF5 by default).
        """
        from . import Param
        def gather_params(self, plist):
            if isinstance(self,Param):
                plist.append(self)
        plist = []
        self.traverse(gather_params, plist)
        names = self.parameter_names(adjust_for_printing=True)
        if ftype=='HDF5':
            try:
                import h5py
                f = h5py.File(filename,'w')
                for p,n in zip(plist,names):
                    n = n.replace('.','_')
                    p = param_to_array(p)
                    d = f.create_dataset(n,p.shape,dtype=p.dtype)
                    d[:] = p
                f.close()
            except:
                raise 'Fails to write the parameters into a HDF5 file!'

class ODE_UYC(Kern):
    def __init__(self, input_dim, variance_U=3., variance_Y=1., lengthscale_U=1., lengthscale_Y=1., ubias =1. ,active_dims=None, name='ode_uyc'):
        super(ODE_UYC, self).__init__(input_dim, active_dims, name)

        self.variance_U = Param('variance_U', variance_U, Logexp())
        self.lengthscale_U = Param('lengthscale_U', lengthscale_U, Logexp())
        self.ubias = Param('ubias', ubias, Logexp())

        self.add_parameters(self.variance_Y, self.variance_U, self.lengthscale_Y, self.lengthscale_U, self.ubias)

        else:

        #stop
        #Vy=ly/2
        #stop

        kuu = lambda dist:Vu * (1 + lu* np.abs(dist)) * np.exp(-lu * np.abs(dist)) +self.ubias

                        else:
        return K

                if i==0:
                    Kdiag[s1]+= self.variance_U + self.ubias
                elif i==1:
                else:

        else:

        rd=rdist.shape[0]
        dktheta1 = np.zeros([rd,rd])
        dktheta2 = np.zeros([rd,rd])
        dkUdvar = np.zeros([rd,rd])
        dkYdvar = np.zeros([rd,rd])

        dkdubias = np.zeros([rd,rd])

            )

            )

                            dkdubias[ss1,ss2] = 1
                            dkdubias[ss1,ss2] = 0
                        else:
        #stop

        self.ubias.gradient = np.sum(dkdubias * dL_dK) 

def sigmoid(x):
    return 1./(1.+np.exp(-x))

def softmax(x):
    ex = np.exp(x-x.max(1)[:,None])
    return ex/ex.sum(1)[:,np.newaxis]

def single_softmax(x):
    ex = np.exp(x)
    return ex/ex.sum()

# Copyright (c) 2013, Ricardo Andrade

from ..likelihoods import likelihood
from ..inference.latent_function_inference import expectation_propagation_dtc

class SparseGPClassification(SparseGP):
    """
    sparse Gaussian Process model for classification

    This is a thin wrapper around the sparse_GP class, with a set of sensible defaults

    :param likelihood: a GPy likelihood, defaults to Binomial with probit link_function
    :param normalize_X:  whether to normalize the input data before computing (predictions will be in original scales)
    :type normalize_X: False|True
    :param normalize_Y:  whether to normalize the input data before computing (predictions will be in original scales)
    :type normalize_Y: False|True

    """

    #def __init__(self, X, Y=None, likelihood=None, kernel=None, normalize_X=False, normalize_Y=False, Z=None, num_inducing=10):
    def __init__(self, X, Y=None, likelihood=None, kernel=None, Z=None, num_inducing=10, Y_metadata=None):

            i = np.random.permutation(X.shape[0])[:num_inducing]
        else:
            assert Z.shape[1] == X.shape[1]

        SparseGP.__init__(self, X, Y, Z, kernel, likelihood, inference_method=expectation_propagation_dtc.EPDTC(), name='SparseGPClassification',Y_metadata=Y_metadata)
    #def __init__(self, X, Y, Z, kernel, likelihood, inference_method=None, name='sparse gp', Y_metadata=None):

class Static(Kern):
    def __init__(self, input_dim, variance, active_dims, name):
        super(Static, self).__init__(input_dim, active_dims, name)

        ret = np.empty((X.shape[0],), dtype=np.float64)
        return ret

        return np.zeros(Z.shape)

        return np.zeros(variational_posterior.shape), np.zeros(variational_posterior.shape)

        return self.Kdiag(variational_posterior.mean)

        return self.K(variational_posterior.mean, Z)

        K = self.K(variational_posterior.mean, Z)
        return np.einsum('ij,ik->jk',K,K) #K[:,:,None]*K[:,None,:] # NB. more efficient implementations on inherriting classes

            return super(Static, self).input_sensitivity(summarize=summarize)
        else:
            return np.ones(self.input_dim) * self.variance

class White(Static):
    def __init__(self, input_dim, variance=1., active_dims=None, name='white'):
        super(White, self).__init__(input_dim, variance, active_dims, name)

            return np.eye(X.shape[0])*self.variance
        else:
            return np.zeros((X.shape[0], X2.shape[0]))

        return np.zeros((Z.shape[0], Z.shape[0]), dtype=np.float64)

        self.variance.gradient = np.trace(dL_dK)

        self.variance.gradient = dL_dKdiag.sum()

        self.variance.gradient = dL_dpsi0.sum()

class Bias(Static):
    def __init__(self, input_dim, variance=1., active_dims=None, name='bias'):
        super(Bias, self).__init__(input_dim, variance, active_dims, name)

        shape = (X.shape[0], X.shape[0] if X2 is None else X2.shape[0])
        ret = np.empty(shape, dtype=np.float64)
        return ret

        self.variance.gradient = dL_dK.sum()

        ret = np.empty((Z.shape[0], Z.shape[0]), dtype=np.float64)
        ret[:] = self.variance*self.variance*variational_posterior.shape[0]
        return ret

        self.variance.gradient = dL_dpsi0.sum() + dL_dpsi1.sum() + 2.*self.variance*dL_dpsi2.sum()*variational_posterior.shape[0]

class Fixed(Static):
    def __init__(self, input_dim, covariance_matrix, variance=1., active_dims=None, name='fixed'):
        """
        :param input_dim: the number of input dimensions
        :param variance: the variance of the kernel
        """
        super(Fixed, self).__init__(input_dim, variance, active_dims, name)
        self.fixed_K = covariance_matrix
    def K(self, X, X2):
        return self.variance * self.fixed_K

        return self.variance * self.fixed_K.diag()

        self.variance.gradient = np.einsum('ij,ij', dL_dK, self.fixed_K)

        self.variance.gradient = np.einsum('i,i', dL_dKdiag, self.fixed_K)

# Copyright (c) 2014, Max Zwiessele, James Hensman

import numpy; np = numpy
from re import compile, _pattern_type
from param import ParamConcatenation
from parameter_core import HierarchyError, Parameterizable, adjust_name_for_printing

from GPy.core.parameterization.index_operations import ParameterIndexOperationsView
logger = logging.getLogger("parameters changed meta")

class ParametersChangedMeta(type):
        self._in_init_ = True
        #import ipdb;ipdb.set_trace()
        self = super(ParametersChangedMeta, self).__call__(*args, **kw)
        logger.debug("finished init")
        self._in_init_ = False
        logger.debug("connecting parameters")
        self._highest_parent_._connect_parameters()
        #self._highest_parent_._notify_parent_change()
        self._highest_parent_._connect_fixes()
        logger.debug("calling parameters changed")

class Parameterized(Parameterizable):
    """
    Parameterized class

    Say m is a handle to a parameterized class.

    Printing parameters:

        - print m:           prints a nice summary over all parameters
        - print m.name:      prints details for param with name 'name'
        - print m[regexp]: prints details for all the parameters
                             which match (!) regexp
        - print m['']:       prints details for all parameters

        Fields:

            Name:       The name of the param, can be renamed!
            Value:      Shape or value, if one-valued
            Constrain:  constraint of the param, curly "{c}" brackets indicate
                        some parameters are constrained by c. See detailed print
                        to get exact constraints.
            Tied_to:    which paramter it is tied to.

    Getting and setting parameters:

        Set all values in param to one:

            m.name.to.param = 1

    Handling of constraining, fixing and tieing parameters:

        You can constrain parameters by calling the constrain on the param itself, e.g:

            - m.name[:,1].constrain_positive()
            - m.name[0].tie_to(m.name[1])

        Fixing parameters will fix them to the value they are right now. If you change
        the parameters value, the param will be fixed to the new value!

        If you want to operate on all parameters use m[''] to wildcard select all paramters
        and concatenate them. Printing m[''] will result in printing of all parameters in detail.
    """
    # Metaclass for parameters changed after init.
    # This makes sure, that parameters changed will always be called after __init__
    # **Never** call parameters_changed() yourself
    __metaclass__ = ParametersChangedMeta
    def __init__(self, name=None, parameters=[], *a, **kw):
        super(Parameterized, self).__init__(name=name, *a, **kw)
        self.size = sum(p.size for p in self.parameters)
        if not self._has_fixes():
        self._param_slices_ = []
        #self._connect_parameters()
        self.link_parameters(*parameters)

    def build_pydot(self, G=None):
        import pydot  # @UnresolvedImport
        iamroot = False
        if G is None:
            G = pydot.Dot(graph_type='digraph', bgcolor=None)
            iamroot=True
        node = pydot.Node(id(self), shape='box', label=self.name)#, color='white')
        G.add_node(node)
        for child in self.parameters:
            child_node = child.build_pydot(G)
            G.add_edge(pydot.Edge(node, child_node))#, color='white'))

        for _, o, _ in self.observers:
            label = o.name if hasattr(o, 'name') else str(o)
            observed_node = pydot.Node(id(o), label=label)
            G.add_node(observed_node)
            edge = pydot.Edge(str(id(self)), str(id(o)), color='darkorange2', arrowhead='vee')
            G.add_edge(edge)

        if iamroot:
            return G
        return node

    # Add remove parameters:
    def link_parameter(self, param, index=None, _ignore_added_names=False):
        """
        :param parameters:  the parameters to add
        :type parameters:   list of or one :py:class:`GPy.core.param.Param`
        :param [index]:     index of where to put parameters

        :param bool _ignore_added_names: whether the name of the parameter overrides a possibly existing field

        Add all parameters to this param class, you can insert parameters
        at any given index using the :func:`list.insert` syntax
        """
        if param in self.parameters and index is not None:
            self.unlink_parameter(param)
            self.link_parameter(param, index)
        # elif param.has_parent():
        #    raise HierarchyError, "parameter {} already in another model ({}), create new object (or copy) for adding".format(param._short(), param._highest_parent_._short())
        elif param not in self.parameters:
            if param.has_parent():
                def visit(parent, self):
                    if parent is self:
                        raise HierarchyError, "You cannot add a parameter twice into the hierarchy"
                param.traverse_parents(visit, self)
                param._parent_.unlink_parameter(param)
            # make sure the size is set
            if index is None:
                start = sum(p.size for p in self.parameters)
                self.constraints.shift_right(start, param.size)
                self.priors.shift_right(start, param.size)
                self.constraints.update(param.constraints, self.size)
                self.priors.update(param.priors, self.size)
                param._parent_ = self
                param._parent_index_ = len(self.parameters)
                self.parameters.append(param)
            else:
                start = sum(p.size for p in self.parameters[:index])
                self.constraints.update(param.constraints, start)
                self.priors.update(param.priors, start)
                param._parent_index_ = index if index>=0 else len(self.parameters[:index])
                for p in self.parameters[index:]:
                    p._parent_index_ += 1
                self.parameters.insert(index, param)

            param.add_observer(self, self._pass_through_notify_observers, -np.inf)

            parent = self
            while parent is not None:
                parent.size += param.size
                parent = parent._parent_
            self._notify_parent_change()

            if not self._in_init_:
                #self._connect_parameters()
                #self._notify_parent_change()

                self._highest_parent_._connect_parameters(ignore_added_names=_ignore_added_names)
                self._highest_parent_._notify_parent_change()
                self._highest_parent_._connect_fixes()

        else:
            raise HierarchyError, """Parameter exists already, try making a copy"""

    def link_parameters(self, *parameters):
        """
        convenience method for adding several
        parameters without gradient specification
        """
        [self.link_parameter(p) for p in parameters]

    def unlink_parameter(self, param):
        """
        :param param: param object to remove from being a parameter of this parameterized object.
        """
        if not param in self.parameters:
            try:
                raise RuntimeError, "{} does not belong to this object {}, remove parameters directly from their respective parents".format(param._short(), self.name)
                raise RuntimeError, "{} does not seem to be a parameter, remove parameters directly from their respective parents".format(str(param))

        start = sum([p.size for p in self.parameters[:param._parent_index_]])
        self._remove_parameter_name(param)
        self.size -= param.size
        del self.parameters[param._parent_index_]

        param._disconnect_parent()
        param.remove_observer(self, self._pass_through_notify_observers)
        self.constraints.shift_left(start, param.size)

        parent = self._parent_
        while parent is not None:
            parent.size -= param.size
            parent = parent._parent_

        self._highest_parent_._notify_parent_change()

    def add_parameter(self, *args, **kwargs):
        raise DeprecationWarning, "add_parameter was renamed to link_parameter to avoid confusion of setting variables"
    def remove_parameter(self, *args, **kwargs):
        raise DeprecationWarning, "remove_parameter was renamed to link_parameter to avoid confusion of setting variables"

    def _connect_parameters(self, ignore_added_names=False):
        # connect parameterlist to this parameterized object
        # This just sets up the right connection for the params objects
        # to be used as parameters
        # it also sets the constraints for each parameter to the constraints
        # of their respective parents
        if not hasattr(self, "parameters") or len(self.parameters) < 1:
            # no parameters for this class
            return

        old_size = 0
        for i, p in enumerate(self.parameters):
            if not p.param_array.flags['C_CONTIGUOUS']:
                raise ValueError, "This should not happen! Please write an email to the developers with the code, which reproduces this error. All parameter arrays must be C_CONTIGUOUS"

            p._parent_ = self
            p._parent_index_ = i

            pslice = slice(old_size, old_size + p.size)

            # first connect all children
            p._propagate_param_grad(self.param_array[pslice], self.gradient_full[pslice])

            # then connect children to self
            self.param_array[pslice] = p.param_array.flat  # , requirements=['C', 'W']).ravel(order='C')
            self.gradient_full[pslice] = p.gradient_full.flat  # , requirements=['C', 'W']).ravel(order='C')

            p.param_array.data = self.param_array[pslice].data
            p.gradient_full.data = self.gradient_full[pslice].data

            self._param_slices_.append(pslice)

            self._add_parameter_name(p, ignore_added_names=ignore_added_names)
            old_size += p.size

    # Get/set parameters:
    def grep_param_names(self, regexp):
        """
        create a list of parameters, matching regular expression regexp
        """
        if not isinstance(regexp, _pattern_type): regexp = compile(regexp)
        found_params = []
        for n, p in itertools.izip(self.parameter_names(False, False, True), self.flattened_parameters):
            if regexp.match(n) is not None:
                found_params.append(p)
        return found_params

    def __getitem__(self, name, paramlist=None):
        if isinstance(name, (int, slice, tuple, np.ndarray)):
            return self.param_array[name]
        else:
            if paramlist is None:
                paramlist = self.grep_param_names(name)
            if len(paramlist) < 1: raise AttributeError, name
            if len(paramlist) == 1:
                if isinstance(paramlist[-1], Parameterized):
                    paramlist = paramlist[-1].flattened_parameters
                    if len(paramlist) != 1:
                        return ParamConcatenation(paramlist)
                return paramlist[-1]
            return ParamConcatenation(paramlist)

    def __setitem__(self, name, value, paramlist=None):
        if value is None:
            return # nothing to do here
        if isinstance(name, (slice, tuple, np.ndarray)):
            try:
                self.param_array[name] = value
            except:
                raise ValueError, "Setting by slice or index only allowed with array-like"
            self._trigger_params_changed()
        else:
            try: param = self.__getitem__(name, paramlist)
            except: raise
            param[:] = value

    def __setattr__(self, name, val):
        # override the default behaviour, if setting a param, so broadcasting can by used
        if hasattr(self, "parameters"):
            try:
                pnames = self.parameter_names(False, adjust_for_printing=True, recursive=False)
                if name in pnames:
                    param = self.parameters[pnames.index(name)]
                    param[:] = val; return
                pass
        object.__setattr__(self, name, val);

    # Pickling
        super(Parameterized, self).__setstate__(state)
        try:
            self._connect_parameters()
            self._connect_fixes()
            self.parameters_changed()
            print "WARNING: caught exception {!s}, trying to continue".format(e)

    def copy(self, memo=None):
            memo = {}
        memo[id(self.optimizer_array)] = None # and param_array
        memo[id(self.param_array)] = None # and param_array
        copy = super(Parameterized, self).copy(memo)
        copy._connect_parameters()
        copy._connect_fixes()
        copy._notify_parent_change()

    # Printing:
    def _short(self):
        return self.hierarchy_name()
    @property
    def flattened_parameters(self):
        return [xi for x in self.parameters for xi in x.flattened_parameters]
    @property
    def _parameter_sizes_(self):
        return [x.size for x in self.parameters]
    @property
    def parameter_shapes(self):
        return [xi for x in self.parameters for xi in x.parameter_shapes]
    @property
    def _constraints_str(self):
        return [cs for p in self.parameters for cs in p._constraints_str]
    @property
    def _priors_str(self):
        return [cs for p in self.parameters for cs in p._priors_str]
    @property
    def _description_str(self):
        return [xi for x in self.parameters for xi in x._description_str]
    @property
    def _ties_str(self):
        return [','.join(x._ties_str) for x in self.flattened_parameters]

    def _repr_html_(self, header=True):
        """Representation of the parameters in html for notebook display."""
        name = adjust_name_for_printing(self.name) + "."
        constrs = self._constraints_str;
        ts = self._ties_str
        prirs = self._priors_str
        desc = self._description_str; names = self.parameter_names()
        nl = max([len(str(x)) for x in names + [name]])
        sl = max([len(str(x)) for x in desc + ["Value"]])
        cl = max([len(str(x)) if x else 0 for x in constrs + ["Constraint"]])
        tl = max([len(str(x)) if x else 0 for x in ts + ["Tied to"]])
        pl = max([len(str(x)) if x else 0 for x in prirs + ["Prior"]])
        format_spec = "<tr><td>{{name:<{0}s}}</td><td align=\"right\">{{desc:>{1}s}}</td><td>{{const:^{2}s}}</td><td>{{pri:^{3}s}}</td><td>{{t:^{4}s}}</td></tr>".format(nl, sl, cl, pl, tl)
        to_print = []
        for n, d, c, t, p in itertools.izip(names, desc, constrs, ts, prirs):
            to_print.append(format_spec.format(name=n, desc=d, const=c, t=t, pri=p))
        sep = '-' * (nl + sl + cl + + pl + tl + 8 * 2 + 3)
        if header:
            header = """
<tr>
  <td><b>{name}</b>
  <td><b>Value</b></td>
  <td><b>Constraint</b></td>
  <td><b>Prior</b></td>
  <td><b>Tied to</b></td>""".format(name=name)
            to_print.insert(0, header)
        return '<table>' + '\n'.format(sep).join(to_print) + '\n</table>'

    def __str__(self, header=True):
        format_spec = "  \033[1m{{name:<{0}s}}\033[0;0m  |  {{desc:>{1}s}}  |  {{const:^{2}s}}  |  {{pri:^{3}s}}  |  {{t:^{4}s}}".format(nl, sl, cl, pl, tl)
        if header:
            header = "  {{0:<{0}s}}  |  {{1:^{1}s}}  |  {{2:^{2}s}}  |  {{3:^{3}s}}  |  {{4:^{4}s}}".format(nl, sl, cl, pl, tl).format(name, "Value", "Constraint", "Prior", "Tied to")
        return '\n'.format(sep).join(to_print)
    pass

"""
The module contains the tools for ploting 2D image visualizations
"""

from matplotlib.cm import jet

width_max = 15
height_max = 12

def _calculateFigureSize(x_size, y_size, fig_ncols, fig_nrows, pad):
    width = (x_size*fig_ncols+pad*(fig_ncols-1))
    height = (y_size*fig_nrows+pad*(fig_nrows-1))
    if width > float(height)/height_max*width_max:
        return (width_max, float(width_max)/width*height)
    else:
        return (float(height_max)/height*width, height_max)

def plot_2D_images(figure, arr, symmetric=False, pad=None, zoom=None, mode=None, interpolation='nearest'):
    ax = figure.add_subplot(111)
    if len(arr.shape)==2:
        arr = arr.reshape(*((1,)+arr.shape))
    fig_num = arr.shape[0]
    y_size = arr.shape[1]
    x_size = arr.shape[2]
    fig_ncols = int(np.ceil(np.sqrt(fig_num)))
    fig_nrows = int(np.ceil((float)(fig_num)/fig_ncols))
    if pad==None:
        pad = max(int(min(y_size,x_size)/10),1)
    
    figsize = _calculateFigureSize(x_size, y_size, fig_ncols, fig_nrows, pad)
    #figure.set_size_inches(figsize,forward=True)
    #figure.subplots_adjust(left=0.05, bottom=0.05, right=0.95, top=0.95)
    
    if symmetric:
        # symmetric around zero: fix zero as the middle color
        mval = max(abs(arr.max()),abs(arr.min()))
        arr = arr/(2.*mval)+0.5
    else:
        minval,maxval = arr.min(),arr.max()
        arr = (arr-minval)/(maxval-minval)

    if mode=='L':
        arr_color = np.empty(arr.shape+(3,))
        arr_color[:] = arr.reshape(*(arr.shape+(1,)))
    elif mode==None or mode=='jet':
        arr_color = jet(arr)
    
    buf = np.ones((y_size*fig_nrows+pad*(fig_nrows-1), x_size*fig_ncols+pad*(fig_ncols-1), 3),dtype=arr.dtype)
    
    for y in xrange(fig_nrows):
        for x in xrange(fig_ncols):
            if y*fig_ncols+x<fig_num:
                buf[y*y_size+y*pad:(y+1)*y_size+y*pad, x*x_size+x*pad:(x+1)*x_size+x*pad] = arr_color[y*fig_ncols+x,:,:,:3]
    img_plot = ax.imshow(buf, interpolation=interpolation)
    ax.axis('off')

class GPHeteroscedasticRegression(GP):
    """
    Gaussian Process model for heteroscedastic regression

    """
    def __init__(self, X, Y, kernel=None, Y_metadata=None):

        Ny = Y.shape[0]

        if Y_metadata is None:
            Y_metadata = {'output_index':np.arange(Ny)[:,None]}
        else:
            assert Y_metadata['output_index'].shape[0] == Ny

        #likelihoods_list = [likelihoods.Gaussian(name="Gaussian_noise_%s" %j) for j in range(Ny)]
        likelihoods_list = [likelihoods.Gaussian(name="Gaussian_noise_%s" %j) for j in noise_terms]
        likelihood = likelihoods.MixedNoise(likelihoods_list=likelihoods_list)

        super(GPHeteroscedasticRegression, self).__init__(X,Y,kernel,likelihood, Y_metadata=Y_metadata)

    def plot(self,*args):
        return NotImplementedError

import matplotlib as mpl
import sys
#sys.path.append('/home/james/mlprojects/sitran_cluster/')
#from switch_pylab_backend import *

#this stuff isn;t really Tango related: maybe it could be moved out? TODO

colorsHex = {\
"Aluminium6":"#2e3436",\
"Aluminium5":"#555753",\
"Aluminium4":"#888a85",\
"Aluminium3":"#babdb6",\
"Aluminium2":"#d3d7cf",\
"Aluminium1":"#eeeeec",\
"lightPurple":"#ad7fa8",\
"mediumPurple":"#75507b",\
"darkPurple":"#5c3566",\
"lightBlue":"#729fcf",\
"mediumBlue":"#3465a4",\
"darkBlue": "#204a87",\
"lightGreen":"#8ae234",\
"mediumGreen":"#73d216",\
"darkGreen":"#4e9a06",\
"lightChocolate":"#e9b96e",\
"mediumChocolate":"#c17d11",\
"darkChocolate":"#8f5902",\
"lightRed":"#ef2929",\
"mediumRed":"#cc0000",\
"darkRed":"#a40000",\
"lightOrange":"#fcaf3e",\
"mediumOrange":"#f57900",\
"darkOrange":"#ce5c00",\
"lightButter":"#fce94f",\
"mediumButter":"#edd400",\
"darkButter":"#c4a000"}

darkList = [colorsHex['darkBlue'],colorsHex['darkRed'],colorsHex['darkGreen'], colorsHex['darkOrange'], colorsHex['darkButter'], colorsHex['darkPurple'], colorsHex['darkChocolate'], colorsHex['Aluminium6']]
mediumList = [colorsHex['mediumBlue'], colorsHex['mediumRed'],colorsHex['mediumGreen'], colorsHex['mediumOrange'], colorsHex['mediumButter'], colorsHex['mediumPurple'], colorsHex['mediumChocolate'], colorsHex['Aluminium5']]
lightList = [colorsHex['lightBlue'], colorsHex['lightRed'],colorsHex['lightGreen'], colorsHex['lightOrange'], colorsHex['lightButter'], colorsHex['lightPurple'], colorsHex['lightChocolate'], colorsHex['Aluminium4']]

def currentDark():
    return darkList[-1]
def currentMedium():
    return mediumList[-1]
def currentLight():
    return lightList[-1]

def nextDark():
    darkList.append(darkList.pop(0))
def nextMedium():
    mediumList.append(mediumList.pop(0))
def nextLight():
    lightList.append(lightList.pop(0))

def reset():
    while not darkList[0]==colorsHex['darkBlue']:
        darkList.append(darkList.pop(0))
    while not mediumList[0]==colorsHex['mediumBlue']:
        mediumList.append(mediumList.pop(0))
    while not lightList[0]==colorsHex['lightBlue']:
        lightList.append(lightList.pop(0))

def setLightFigures():
    mpl.rcParams['axes.edgecolor']=colorsHex['Aluminium6']
    mpl.rcParams['axes.facecolor']=colorsHex['Aluminium2']
    mpl.rcParams['axes.labelcolor']=colorsHex['Aluminium6']
    mpl.rcParams['figure.edgecolor']=colorsHex['Aluminium6']
    mpl.rcParams['figure.facecolor']=colorsHex['Aluminium2']
    mpl.rcParams['grid.color']=colorsHex['Aluminium6']
    mpl.rcParams['savefig.edgecolor']=colorsHex['Aluminium2']
    mpl.rcParams['savefig.facecolor']=colorsHex['Aluminium2']
    mpl.rcParams['text.color']=colorsHex['Aluminium6']
    mpl.rcParams['xtick.color']=colorsHex['Aluminium6']
    mpl.rcParams['ytick.color']=colorsHex['Aluminium6']

def setDarkFigures():
    mpl.rcParams['axes.edgecolor']=colorsHex['Aluminium2']
    mpl.rcParams['axes.facecolor']=colorsHex['Aluminium6']
    mpl.rcParams['axes.labelcolor']=colorsHex['Aluminium2']
    mpl.rcParams['figure.edgecolor']=colorsHex['Aluminium2']
    mpl.rcParams['figure.facecolor']=colorsHex['Aluminium6']
    mpl.rcParams['grid.color']=colorsHex['Aluminium2']
    mpl.rcParams['savefig.edgecolor']=colorsHex['Aluminium6']
    mpl.rcParams['savefig.facecolor']=colorsHex['Aluminium6']
    mpl.rcParams['text.color']=colorsHex['Aluminium2']
    mpl.rcParams['xtick.color']=colorsHex['Aluminium2']
    mpl.rcParams['ytick.color']=colorsHex['Aluminium2']

def hex2rgb(hexcolor):
    hexcolor = [hexcolor[1+2*i:1+2*(i+1)] for i in range(3)]
    r,g,b = [int(n,16) for n in hexcolor]
    return (r,g,b)

colorsRGB = dict([(k,hex2rgb(i)) for k,i in colorsHex.items()])

cdict_RB = {'red' :((0.,colorsRGB['mediumRed'][0]/256.,colorsRGB['mediumRed'][0]/256.),
                     (.5,colorsRGB['mediumPurple'][0]/256.,colorsRGB['mediumPurple'][0]/256.),
                     (1.,colorsRGB['mediumBlue'][0]/256.,colorsRGB['mediumBlue'][0]/256.)),
            'green':((0.,colorsRGB['mediumRed'][1]/256.,colorsRGB['mediumRed'][1]/256.),
                     (.5,colorsRGB['mediumPurple'][1]/256.,colorsRGB['mediumPurple'][1]/256.),
                     (1.,colorsRGB['mediumBlue'][1]/256.,colorsRGB['mediumBlue'][1]/256.)),
            'blue':((0.,colorsRGB['mediumRed'][2]/256.,colorsRGB['mediumRed'][2]/256.),
                      (.5,colorsRGB['mediumPurple'][2]/256.,colorsRGB['mediumPurple'][2]/256.),
                      (1.,colorsRGB['mediumBlue'][2]/256.,colorsRGB['mediumBlue'][2]/256.))}

cdict_BGR = {'red' :((0.,colorsRGB['mediumBlue'][0]/256.,colorsRGB['mediumBlue'][0]/256.),
                     (.5,colorsRGB['mediumGreen'][0]/256.,colorsRGB['mediumGreen'][0]/256.),
                     (1.,colorsRGB['mediumRed'][0]/256.,colorsRGB['mediumRed'][0]/256.)),
            'green':((0.,colorsRGB['mediumBlue'][1]/256.,colorsRGB['mediumBlue'][1]/256.),
                     (.5,colorsRGB['mediumGreen'][1]/256.,colorsRGB['mediumGreen'][1]/256.),
                     (1.,colorsRGB['mediumRed'][1]/256.,colorsRGB['mediumRed'][1]/256.)),
            'blue':((0.,colorsRGB['mediumBlue'][2]/256.,colorsRGB['mediumBlue'][2]/256.),
                      (.5,colorsRGB['mediumGreen'][2]/256.,colorsRGB['mediumGreen'][2]/256.),
                      (1.,colorsRGB['mediumRed'][2]/256.,colorsRGB['mediumRed'][2]/256.))}

cdict_Alu = {'red' :((0./5,colorsRGB['Aluminium1'][0]/256.,colorsRGB['Aluminium1'][0]/256.),
                     (1./5,colorsRGB['Aluminium2'][0]/256.,colorsRGB['Aluminium2'][0]/256.),
                     (2./5,colorsRGB['Aluminium3'][0]/256.,colorsRGB['Aluminium3'][0]/256.),
                     (3./5,colorsRGB['Aluminium4'][0]/256.,colorsRGB['Aluminium4'][0]/256.),
                     (4./5,colorsRGB['Aluminium5'][0]/256.,colorsRGB['Aluminium5'][0]/256.),
                     (5./5,colorsRGB['Aluminium6'][0]/256.,colorsRGB['Aluminium6'][0]/256.)),
           'green' :((0./5,colorsRGB['Aluminium1'][1]/256.,colorsRGB['Aluminium1'][1]/256.),
                     (1./5,colorsRGB['Aluminium2'][1]/256.,colorsRGB['Aluminium2'][1]/256.),
                     (2./5,colorsRGB['Aluminium3'][1]/256.,colorsRGB['Aluminium3'][1]/256.),
                     (3./5,colorsRGB['Aluminium4'][1]/256.,colorsRGB['Aluminium4'][1]/256.),
                     (4./5,colorsRGB['Aluminium5'][1]/256.,colorsRGB['Aluminium5'][1]/256.),
                     (5./5,colorsRGB['Aluminium6'][1]/256.,colorsRGB['Aluminium6'][1]/256.)),
            'blue' :((0./5,colorsRGB['Aluminium1'][2]/256.,colorsRGB['Aluminium1'][2]/256.),
                     (1./5,colorsRGB['Aluminium2'][2]/256.,colorsRGB['Aluminium2'][2]/256.),
                     (2./5,colorsRGB['Aluminium3'][2]/256.,colorsRGB['Aluminium3'][2]/256.),
                     (3./5,colorsRGB['Aluminium4'][2]/256.,colorsRGB['Aluminium4'][2]/256.),
                     (4./5,colorsRGB['Aluminium5'][2]/256.,colorsRGB['Aluminium5'][2]/256.),
                     (5./5,colorsRGB['Aluminium6'][2]/256.,colorsRGB['Aluminium6'][2]/256.))}
# cmap_Alu = mpl.colors.LinearSegmentedColormap('TangoAluminium',cdict_Alu,256)
# cmap_BGR = mpl.colors.LinearSegmentedColormap('TangoRedBlue',cdict_BGR,256)
# cmap_RB = mpl.colors.LinearSegmentedColormap('TangoRedBlue',cdict_RB,256)
if __name__=='__main__':
    pb.pcolor(pb.rand(10,10),cmap=cmap_RB)
    pb.colorbar()
    pb.show()

import sys
from GPy.models.sparse_gp_regression import SparseGPRegression

class SparseGPLVM(SparseGPRegression):
    """
    Sparse Gaussian Process Latent Variable Model

    """
    def __init__(self, Y, input_dim, X=None, kernel=None, init='PCA', num_inducing=10):
        SparseGPRegression.__init__(self, X, Y, kernel=kernel, num_inducing=num_inducing)

        super(SparseGPLVM, self).parameters_changed()
        self.X.gradient = self.kern.gradients_X_diag(self.grad_dict['dL_dKdiag'], self.X)
        self.X.gradient += self.kern.gradients_X(self.grad_dict['dL_dKnm'], self.X, self.Z)

                plot_limits=None, 

"""
Gaussian Processes regression examples
"""
try:
except:
    pass
import GPy

def olympic_marathon_men(optimize=True, plot=True):
    """Run a standard Gaussian process regression on the Olympic marathon data."""
        print 'pods unavailable, see https://github.com/sods/ods for example datasets'
        return
    data = pods.datasets.olympic_marathon_men()

    # create simple GP Model
    m = GPy.models.GPRegression(data['X'], data['Y'])

    # set the lengthscale to be something sensible (defaults to 1)
    m.kern.lengthscale = 10.

        m.optimize('bfgs', max_iters=200)
    if plot:
        m.plot(plot_limits=(1850, 2050))

    return m

def coregionalization_toy(optimize=True, plot=True):
    """
    A simple demonstration of coregionalization on two sinusoidal functions.
    """
    #build a design matrix with a column of integers indicating the output
    X1 = np.random.rand(50, 1) * 8
    X2 = np.random.rand(30, 1) * 5

    #build a suitable set of observed variables
    Y1 = np.sin(X1) + np.random.randn(*X1.shape) * 0.05
    Y2 = np.sin(X2) + np.random.randn(*X2.shape) * 0.05 + 2.

    m = GPy.models.GPCoregionalizedRegression(X_list=[X1,X2], Y_list=[Y1,Y2])

        m.optimize('bfgs', max_iters=100)

    if plot:
        slices = GPy.util.multioutput.get_slices([X1,X2])
        m.plot(fixed_inputs=[(1,0)],which_data_rows=slices[0],Y_metadata={'output_index':0})
        m.plot(fixed_inputs=[(1,1)],which_data_rows=slices[1],Y_metadata={'output_index':1},ax=pb.gca())
    return m

def coregionalization_sparse(optimize=True, plot=True):
    """
    A simple demonstration of coregionalization on two sinusoidal functions using sparse approximations.
    """

    m = GPy.models.SparseGPCoregionalizedRegression(X_list=[X1,X2], Y_list=[Y1,Y2])

    if plot:
        pb.ylim(-3,)

    return m

def epomeo_gpx(max_iters=200, optimize=True, plot=True):
    """
    Perform Gaussian process regression on the latitude and longitude data
    from the Mount Epomeo runs. Requires gpxpy to be installed on your system
    to load in the data.
    """
        return
    data = pods.datasets.epomeo_gpx()
    num_data_list = []
    for Xpart in data['X']:
        num_data_list.append(Xpart.shape[0])

    num_data_array = np.array(num_data_list)
    num_data = num_data_array.sum()
    Y = np.zeros((num_data, 2))
    t = np.zeros((num_data, 2))
    start = 0
    for Xpart, index in zip(data['X'], range(len(data['X']))):
        end = start+Xpart.shape[0]
        t[start:end, :] = np.hstack((Xpart[:, 0:1],
                                    index*np.ones((Xpart.shape[0], 1))))
        Y[start:end, :] = Xpart[:, 1:3]

    num_inducing = 200
    Z = np.hstack((np.linspace(t[:,0].min(), t[:, 0].max(), num_inducing)[:, None],
                   np.random.randint(0, 4, num_inducing)[:, None]))

    k1 = GPy.kern.RBF(1)
    k2 = GPy.kern.Coregionalize(output_dim=5, rank=5)
    k = k1**k2

    m = GPy.models.SparseGPRegression(t, Y, kernel=k, Z=Z, normalize_Y=True)
    m.constrain_fixed('.*variance', 1.)
    m.inducing_inputs.constrain_fixed()
    m.Gaussian_noise.variance.constrain_bounded(1e-3, 1e-1)
    m.optimize(max_iters=max_iters,messages=True)

    return m

def multiple_optima(gene_number=937, resolution=80, model_restarts=10, seed=10000, max_iters=300, optimize=True, plot=True):
    """
    Show an example of a multimodal error surface for Gaussian process
    regression. Gene 939 has bimodal behaviour where the noisy mode is
    higher.
    """

    # Contour over a range of length scales and signal/noise ratios.
    length_scales = np.linspace(0.1, 60., resolution)
    log_SNRs = np.linspace(-3., 4., resolution)

        return
    data = pods.datasets.della_gatta_TRP63_gene_expression(data_set='della_gatta',gene_number=gene_number)
    # data['Y'] = data['Y'][0::2, :]
    # data['X'] = data['X'][0::2, :]

    data['Y'] = data['Y'] - np.mean(data['Y'])

    lls = GPy.examples.regression._contour_data(data, length_scales, log_SNRs, GPy.kern.RBF)
    if plot:
        pb.contour(length_scales, log_SNRs, np.exp(lls), 20, cmap=pb.cm.jet)
        ax = pb.gca()
        pb.xlabel('length scale')
        pb.ylabel('log_10 SNR')

        xlim = ax.get_xlim()
        ylim = ax.get_ylim()

    # Now run a few optimizations
    models = []
    optim_point_x = np.empty(2)
    optim_point_y = np.empty(2)
    for i in range(0, model_restarts):
        # kern = GPy.kern.RBF(1, variance=np.random.exponential(1.), lengthscale=np.random.exponential(50.))
        kern = GPy.kern.RBF(1, variance=np.random.uniform(1e-3, 1), lengthscale=np.random.uniform(5, 50))

        m = GPy.models.GPRegression(data['X'], data['Y'], kernel=kern)
        m.likelihood.variance = np.random.uniform(1e-3, 1)
        optim_point_x[0] = m.rbf.lengthscale
        optim_point_y[0] = np.log10(m.rbf.variance) - np.log10(m.likelihood.variance);

        # optimize
        if optimize:
            m.optimize('scg', xtol=1e-6, ftol=1e-6, max_iters=max_iters)

        optim_point_x[1] = m.rbf.lengthscale
        optim_point_y[1] = np.log10(m.rbf.variance) - np.log10(m.likelihood.variance);

        if plot:
            pb.arrow(optim_point_x[0], optim_point_y[0], optim_point_x[1] - optim_point_x[0], optim_point_y[1] - optim_point_y[0], label=str(i), head_length=1, head_width=0.5, fc='k', ec='k')
        models.append(m)

    if plot:
    return m # (models, lls)

def _contour_data(data, length_scales, log_SNRs, kernel_call=GPy.kern.RBF):
    """
    Evaluate the GP objective function for a given data set for a range of
    signal to noise ratios and a range of lengthscales.

    :data_set: A data set from the utils.datasets director.
    :length_scales: a list of length scales to explore for the contour plot.
    :log_SNRs: a list of base 10 logarithm signal to noise ratios to explore for the contour plot.
    :kernel: a kernel to use for the 'signal' portion of the data.
    """

    lls = []
    total_var = np.var(data['Y'])
    kernel = kernel_call(1, variance=1., lengthscale=1.)
    model = GPy.models.GPRegression(data['X'], data['Y'], kernel=kernel)
    for log_SNR in log_SNRs:
        SNR = 10.**log_SNR
        noise_var = total_var / (1. + SNR)
        signal_var = total_var - noise_var
        model.kern['.*variance'] = signal_var
        model.likelihood.variance = noise_var
        length_scale_lls = []

        for length_scale in length_scales:
            model['.*lengthscale'] = length_scale
            length_scale_lls.append(model.log_likelihood())

        lls.append(length_scale_lls)

    return np.array(lls)

def olympic_100m_men(optimize=True, plot=True):
    """Run a standard Gaussian process regression on the Rogers and Girolami olympics data."""
        return
    data = pods.datasets.olympic_100m_men()

    m.rbf.lengthscale = 10

    if plot:
    return m

def toy_rbf_1d(optimize=True, plot=True):
    """Run a simple demonstration of a standard Gaussian process fitting it to data sampled from an RBF covariance."""
        return
    data = pods.datasets.toy_rbf_1d()

        m.optimize('bfgs')
    if plot:
        m.plot()

    return m

def toy_rbf_1d_50(optimize=True, plot=True):
        return
    data = pods.datasets.toy_rbf_1d_50()

    if plot:
        m.plot()

    return m

def toy_poisson_rbf_1d_laplace(optimize=True, plot=True):
    optimizer='scg'
    x_len = 30
    X = np.linspace(0, 10, x_len)[:, None]
    f_true = np.random.multivariate_normal(np.zeros(x_len), GPy.kern.RBF(1).K(X))
    Y = np.array([np.random.poisson(np.exp(f)) for f in f_true])[:,None]

    kern = GPy.kern.RBF(1)
    poisson_lik = GPy.likelihoods.Poisson()

    m = GPy.core.GP(X, Y, kernel=kern, likelihood=poisson_lik, inference_method=laplace_inf)

        m.optimize(optimizer)
    if plot:
        m.plot()
        # plot the real underlying rate function
        pb.plot(X, np.exp(f_true), '--k', linewidth=2)

    return m

def toy_ARD(max_iters=1000, kernel_type='linear', num_samples=300, D=4, optimize=True, plot=True):
    # Create an artificial dataset where the values in the targets (Y)
    # only depend in dimensions 1 and 3 of the inputs (X). Run ARD to
    # see if this dependency can be recovered
    X1 = np.sin(np.sort(np.random.rand(num_samples, 1) * 10, 0))
    X2 = np.cos(np.sort(np.random.rand(num_samples, 1) * 10, 0))
    X3 = np.exp(np.sort(np.random.rand(num_samples, 1), 0))
    X4 = np.log(np.sort(np.random.rand(num_samples, 1), 0))
    X = np.hstack((X1, X2, X3, X4))

    Y1 = np.asarray(2 * X[:, 0] + 3).reshape(-1, 1)
    Y2 = np.asarray(4 * (X[:, 2] - 1.5 * X[:, 0])).reshape(-1, 1)
    Y = np.hstack((Y1, Y2))

    Y = np.dot(Y, np.random.rand(2, D));
    Y = Y + 0.2 * np.random.randn(Y.shape[0], Y.shape[1])

    if kernel_type == 'linear':
        kernel = GPy.kern.Linear(X.shape[1], ARD=1)
    elif kernel_type == 'rbf_inv':
        kernel = GPy.kern.RBF_inv(X.shape[1], ARD=1)
    else:
        kernel = GPy.kern.RBF(X.shape[1], ARD=1)
    kernel += GPy.kern.White(X.shape[1]) + GPy.kern.Bias(X.shape[1])
    m = GPy.models.GPRegression(X, Y, kernel)
    # len_prior = GPy.priors.inverse_gamma(1,18) # 1, 25
    # m.set_prior('.*lengthscale',len_prior)

        m.optimize(optimizer='scg', max_iters=max_iters)

    if plot:

    return m

def toy_ARD_sparse(max_iters=1000, kernel_type='linear', num_samples=300, D=4, optimize=True, plot=True):

    Y1 = np.asarray(2 * X[:, 0] + 3)[:, None]
    Y2 = np.asarray(4 * (X[:, 2] - 1.5 * X[:, 0]))[:, None]

    else:
    #kernel += GPy.kern.Bias(X.shape[1])
    X_variance = np.ones(X.shape) * 0.5
    m = GPy.models.SparseGPRegression(X, Y, kernel, X_variance=X_variance)

    if plot:

    return m

def robot_wireless(max_iters=100, kernel=None, optimize=True, plot=True):
    """Predict the location of a robot given wirelss signal strength readings."""
        return

    m = GPy.models.GPRegression(data['Y'], data['X'], kernel=kernel)

    # optimize
        m.optimize(max_iters=max_iters)

    Xpredict = m.predict(data['Ytest'])[0]
    if plot:
        pb.plot(data['Xtest'][:, 0], data['Xtest'][:, 1], 'r-')
        pb.plot(Xpredict[:, 0], Xpredict[:, 1], 'b-')
        pb.axis('equal')
        pb.title('WiFi Localization with Gaussian Processes')
        pb.legend(('True Location', 'Predicted Location'))

    sse = ((data['Xtest'] - Xpredict)**2).sum()

    print('Sum of squares error on test data: ' + str(sse))
    return m

def silhouette(max_iters=100, optimize=True, plot=True):
    """Predict the pose of a figure given a silhouette. This is a task from Agarwal and Triggs 2004 ICML paper."""
        return
    data = pods.datasets.silhouette()

    # optimize
        m.optimize(messages=True, max_iters=max_iters)

    print m
    return m

def sparse_GP_regression_1D(num_samples=400, num_inducing=5, max_iters=100, optimize=True, plot=True, checkgrad=False):
    """Run a 1D example of a sparse GP regression."""
    # sample inputs and outputs
    X = np.random.uniform(-3., 3., (num_samples, 1))
    Y = np.sin(X) + np.random.randn(num_samples, 1) * 0.05
    # construct kernel
    rbf = GPy.kern.RBF(1)
    m = GPy.models.SparseGPRegression(X, Y, kernel=rbf, num_inducing=num_inducing)

    if checkgrad:

        m.optimize('tnc', max_iters=max_iters)

    if plot:
        m.plot()

    return m

def sparse_GP_regression_2D(num_samples=400, num_inducing=50, max_iters=100, optimize=True, plot=True, nan=False):
    """Run a 2D example of a sparse GP regression."""
    np.random.seed(1234)
    X = np.random.uniform(-3., 3., (num_samples, 2))
    Y = np.sin(X[:, 0:1]) * np.sin(X[:, 1:2]) + np.random.randn(num_samples, 1) * 0.05
    if nan:
        inan = np.random.binomial(1,.2,size=Y.shape)
        Y[inan] = np.nan

    rbf = GPy.kern.RBF(2)

    # contrain all parameters to be positive (but not inducing inputs)
    m['.*len'] = 2.

    m.checkgrad()

    # optimize
        m.optimize('tnc', messages=1, max_iters=max_iters)

    # plot
    if plot:
        m.plot()

    print m
    return m

def uncertain_inputs_sparse_regression(max_iters=200, optimize=True, plot=True):
    """Run a 1D example of a sparse GP regression with uncertain inputs."""
    fig, axes = pb.subplots(1, 2, figsize=(12, 5), sharex=True, sharey=True)

    S = np.ones((20, 1))
    X = np.random.uniform(-3., 3., (20, 1))
    Y = np.sin(X) + np.random.randn(20, 1) * 0.05
    # likelihood = GPy.likelihoods.Gaussian(Y)
    Z = np.random.uniform(-3., 3., (7, 1))

    k = GPy.kern.RBF(1)
    # create simple GP Model - no input uncertainty on this one
    m = GPy.models.SparseGPRegression(X, Y, kernel=k, Z=Z)

        m.optimize('scg', messages=1, max_iters=max_iters)

    if plot:
        m.plot(ax=axes[0])
        axes[0].set_title('no input uncertainty')
    print m

    # the same Model with uncertainty
    m = GPy.models.SparseGPRegression(X, Y, kernel=GPy.kern.RBF(1), Z=Z, X_variance=S)
    if plot:
        axes[1].set_title('with input uncertainty')

    print m
    return m

from parameter_core import Pickleable
from observable import Observable

class ObsAr(np.ndarray, Pickleable, Observable):
    """
    An ndarray which reports changes to its observers.
    The observers can add themselves with a callable, which
    will be called every time this array changes. The callable
    takes exactly one argument, which is this array itself.
    """
    __array_priority__ = -1 # Never give back ObsAr
    def __new__(cls, input_array, *a, **kw):
        # allways make a copy of input paramters, as we need it to be in C order:
        if not isinstance(input_array, ObsAr):
            obj = np.atleast_1d(np.require(input_array, dtype=np.float64, requirements=['W', 'C'])).view(cls)
        else: obj = input_array
        super(ObsAr, obj).__init__(*a, **kw)
        return obj

    def __array_finalize__(self, obj):
        # see InfoArray.__array_finalize__ for comments
        if obj is None: return
        self.observers = getattr(obj, 'observers', None)

    def __array_wrap__(self, out_arr, context=None):
        return out_arr.view(np.ndarray)

        # do not setup anything, as observable arrays do not have default observers
        pass

    @property
    def values(self):
        return self.view(np.ndarray)

        memo = {}
        memo[id(self)] = self
        memo[id(self.observers)] = ObserverList()
        return self.__deepcopy__(memo)

        s = self.__new__(self.__class__, input_array=self.view(np.ndarray).copy())
        memo[id(self)] = s
        Pickleable.__setstate__(s, copy.deepcopy(self.__getstate__(), memo))
        return s

    def __reduce__(self):
        func, args, state = super(ObsAr, self).__reduce__()
        return func, args, (state, Pickleable.__getstate__(self))

        np.ndarray.__setstate__(self, state[0])
        Pickleable.__setstate__(self, state[1])

    def __setitem__(self, s, val):
        super(ObsAr, self).__setitem__(s, val)
        self.notify_observers()

    def __getslice__(self, start, stop):
        return self.__getitem__(slice(start, stop))

    def __setslice__(self, start, stop, val):
        return self.__setitem__(slice(start, stop), val)

    def __ilshift__(self, *args, **kwargs):
        r = np.ndarray.__ilshift__(self, *args, **kwargs)
        return r

    def __irshift__(self, *args, **kwargs):
        r = np.ndarray.__irshift__(self, *args, **kwargs)
        return r

    def __ixor__(self, *args, **kwargs):
        r = np.ndarray.__ixor__(self, *args, **kwargs)
        return r

    def __ipow__(self, *args, **kwargs):
        r = np.ndarray.__ipow__(self, *args, **kwargs)
        return r

    def __ifloordiv__(self, *args, **kwargs):
        r = np.ndarray.__ifloordiv__(self, *args, **kwargs)
        return r

    def __isub__(self, *args, **kwargs):
        r = np.ndarray.__isub__(self, *args, **kwargs)
        return r

    def __ior__(self, *args, **kwargs):
        r = np.ndarray.__ior__(self, *args, **kwargs)
        return r

    def __itruediv__(self, *args, **kwargs):
        r = np.ndarray.__itruediv__(self, *args, **kwargs)
        return r

    def __idiv__(self, *args, **kwargs):
        r = np.ndarray.__idiv__(self, *args, **kwargs)
        return r

    def __iand__(self, *args, **kwargs):
        r = np.ndarray.__iand__(self, *args, **kwargs)
        return r

    def __imod__(self, *args, **kwargs):
        r = np.ndarray.__imod__(self, *args, **kwargs)
        return r

    def __iadd__(self, *args, **kwargs):
        r = np.ndarray.__iadd__(self, *args, **kwargs)
        return r

    def __imul__(self, *args, **kwargs):
        r = np.ndarray.__imul__(self, *args, **kwargs)
        return r

import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import GPy
try:
    import visual
    visual_available = True

    visual_available = False

class data_show:
    """
    The data_show class is a base class which describes how to visualize a
    particular data set. For example, motion capture data can be plotted as a
    stick figure, or images are shown using imshow. This class enables latent
    to data visualizations for the GP-LVM.
    """
    def __init__(self, vals):
        self.vals = vals.copy()
        # If no axes are defined, create some.

    def modify(self, vals):
        raise NotImplementedError, "this needs to be implemented to use the data_show class"

class vpython_show(data_show):
    """
    the vpython_show class is a base class for all visualization methods that use vpython to display. It is initialized with a scene. If the scene is set to None it creates a scene window.
    """

    def __init__(self, vals, scene=None):
        data_show.__init__(self, vals)

        if scene==None:
            self.scene = visual.display(title='Data Visualization')
        else:
            self.scene = scene

        self.scene.exit()

class matplotlib_show(data_show):
    """
    the matplotlib_show class is a base class for all visualization methods that use matplotlib. It is initialized with an axis. If the axis is set to None it creates a figure window.
    """
    def __init__(self, vals, axes=None):

        if axes==None:
            fig = plt.figure()
            self.axes = fig.add_subplot(111)
        else:
            self.axes = axes

        plt.close(self.axes.get_figure())

class vector_show(matplotlib_show):
    """
    A base visualization class that just shows a data vector as a plot of
    vector elements alongside their indices.
    """
        matplotlib_show.__init__(self, vals, axes)
        #assert vals.ndim == 2, "Please give a vector in [n x 1] to plot"
        #assert vals.shape[1] == 1, "only showing a vector in one dimension"
        self.size = vals.size
        self.handle = self.axes.plot(np.arange(0, vals.size)[:, None], vals)[0]

        xdata, ydata = self.handle.get_data()
        assert vals.size == self.size, "values passed into modify changed size! vals.size:{} != in.size:{}".format(vals.size, self.size)
        self.handle.set_data(xdata, self.vals)
        self.axes.figure.canvas.draw()

class lvm(matplotlib_show):
    def __init__(self, vals, model, data_visualize, latent_axes=None, sense_axes=None, latent_index=[0,1], disable_drag=False):
        """Visualize a latent variable model

        :param model: the latent variable model to visualize.
        :param data_visualize: the object used to visualize the data which has been modelled.
        :type data_visualize: visualize.data_show  type.
        :param latent_axes: the axes where the latent visualization should be plotted.
        """
        if vals is None:
            if isinstance(model.X, VariationalPosterior):
                vals = model.X.mean.values
            else:
                vals = model.X.values
        if len(vals.shape)==1:
            vals = vals[None,:]
        matplotlib_show.__init__(self, vals, axes=latent_axes)

        if isinstance(latent_axes,mpl.axes.Axes):
            self.cid = latent_axes.figure.canvas.mpl_connect('button_press_event', self.on_click)
            if not disable_drag:
                self.cid = latent_axes.figure.canvas.mpl_connect('motion_notify_event', self.on_move)
            self.cid = latent_axes.figure.canvas.mpl_connect('axes_leave_event', self.on_leave)
            self.cid = latent_axes.figure.canvas.mpl_connect('axes_enter_event', self.on_enter)
        else:
            self.cid = latent_axes[0].figure.canvas.mpl_connect('button_press_event', self.on_click)
                self.cid = latent_axes[0].figure.canvas.mpl_connect('motion_notify_event', self.on_move)
            self.cid = latent_axes[0].figure.canvas.mpl_connect('axes_leave_event', self.on_leave)
            self.cid = latent_axes[0].figure.canvas.mpl_connect('axes_enter_event', self.on_enter)

        self.data_visualize = data_visualize
        self.latent_axes = latent_axes
        self.sense_axes = sense_axes
        self.called = False
        self.move_on = False
        self.latent_index = latent_index
        self.latent_dim = model.input_dim
        self.disable_drag = disable_drag

        # The red cross which shows current latent point.
        self.latent_values = vals
        self.latent_handle = self.latent_axes.plot([0],[0],'rx',mew=2)[0]
        self.modify(vals)
        self.show_sensitivities()

        """When latent values are modified update the latent representation and ulso update the output visualization."""
        self.vals = vals.view(np.ndarray).copy()
        y = self.model.predict(self.vals)[0]
        self.data_visualize.modify(y)
        self.latent_handle.set_data(self.vals[0,self.latent_index[0]], self.vals[0,self.latent_index[1]])

    def on_enter(self,event):
        pass
    def on_leave(self,event):
        pass

    def on_click(self, event):
        if event.inaxes!=self.latent_axes: return
        if self.disable_drag:
            self.move_on = True
            self.called = True
            self.on_move(event)
        else:
            self.move_on = not self.move_on

    def on_move(self, event):
        if self.called and self.move_on:
            # Call modify code on move
            self.latent_values[:, self.latent_index[0]]=event.xdata
            self.latent_values[:, self.latent_index[1]]=event.ydata
            self.modify(self.latent_values)

    def show_sensitivities(self):
        # A click in the bar chart axis for selection a dimension.
        if self.sense_axes != None:
            self.sense_axes.cla()
            self.sense_axes.bar(np.arange(self.model.input_dim), self.model.input_sensitivity(), color='b')

            if self.latent_index[1] == self.latent_index[0]:
                self.sense_axes.bar(np.array(self.latent_index[0]), self.model.input_sensitivity()[self.latent_index[0]], color='y')
                self.sense_axes.bar(np.array(self.latent_index[1]), self.model.input_sensitivity()[self.latent_index[1]], color='y')

            else:
                self.sense_axes.bar(np.array(self.latent_index[0]), self.model.input_sensitivity()[self.latent_index[0]], color='g')
                self.sense_axes.bar(np.array(self.latent_index[1]), self.model.input_sensitivity()[self.latent_index[1]], color='r')

            self.sense_axes.figure.canvas.draw()

class lvm_subplots(lvm):
    """
    latent_axes is a np array of dimension np.ceil(input_dim/2),
    one for each pair of the latent dimensions.
    """
    def __init__(self, vals, Model, data_visualize, latent_axes=None, sense_axes=None):
        self.nplots = int(np.ceil(Model.input_dim/2.))+1
        assert len(latent_axes)==self.nplots
        if vals==None:
            vals = Model.X[0, :]

        for i, axis in enumerate(latent_axes):
            if i == self.nplots-1:
                if self.nplots*2!=Model.input_dim:
                    latent_index = [i*2, i*2]
                lvm.__init__(self, self.latent_vals, Model, data_visualize, axis, sense_axes, latent_index=latent_index)
            else:
                latent_index = [i*2, i*2+1]
                lvm.__init__(self, self.latent_vals, Model, data_visualize, axis, latent_index=latent_index)

class lvm_dimselect(lvm):
    """
    A visualizer for latent variable models which allows selection of the latent dimensions to use by clicking on a bar chart of their length scales.

    For an example of the visualizer's use try:

    GPy.examples.dimensionality_reduction.BGPVLM_oil()

    """
    def __init__(self, vals, model, data_visualize, latent_axes=None, sense_axes=None, latent_index=[0, 1], labels=None):
        if latent_axes==None and sense_axes==None:
            self.fig,(latent_axes,self.sense_axes) = plt.subplots(1,2)
        elif sense_axes==None:
            fig=plt.figure()
            self.sense_axes = fig.add_subplot(111)
        else:
            self.sense_axes = sense_axes
        self.labels = labels
        lvm.__init__(self,vals,model,data_visualize,latent_axes,sense_axes,latent_index)
        print self.latent_values
        print "use left and right mouse buttons to select dimensions"

        if event.inaxes==self.sense_axes:
            new_index = max(0,min(int(np.round(event.xdata-0.5)),self.model.input_dim-1))
            if event.button == 1:
                # Make it red if and y-axis (red=port=left) if it is a left button click
                self.latent_index[1] = new_index
            else:
                # Make it green and x-axis (green=starboard=right) if it is a right button click
                self.latent_index[0] = new_index

            self.show_sensitivities()

            self.latent_axes.cla()
            self.model.plot_latent(which_indices=self.latent_index,
                                   ax=self.latent_axes, labels=self.labels)
            self.latent_handle = self.latent_axes.plot([0],[0],'rx',mew=2)[0]

        elif event.inaxes==self.latent_axes:

        self.called = True

        print type(self.latent_values)
        latent_values = self.latent_values.copy()
        y = self.model.predict(latent_values[None,:])[0]

class image_show(matplotlib_show):
    """Show a data vector as an image. This visualizer rehapes the output vector and displays it as an image.

    :param vals: the values of the output to display.
    :type vals: ndarray
    :param axes: the axes to show the output on.
    :type vals: axes handle
    :param dimensions: the dimensions that the image needs to be transposed to for display.
    :type dimensions: tuple
    :param transpose: whether to transpose the image before display.
    :type bool: default is False.
    :param order: whether array is in Fortan ordering ('F') or Python ordering ('C'). Default is python ('C').
    :type order: string
    :param invert: whether to invert the pixels or not (default False).
    :type invert: bool
    :param palette: a palette to use for the image.
    :param preset_mean: the preset mean of a scaled image.
    :type preset_mean: double
    :param preset_std: the preset standard deviation of a scaled image.
    :type preset_std: double
    :param cmap: the colormap for image visualization
    :type cmap: matplotlib.cm"""

    def __init__(self, vals, axes=None, dimensions=(16,16), transpose=False, order='C', invert=False, scale=False, palette=[], preset_mean=0., preset_std=1., select_image=0, cmap=None):
        self.dimensions = dimensions
        self.transpose = transpose
        self.order = order
        self.invert = invert
        self.scale = scale
        self.palette = palette
        self.preset_mean = preset_mean
        self.preset_std = preset_std
        self.select_image = select_image # This is used when the y vector contains multiple images concatenated.

        self.set_image(self.vals)
        if not self.palette == []: # Can just show the image (self.set_image() took care of setting the palette)
            self.handle = self.axes.imshow(self.vals, interpolation='nearest')
        elif cmap==None: # Use a jet map.
            self.handle = self.axes.imshow(self.vals, cmap=plt.cm.jet, interpolation='nearest') # @UndefinedVariable
        else: # Use the selected map.
            self.handle = self.axes.imshow(self.vals, cmap=cmap, interpolation='nearest') # @UndefinedVariable
        plt.show()

        self.set_image(vals.copy())
        self.handle.set_array(self.vals)

    def set_image(self, vals):
        dim = self.dimensions[0] * self.dimensions[1]
        num_images = np.sqrt(vals[0,].size/dim)
        if num_images > 1 and num_images.is_integer(): # Show a mosaic of images
            num_images = np.int(num_images)
            self.vals = np.zeros((self.dimensions[0]*num_images, self.dimensions[1]*num_images))
            for iR in range(num_images):
                for iC in range(num_images):
                    cur_img_id = iR*num_images + iC
                    cur_img = np.reshape(vals[0,dim*cur_img_id+np.array(range(dim))], self.dimensions, order=self.order)
                    first_row = iR*self.dimensions[0]
                    last_row = (iR+1)*self.dimensions[0]
                    first_col = iC*self.dimensions[1]
                    last_col = (iC+1)*self.dimensions[1]
                    self.vals[first_row:last_row, first_col:last_col] = cur_img

        else:
            self.vals = np.reshape(vals[0,dim*self.select_image+np.array(range(dim))], self.dimensions, order=self.order)
        if self.transpose:
            self.vals = self.vals.T
        # if not self.scale:
        #     self.vals = self.vals
        if self.invert:
            self.vals = -self.vals

        # un-normalizing, for visualisation purposes:
        self.vals = self.vals*self.preset_std + self.preset_mean
        # Clipping the values:
        #self.vals[self.vals < 0] = 0
        #self.vals[self.vals > 255] = 255
        #else:
            #self.vals = 255*(self.vals - self.vals.min())/(self.vals.max() - self.vals.min())
        if not self.palette == []: # applying using an image palette (e.g. if the image has been quantized)
            from PIL import Image
            self.vals = Image.fromarray(self.vals.astype('uint8'))
            self.vals.putpalette(self.palette) # palette is a list, must be loaded before calling this function

class mocap_data_show_vpython(vpython_show):
    """Base class for visualizing motion capture data using visual module."""

    def __init__(self, vals, scene=None, connect=None, radius=0.1):
        vpython_show.__init__(self, vals, scene)
        self.radius = radius
        self.connect = connect
        self.process_values()
        self.draw_edges()
        self.draw_vertices()

    def draw_vertices(self):
        self.spheres = []
        for i in range(self.vals.shape[0]):
            self.spheres.append(visual.sphere(pos=(self.vals[i, 0], self.vals[i, 2], self.vals[i, 1]), radius=self.radius))
        self.scene.visible=True

    def draw_edges(self):
        self.rods = []
        self.line_handle = []
        if not self.connect==None:
            self.I, self.J = np.nonzero(self.connect)
            for i, j in zip(self.I, self.J):
                pos, axis = self.pos_axis(i, j)
                self.rods.append(visual.cylinder(pos=pos, axis=axis, radius=self.radius))

    def modify_vertices(self):
            self.spheres[i].pos = (self.vals[i, 0], self.vals[i, 2], self.vals[i, 1])

    def modify_edges(self):
            for rod, i, j in zip(self.rods, self.I, self.J):
                rod.pos, rod.axis = self.pos_axis(i, j)

    def pos_axis(self, i, j):
        pos = []
        axis = []
        pos.append(self.vals[i, 0])
        axis.append(self.vals[j, 0]-self.vals[i,0])
        pos.append(self.vals[i, 2])
        axis.append(self.vals[j, 2]-self.vals[i,2])
        pos.append(self.vals[i, 1])
        axis.append(self.vals[j, 1]-self.vals[i,1])
        return pos, axis

        self.modify_edges()
        self.modify_vertices()

    def process_values(self):

class mocap_data_show(matplotlib_show):
    """Base class for visualizing motion capture data."""

    def __init__(self, vals, axes=None, connect=None):
            axes = fig.add_subplot(111, projection='3d', aspect='equal')

        self.initialize_axes()
        self.finalize_axes()

        self.points_handle = self.axes.scatter(self.vals[:, 0], self.vals[:, 1], self.vals[:, 2])

            x = []
            y = []
            z = []
                x.append(self.vals[i, 0])
                x.append(self.vals[j, 0])
                x.append(np.NaN)
                y.append(self.vals[i, 1])
                y.append(self.vals[j, 1])
                y.append(np.NaN)
                z.append(self.vals[i, 2])
                z.append(self.vals[j, 2])
                z.append(np.NaN)
            self.line_handle = self.axes.plot(np.array(x), np.array(y), np.array(z), 'b-')

        self.initialize_axes_modify()
        self.finalize_axes_modify()

    def initialize_axes(self, boundary=0.05):
        """Set up the axes with the right limits and scaling."""
        bs = [(self.vals[:, i].max()-self.vals[:, i].min())*boundary for i in xrange(3)]
        self.x_lim = np.array([self.vals[:, 0].min()-bs[0], self.vals[:, 0].max()+bs[0]])
        self.y_lim = np.array([self.vals[:, 1].min()-bs[1], self.vals[:, 1].max()+bs[1]])
        self.z_lim = np.array([self.vals[:, 2].min()-bs[2], self.vals[:, 2].max()+bs[2]])

    def initialize_axes_modify(self):
        self.points_handle.remove()
        self.line_handle[0].remove()

    def finalize_axes(self):
        self.axes.set_xlim(self.x_lim)
        self.axes.set_ylim(self.y_lim)
        self.axes.set_zlim(self.z_lim)
        self.axes.auto_scale_xyz([-1., 1.], [-1., 1.], [-1., 1.])

#        self.axes.set_aspect('equal')
#         self.axes.autoscale(enable=False)

    def finalize_axes_modify(self):

class stick_show(mocap_data_show):
    """Show a three dimensional point cloud as a figure. Connect elements of the figure together using the matrix connect."""
    def __init__(self, vals, connect=None, axes=None):
        mocap_data_show.__init__(self, vals, axes=axes, connect=connect)

        self.vals = self.vals.reshape((3, self.vals.shape[1]/3)).T

class skeleton_show(mocap_data_show):
    """data_show class for visualizing motion capture data encoded as a skeleton with angles."""
    def __init__(self, vals, skel, axes=None, padding=0):
        """data_show class for visualizing motion capture data encoded as a skeleton with angles.
        :param vals: set of modeled angles to use for printing in the axis when it's first created.
        :type vals: np.array
        :param skel: skeleton object that has the parameters of the motion capture skeleton associated with it.
        :type skel: mocap.skeleton object
        :param padding:
        :type int
        """
        self.skel = skel
        self.padding = padding
        connect = skel.connection_matrix()
        """Takes a set of angles and converts them to the x,y,z coordinates in the internal prepresentation of the class, ready for plotting.

        :param vals: the values that are being modelled."""

        if self.padding>0:
            channels = np.zeros((self.vals.shape[0], self.vals.shape[1]+self.padding))
            channels[:, 0:self.vals.shape[0]] = self.vals
        else:
            channels = self.vals
        vals_mat = self.skel.to_xyz(channels.flatten())
        self.vals = np.zeros_like(vals_mat)
        # Flip the Y and Z axes
        self.vals[:, 0] = vals_mat[:, 0].copy()
        self.vals[:, 1] = vals_mat[:, 2].copy()
        self.vals[:, 2] = vals_mat[:, 1].copy()

    def wrap_around(self, lim, connect):
        quot = lim[1] - lim[0]
        self.vals = rem(self.vals, quot)+lim[0]
        nVals = floor(self.vals/quot)
        for i in range(connect.shape[0]):
            for j in find(connect[i, :]):
                if nVals[i] != nVals[j]:
                    connect[i, j] = False
        return connect

def data_play(Y, visualizer, frame_rate=30):
    """Play a data set using the data_show object given.

    :Y: the data set to be visualized.
    :param visualizer: the data show objectwhether to display during optimisation
    :type visualizer: data_show

    Example usage:

    This example loads in the CMU mocap database (http://mocap.cs.cmu.edu) subject number 35 motion number 01. It then plays it using the mocap_show visualize object.

    .. code-block:: python

       data = GPy.util.datasets.cmu_mocap(subject='35', train_motions=['01'])
       Y = data['Y']
       Y[:, 0:3] = 0.   # Make figure walk in place
       visualize = GPy.util.visualize.skeleton_show(Y[0, :], data['skel'])
       GPy.util.visualize.data_play(Y, visualize)

    """

    for y in Y:
        visualizer.modify(y[None, :])
        time.sleep(1./float(frame_rate))

# Copyright (c) 2012-2014, James Hensman

class DTC(LatentFunctionInference):
    """

    NB. It's not recommended to use this function! It's here for historical purposes. 

    """
        self.const_jitter = 1e-6

        assert X_variance is None, "cannot use X_variance with DTC. Try varDTC."

        beta = 1./likelihood.gaussian_variance(Y_metadata)
        if beta.size > 1:
            raise NotImplementedError, "no hetero noise with this implementation of DTC"

        U = Knm
        Uy = np.dot(U.T,Y)

        # Compute A
        LiUTbeta = np.dot(Li, U.T)*np.sqrt(beta)
        A = tdot(LiUTbeta) + np.eye(num_inducing)

        # factor A

        tmp, _ = dtrtrs(L, Uy, lower=1)
        b, _ = dtrtrs(LA, tmp*beta, lower=1)

                       0.5*num_data*output_dim*np.log(beta) + \
                       -0.5*beta*np.sum(np.square(Y)) + \

        dL_dU *= beta

        dL_dR = 0.5*(np.sum(U*np.dot(U,P), 1) - 1./beta + np.sum(np.square(Y), 1) - 2.*np.sum(Uv*Y, 1) + np.sum(np.square(Uv), 1))*beta**2

        grad_dict = {'dL_dKmm': dL_dK, 'dL_dKdiag':np.zeros_like(Knn), 'dL_dKnm':dL_dU.T, 'dL_dthetaL':dL_dthetaL}

class vDTC(object):

    def inference(self, kern, X, X_variance, Z, likelihood, Y, Y_metadata):

        U = Knm

        # Compute A
        A_ = tdot(LiUTbeta)
        trace_term = -0.5*(np.sum(Knn)*beta - np.trace(A_))
        A = A_ + np.eye(num_inducing)

        # factor A

        stop

                       0.5*np.sum(np.square(b)) + \
                       trace_term

        LAL = Li.T.dot(A).dot(Li)
        dL_dK = Kmmi - 0.5*(vvT_P + LAL)

        #dL_dU = vY - np.dot(vvT_P, U.T)
        dL_dU = vY - np.dot(vvT_P - Kmmi, U.T)

        dL_dR = 0.5*(np.sum(U*np.dot(U,P), 1) - 1./beta + np.sum(np.square(Y), 1) - 2.*np.sum(Uv*Y, 1) + np.sum(np.square(Uv), 1) )*beta**2
        dL_dR -=beta*trace_term/num_data

        grad_dict = {'dL_dKmm': dL_dK, 'dL_dKdiag':np.zeros_like(Knn) + -0.5*beta, 'dL_dKnm':dL_dU.T, 'dL_dthetaL':dL_dthetaL}

def get_blocks(A, blocksizes):
    assert (A.shape[0]==A.shape[1]) and len(A.shape)==2, "can;t blockify this non-square matrix"
    N = np.sum(blocksizes)
    assert A.shape[0] == N, "bad blocksizes"
    num_blocks = len(blocksizes)
    B = np.empty(shape=(num_blocks, num_blocks), dtype=np.object)
    count_i = 0
    for Bi, i in enumerate(blocksizes):
        count_j = 0
        for Bj, j in enumerate(blocksizes):
            B[Bi, Bj] = A[count_i:count_i + i, count_j : count_j + j]
            count_j += j
        count_i += i
    return B

    A = np.zeros((5,5))
    B = get_blocks(A,[2,3])
    B[0,0] += 7
    print B

# ## Copyright (c) 2012, GPy authors (see AUTHORS.txt).

def get_shape(x):
    if isinstance(x, numpy.ndarray):
        return x.shape
    return ()

def at_least_one_element(x):
    if isinstance(x, (list, tuple)):
        return x
    return [x]

def flatten_if_needed(x):
    return numpy.atleast_1d(x).flatten()

class GradientChecker(Model):

    def __init__(self, f, df, x0, names=None, *args, **kwargs):
        """
        :param f: Function to check gradient for
        :param df: Gradient of function to check
        :param x0:
            Initial guess for inputs x (if it has a shape (a,b) this will be reflected in the parameter names).
            Can be a list of arrays, if takes a list of arrays. This list will be passed
            to f and df in the same order as given here.
            If only one argument, make sure not to pass a list!!!

        :type x0: [array-like] | array-like | float | int
        :param names:
            Names to print, when performing gradcheck. If a list was passed to x0
            a list of names with the same length is expected.
        :param args: Arguments passed as f(x, *args, **kwargs) and df(x, *args, **kwargs)

        Examples:
        ---------
            from GPy.models import GradientChecker
            N, M, Q = 10, 5, 3

            Sinusoid:

                X = numpy.random.rand(N, Q)
                grad = GradientChecker(numpy.sin,numpy.cos,X,'x')

            Using GPy:

                X, Z = numpy.random.randn(N,Q), numpy.random.randn(M,Q)
                kern = GPy.kern.linear(Q, ARD=True) + GPy.kern.rbf(Q, ARD=True)
                grad = GradientChecker(kern.K,
                                       lambda x: 2*kern.dK_dX(numpy.ones((1,1)), x),
                                       x0 = X.copy(),
                                       names='X')
        """
        Model.__init__(self, 'GradientChecker')
        if isinstance(x0, (list, tuple)) and names is None:
            self.shapes = [get_shape(xi) for xi in x0]
            self.names = ['X{i}'.format(i=i) for i in range(len(x0))]
        elif isinstance(x0, (list, tuple)) and names is not None:
            self.names = names
        elif names is None:
            self.names = ['X']
            self.shapes = [get_shape(x0)]
        else:

        for name, xi in zip(self.names, at_least_one_element(x0)):
            self.__setattr__(name, Param(name, xi))
            self.link_parameter(self.__getattribute__(name))
#         self._param_names = []
#         for name, shape in zip(self.names, self.shapes):
#             self._param_names.extend(map(lambda nameshape: ('_'.join(nameshape)).strip('_'), itertools.izip(itertools.repeat(name), itertools.imap(lambda t: '_'.join(map(str, t)), itertools.product(*map(lambda xi: range(xi), shape))))))
        self.args = args
        self.kwargs = kwargs
        self.f = f
        self.df = df

    def _get_x(self):
        if len(self.names) > 1:
            return [self.__getattribute__(name) for name in self.names] + list(self.args)
        return [self.__getattribute__(self.names[0])] + list(self.args)

        return float(numpy.sum(self.f(*self._get_x(), **self.kwargs)))

        return numpy.atleast_1d(self.df(*self._get_x(), **self.kwargs)).flatten()

    #def _get_params(self):
        #return numpy.atleast_1d(numpy.hstack(map(lambda name: flatten_if_needed(self.__getattribute__(name)), self.names)))

    #def _set_params(self, x):
        #current_index = 0
        #for name, shape in zip(self.names, self.shapes):
            #current_size = numpy.prod(shape)
            #self.__setattr__(name, x[current_index:current_index + current_size].reshape(shape))
            #current_index += current_size

    #def _get_param_names(self):
        #_param_names = []
            #_param_names.extend(map(lambda nameshape: ('_'.join(nameshape)).strip('_'), itertools.izip(itertools.repeat(name), itertools.imap(lambda t: '_'.join(map(str, t)), itertools.product(*map(lambda xi: range(xi), shape))))))
        #return _param_names

'''

'''
from axis_event_controller import BufferedAxisChangedController

class ImshowController(BufferedAxisChangedController):
    def __init__(self, ax, plot_function, plot_limits, resolution=50, update_lim=.8, **kwargs):
        """

        """
        super(ImshowController, self).__init__(ax, plot_function, plot_limits, resolution, update_lim, **kwargs)

    def _init_view(self, ax, X, xmin, xmax, ymin, ymax, **kwargs):
        return ax.imshow(X, extent=(xmin, xmax,
                                    ymin, ymax),
                         vmin=X.min(),
                         vmax=X.max(),
                         **kwargs)

        view.set_data(X)
        view.set_extent((xmin, xmax, ymin, ymax))

class ImAnnotateController(ImshowController):
    def __init__(self, ax, plot_function, plot_limits, resolution=20, update_lim=.99, **kwargs):
        """
        :param text_props: kwargs for pyplot.text(**text_props)
        """
        super(ImAnnotateController, self).__init__(ax, plot_function, plot_limits, resolution, update_lim, **kwargs)

    def _init_view(self, ax, X, xmin, xmax, ymin, ymax, text_props={}, **kwargs):
        view = [super(ImAnnotateController, self)._init_view(ax, X[0], xmin, xmax, ymin, ymax, **kwargs)]
        xoffset, yoffset = self._offsets(xmin, xmax, ymin, ymax)
        xlin = numpy.linspace(xmin, xmax, self.resolution, endpoint=False)
        ylin = numpy.linspace(ymin, ymax, self.resolution, endpoint=False)
        for [i, x], [j, y] in itertools.product(enumerate(xlin), enumerate(ylin[::-1])):
            view.append(ax.text(x + xoffset, y + yoffset, "{}".format(X[1][j, i]), ha='center', va='center', **text_props))
        return view

        super(ImAnnotateController, self).update_view(view[0], X[0], xmin, xmax, ymin, ymax)
        for [[i, x], [j, y]], text in itertools.izip(itertools.product(enumerate(xlin), enumerate(ylin[::-1])), view[1:]):
            text.set_x(x + xoffset)
            text.set_y(y + yoffset)
            text.set_text("{}".format(X[1][j, i]))

    def _offsets(self, xmin, xmax, ymin, ymax):
        return (xmax - xmin) / (2 * self.resolution), (ymax - ymin) / (2 * self.resolution)

def chain_1(df_dg, dg_dx):
    """
    Generic chaining function for first derivative

    .. math::
        \\frac{d(f . g)}{dx} = \\frac{df}{dg} \\frac{dg}{dx}
    """
    return df_dg * dg_dx

def chain_2(d2f_dg2, dg_dx, df_dg, d2g_dx2):
    """
    Generic chaining function for second derivative

    .. math::
        \\frac{d^{2}(f . g)}{dx^{2}} = \\frac{d^{2}f}{dg^{2}}(\\frac{dg}{dx})^{2} + \\frac{df}{dg}\\frac{d^{2}g}{dx^{2}}
    """
    return d2f_dg2*(dg_dx**2) + df_dg*d2g_dx2

def chain_3(d3f_dg3, dg_dx, d2f_dg2, d2g_dx2, df_dg, d3g_dx3):
    """
    Generic chaining function for third derivative

    .. math::
        \\frac{d^{3}(f . g)}{dx^{3}} = \\frac{d^{3}f}{dg^{3}}(\\frac{dg}{dx})^{3} + 3\\frac{d^{2}f}{dg^{2}}\\frac{dg}{dx}\\frac{d^{2}g}{dx^{2}} + \\frac{df}{dg}\\frac{d^{3}g}{dx^{3}}
    """
    return d3f_dg3*(dg_dx**3) + 3*d2f_dg2*dg_dx*d2g_dx2 + df_dg*d3g_dx3

def opt_wrapper(m, **kwargs):
    """
    This function just wraps the optimization procedure of a GPy
    object so that optimize() pickleable (necessary for multiprocessing).
    """
    m.optimize(**kwargs)
    return m.optimization_runs[-1]

def linear_grid(D, n = 100, min_max = (-100, 100)):
    """
    Creates a D-dimensional grid of n linearly spaced points

    :param D: dimension of the grid
    :param n: number of points
    :param min_max: (min, max) list

    """

    g = np.linspace(min_max[0], min_max[1], n)
    G = np.ones((n, D))

    return G*g[:,None]

def kmm_init(X, m = 10):
    """
    This is the same initialization algorithm that is used
    in Kmeans++. It's quite simple and very useful to initialize
    the locations of the inducing points in sparse GPs.

    :param X: data
    :param m: number of inducing points

    """

    # compute the distances
    XXT = np.dot(X, X.T)
    D = (-2.*XXT + np.diag(XXT)[:,np.newaxis] + np.diag(XXT)[np.newaxis,:])

    # select the first point
    s = np.random.permutation(X.shape[0])[0]
    inducing = [s]
    prob = D[s]/D[s].sum()

    for z in range(m-1):
        s = np.random.multinomial(1, prob.flatten()).argmax()
        inducing.append(s)
        prob = D[s]/D[s].sum()

    inducing = np.array(inducing)
    return X[inducing]

### make a parameter to its corresponding array:
def param_to_array(*param):
    """
Convert an arbitrary number of parameters to :class:ndarray class objects. This is for
converting parameter objects to numpy arrays, when using scipy.weave.inline routine.
In scipy.weave.blitz there is no automatic array detection (even when the array inherits
from :class:ndarray)"""
    import warnings
    warnings.warn("Please use param.values, as this function will be deprecated in the next release.", DeprecationWarning)
    assert len(param) > 0, "At least one parameter needed"
    if len(param) == 1:
        return param[0].view(np.ndarray)
    return [x.view(np.ndarray) for x in param]

np = numpy
from parameter_core import Parameterizable, adjust_name_for_printing, Pickleable
from observable_array import ObsAr

###### printing
__constraints_name__ = "Constraint"
__index_name__ = "Index"
__tie_name__ = "Tied to"
__priors_name__ = "Prior"
__precision__ = numpy.get_printoptions()['precision'] # numpy printing precision used, sublassing numpy ndarray after all
__print_threshold__ = 5
######

class Param(Parameterizable, ObsAr):
    """
    Parameter object for GPy models.

    :param str name:           name of the parameter to be printed
    :param input_array:        array which this parameter handles
    :type input_array:         numpy.ndarray
    :param default_constraint: The default constraint for this parameter
    :type default_constraint:

    You can add/remove constraints by calling constrain on the parameter itself, e.g:

        - self[:,1].constrain_positive()
        - self[0].tie_to(other)
        - self.untie()
        - self[:3,:].unconstrain()
        - self[1].fix()

    Fixing parameters will fix them to the value they are right now. If you change
    the fixed value, it will be fixed to the new value!

    See :py:class:`GPy.core.parameterized.Parameterized` for more details on constraining etc.

    """
    __array_priority__ = -1 # Never give back Param
    _fixes_ = None
    parameters = []
    def __new__(cls, name, input_array, default_constraint=None):
        obj = numpy.atleast_1d(super(Param, cls).__new__(cls, input_array=input_array))
        obj._current_slice_ = (slice(obj.shape[0]),)
        obj._realshape_ = obj.shape
        obj._realsize_ = obj.size
        obj._realndim_ = obj.ndim
        obj._original_ = obj
        return obj

    def __init__(self, name, input_array, default_constraint=None, *a, **kw):
        super(Param, self).__init__(name=name, default_constraint=default_constraint, *a, **kw)

    def build_pydot(self,G):
        import pydot
        node = pydot.Node(id(self), shape='trapezium', label=self.name)#, fontcolor='white', color='white')

        super(Param, self).__array_finalize__(obj)
        self._parent_ = getattr(obj, '_parent_', None)
        self._parent_index_ = getattr(obj, '_parent_index_', None)
        self._default_constraint_ = getattr(obj, '_default_constraint_', None)
        self._current_slice_ = getattr(obj, '_current_slice_', None)
        self._realshape_ = getattr(obj, '_realshape_', None)
        self._realsize_ = getattr(obj, '_realsize_', None)
        self._realndim_ = getattr(obj, '_realndim_', None)
        self._original_ = getattr(obj, '_original_', None)
        self._name = getattr(obj, '_name', None)
        self._gradient_array_ = getattr(obj, '_gradient_array_', None)
        self.constraints = getattr(obj, 'constraints', None)
        self.priors = getattr(obj, 'priors', None)

    @property
        """
        As we are a leaf, this just returns self
        """

    @property
        """
        Return self as numpy array view
        """

    @property
        """
        Return a view on the gradient, which is in the same shape as this parameter is.
        Note: this is not the real gradient array, it is just a view on it.

        To work on the real gradient array use: self.full_gradient
        """
        if getattr(self, '_gradient_array_', None) is None:
            self._gradient_array_ = numpy.empty(self._realshape_, dtype=numpy.float64)
        return self._gradient_array_#[self._current_slice_]

    # Array operations -> done
    def __getitem__(self, s, *args, **kwargs):
        if not isinstance(s, tuple):
            s = (s,)
        #if not reduce(lambda a, b: a or numpy.any(b is Ellipsis), s, False) and len(s) <= self.ndim:
        #    s += (Ellipsis,)
        new_arr = super(Param, self).__getitem__(s, *args, **kwargs)
        try:
            new_arr._current_slice_ = s
            new_arr._gradient_array_ = self.gradient[s]
            new_arr._original_ = self._original_
        except AttributeError: pass  # returning 0d array or float, double etc
        return new_arr

    def _raveled_index(self, slice_index=None):
        # return an index array on the raveled array, which is formed by the current_slice
        # of this object
        extended_realshape = numpy.cumprod((1,) + self._realshape_[:0:-1])[::-1]
        ind = self._indices(slice_index)
        if ind.ndim < 2: ind = ind[:, None]
        return numpy.asarray(numpy.apply_along_axis(lambda x: numpy.sum(extended_realshape * x), 1, ind), dtype=int)

    def _raveled_index_for(self, obj):
        return self._raveled_index()

    # Constrainable
        if not self._has_fixes(): self._fixes_ = numpy.ones(self._realsize_, dtype=bool)

    # Convenience
    @property
        from transformations import __fixed__
        return self.constraints[__fixed__].size == self.size

        return self._original_

    # Pickling and copying
        return Parameterizable.copy(self, which=self)

        s = self.__new__(self.__class__, name=self.name, input_array=self.view(numpy.ndarray).copy())
        return s
        """

        1: pass through to parent, if present
        """

    # Printing -> done
    @property
        if self.size <= 1:
            return [str(self.view(numpy.ndarray)[0])]
        else: return [str(self.shape)]
        # this is just overwrighting the parameterized calls to parameter names, in order to maintain OOP
        if adjust_for_printing:
            return [adjust_name_for_printing(self.name)]
        return [self.name]
    @property
        return [self]
    @property
        return [self.shape]
    @property
        return 0
    @property
        return [' '.join(map(lambda c: str(c[0]) if c[1].size == self._realsize_ else "{" + str(c[0]) + "}", self.constraints.iteritems()))]
    @property
        return [' '.join(map(lambda c: str(c[0]) if c[1].size == self._realsize_ else "{" + str(c[0]) + "}", self.priors.iteritems()))]
    @property
        return ['']
    def _ties_for(self, ravi):
        return [['N/A']]*ravi.size
    def __repr__(self, *args, **kwargs):
        name = "\033[1m{x:s}\033[0;0m:\n".format(
                            x=self.hierarchy_name())
        return name + super(Param, self).__repr__(*args, **kwargs)
    def _indices(self, slice_index=None):
        # get a int-array containing all indices in the first axis.
        if slice_index is None:
            slice_index = self._current_slice_
        try:
            indices = np.indices(self._realshape_, dtype=int)
            indices = indices[(slice(None),)+slice_index]
            indices = np.rollaxis(indices, 0, indices.ndim).reshape(-1,self._realndim_)
            #print indices_
            #if not np.all(indices==indices__):
            #    import ipdb; ipdb.set_trace()
        except:
            indices = np.rollaxis(indices, 0, indices.ndim)
        return indices
    def _max_len_names(self, gen, header):
        gen = map(lambda x: " ".join(map(str, x)), gen)
        return reduce(lambda a, b:max(a, len(b)), gen, len(header))
    def _max_len_values(self):
        return reduce(lambda a, b:max(a, len("{x:=.{0}g}".format(__precision__, x=b))), self.flat, len(self.hierarchy_name()))
    def _max_len_index(self, ind):
        return reduce(lambda a, b:max(a, len(str(b))), ind, len(__index_name__))
        # short string to print
        name = self.hierarchy_name()
        if self._realsize_ < 2:
            return name
        ind = self._indices()
        if ind.size > 4: indstr = ','.join(map(str, ind[:2])) + "..." + ','.join(map(str, ind[-2:]))
        else: indstr = ','.join(map(str, ind))
        return name + '[' + indstr + ']'

    def _repr_html_(self, constr_matrix=None, indices=None, prirs=None, ties=None):
        """Representation of the parameter in html for notebook display."""
        filter_ = self._current_slice_
        vals = self.flat
        if indices is None: indices = self._indices(filter_)
        ravi = self._raveled_index(filter_)
        if constr_matrix is None: constr_matrix = self.constraints.properties_for(ravi)
        if prirs is None: prirs = self.priors.properties_for(ravi)
        if ties is None: ties = self._ties_for(ravi)
        ties = [' '.join(map(lambda x: x, t)) for t in ties]
        header_format = """
<tr>
  <td><b>{i}</b></td>
  <td><b>{x}</b></td>
  <td><b>{c}</b></td>
  <td><b>{p}</b></td>
  <td><b>{t}</b></td>
</tr>"""
        header = header_format.format(x=self.hierarchy_name(), c=__constraints_name__, i=__index_name__, t=__tie_name__, p=__priors_name__)  # nice header for printing
        if not ties: ties = itertools.cycle([''])
        return "\n".join(['<table>'] + [header] + ["<tr><td>{i}</td><td align=\"right\">{x}</td><td>{c}</td><td>{p}</td><td>{t}</td></tr>".format(x=x, c=" ".join(map(str, c)), p=" ".join(map(str, p)), t=(t or ''), i=i) for i, x, c, t, p in itertools.izip(indices, vals, constr_matrix, ties, prirs)] + ["</table>"])  

    def __str__(self, constr_matrix=None, indices=None, prirs=None, ties=None, lc=None, lx=None, li=None, lp=None, lt=None, only_name=False):
        if lc is None: lc = self._max_len_names(constr_matrix, __constraints_name__)
        if lx is None: lx = self._max_len_values()
        if li is None: li = self._max_len_index(indices)
        if lt is None: lt = self._max_len_names(ties, __tie_name__)
        if lp is None: lp = self._max_len_names(prirs, __tie_name__)
        sep = '-'
        header_format = "  {i:{5}^{2}s}  |  \033[1m{x:{5}^{1}s}\033[0;0m  |  {c:{5}^{0}s}  |  {p:{5}^{4}s}  |  {t:{5}^{3}s}"
        if only_name: header = header_format.format(lc, lx, li, lt, lp, ' ', x=self.hierarchy_name(), c=sep*lc, i=sep*li, t=sep*lt, p=sep*lp)  # nice header for printing
        else: header = header_format.format(lc, lx, li, lt, lp, ' ', x=self.hierarchy_name(), c=__constraints_name__, i=__index_name__, t=__tie_name__, p=__priors_name__)  # nice header for printing
        return "\n".join([header] + ["  {i!s:^{3}s}  |  {x: >{1}.{2}g}  |  {c:^{0}s}  |  {p:^{5}s}  |  {t:^{4}s}  ".format(lc, lx, __precision__, li, lt, lp, x=x, c=" ".join(map(str, c)), p=" ".join(map(str, p)), t=(t or ''), i=i) for i, x, c, t, p in itertools.izip(indices, vals, constr_matrix, ties, prirs)])  # return all the constraints with right indices
        # except: return super(Param, self).__str__()

class ParamConcatenation(object):
    def __init__(self, params):
        """
        Parameter concatenation for convenience of printing regular expression matched arrays
        you can index this concatenation as if it was the flattened concatenation
        of all the parameters it contains, same for setting parameters (Broadcasting enabled).

        See :py:class:`GPy.core.parameter.Param` for more details on constraining.
        """
        # self.params = params
        from lists_and_dicts import ArrayList
        self.params = ArrayList([])
        for p in params:
            for p in p.flattened_parameters:
                if p not in self.params:
                    self.params.append(p)
        self._param_sizes = [p.size for p in self.params]
        startstops = numpy.cumsum([0] + self._param_sizes)
        self._param_slices_ = [slice(start, stop) for start,stop in zip(startstops, startstops[1:])]

        parents = dict()
        for p in self.params:
            if p.has_parent():
                parent = p._parent_
                level = 0
                while parent is not None:
                    if parent in parents:
                        parents[parent] = max(level, parents[parent])
                    else:
                        parents[parent] = level
                    level += 1
                    parent = parent._parent_
        import operator
        self.parents = map(lambda x: x[0], sorted(parents.iteritems(), key=operator.itemgetter(1)))
    # Get/set items, enable broadcasting
        ind = numpy.zeros(sum(self._param_sizes), dtype=bool); ind[s] = True;
        params = [p.param_array.flat[ind[ps]] for p,ps in zip(self.params, self._param_slices_) if numpy.any(p.param_array.flat[ind[ps]])]
        if len(params)==1: return params[0]
        return ParamConcatenation(params)
    def __setitem__(self, s, val, update=True):
        if isinstance(val, ParamConcatenation):
            val = val.values()
        vals = self.values(); vals[s] = val
        for p, ps in zip(self.params, self._param_slices_):
            p.flat[ind[ps]] = vals[ps]
        if update:
            self.update_all_params()
        return numpy.hstack([p.param_array.flat for p in self.params])
    # parameter operations:
    def update_all_params(self):
        for par in self.parents:
            par.notify_observers()

    def constrain(self, constraint, warning=True):
        [param.constrain(constraint, trigger_parent=False) for param in self.params]
        self.update_all_params()
    constrain.__doc__ = Param.constrain.__doc__

    def constrain_positive(self, warning=True):
        [param.constrain_positive(warning, trigger_parent=False) for param in self.params]
    constrain_positive.__doc__ = Param.constrain_positive.__doc__

        [param.constrain_fixed(value, warning, trigger_parent) for param in self.params]
    constrain_fixed.__doc__ = Param.constrain_fixed.__doc__

    def constrain_negative(self, warning=True):
        [param.constrain_negative(warning, trigger_parent=False) for param in self.params]
    constrain_negative.__doc__ = Param.constrain_negative.__doc__

    def constrain_bounded(self, lower, upper, warning=True):
        [param.constrain_bounded(lower, upper, warning, trigger_parent=False) for param in self.params]
    constrain_bounded.__doc__ = Param.constrain_bounded.__doc__

    def unconstrain(self, *constraints):
        [param.unconstrain(*constraints) for param in self.params]
    unconstrain.__doc__ = Param.unconstrain.__doc__

        [param.unconstrain_negative() for param in self.params]
    unconstrain_negative.__doc__ = Param.unconstrain_negative.__doc__

        [param.unconstrain_positive() for param in self.params]
    unconstrain_positive.__doc__ = Param.unconstrain_positive.__doc__

        [param.unconstrain_fixed() for param in self.params]
    unconstrain_fixed.__doc__ = Param.unconstrain_fixed.__doc__

        [param.unconstrain_bounded(lower, upper) for param in self.params]
    unconstrain_bounded.__doc__ = Param.unconstrain_bounded.__doc__

    def untie(self, *ties):
        [param.untie(*ties) for param in self.params]

    def checkgrad(self, verbose=0, step=1e-6, tolerance=1e-3):
        return self.params[0]._highest_parent_._checkgrad(self, verbose, step, tolerance)
    #checkgrad.__doc__ = Gradcheckable.checkgrad.__doc__

    __lt__ = lambda self, val: self.values() < val
    __le__ = lambda self, val: self.values() <= val
    __eq__ = lambda self, val: self.values() == val
    __ne__ = lambda self, val: self.values() != val
    __gt__ = lambda self, val: self.values() > val
    __ge__ = lambda self, val: self.values() >= val
        def f(p):
            ind = p._raveled_index()
            return p.constraints.properties_for(ind), p._ties_for(ind), p.priors.properties_for(ind)
        params = self.params
        constr_matrices, ties_matrices, prior_matrices = zip(*map(f, params))
        indices = [p._indices() for p in params]
        lc = max([p._max_len_names(cm, __constraints_name__) for p, cm in itertools.izip(params, constr_matrices)])
        lx = max([p._max_len_values() for p in params])
        li = max([p._max_len_index(i) for p, i in itertools.izip(params, indices)])
        lt = max([p._max_len_names(tm, __tie_name__) for p, tm in itertools.izip(params, ties_matrices)])
        lp = max([p._max_len_names(pm, __constraints_name__) for p, pm in itertools.izip(params, prior_matrices)])
        strings = []
        start = True
        for p, cm, i, tm, pm in itertools.izip(params,constr_matrices,indices,ties_matrices,prior_matrices):
            strings.append(p.__str__(constr_matrix=cm, indices=i, prirs=pm, ties=tm, lc=lc, lx=lx, li=li, lp=lp, lt=lt, only_name=(1-start)))
            start = False
        return "\n".join(strings)
        return "\n".join(map(repr,self.params))

        self[:] = np.ndarray.__ilshift__(self.values(), *args, **kwargs)

        self[:] = np.ndarray.__irshift__(self.values(), *args, **kwargs)

        self[:] = np.ndarray.__ixor__(self.values(), *args, **kwargs)

        self[:] = np.ndarray.__ipow__(self.values(), *args, **kwargs)

        self[:] = np.ndarray.__ifloordiv__(self.values(), *args, **kwargs)

        self[:] = np.ndarray.__isub__(self.values(), *args, **kwargs)

        self[:] = np.ndarray.__ior__(self.values(), *args, **kwargs)

        self[:] = np.ndarray.__itruediv__(self.values(), *args, **kwargs)

        self[:] = np.ndarray.__idiv__(self.values(), *args, **kwargs)

        self[:] = np.ndarray.__iand__(self.values(), *args, **kwargs)

        self[:] = np.ndarray.__imod__(self.values(), *args, **kwargs)

        self[:] = np.ndarray.__iadd__(self.values(), *args, **kwargs)

        self[:] = np.ndarray.__imul__(self.values(), *args, **kwargs)

import GPy
from GPy.util import datasets
try:
    import matplotlib.pyplot as plt
except:
    pass

def student_t_approx(optimize=True, plot=True):
    """
    Example of regressing with a student t likelihood using Laplace
    """
    real_std = 0.1
    #Start a function, any function
    X = np.linspace(0.0, np.pi*2, 100)[:, None]
    Y = np.sin(X) + np.random.randn(*X.shape)*real_std
    Y = Y/Y.max()
    Yc = Y.copy()

    X_full = np.linspace(0.0, np.pi*2, 500)[:, None]
    Y_full = np.sin(X_full)
    Y_full = Y_full/Y_full.max()

    #Slightly noisy data
    Yc[75:80] += 1

    #Very noisy data
    #Yc[10] += 100
    #Yc[25] += 10
    #Yc[23] += 10
    #Yc[26] += 1000
    #Yc[24] += 10
    #Yc = Yc/Yc.max()

    #Add student t random noise to datapoints
    deg_free = 1
    print "Real noise: ", real_std
    initial_var_guess = 0.5
    edited_real_sd = initial_var_guess

    # Kernel object
    kernel1 = GPy.kern.RBF(X.shape[1]) + GPy.kern.White(X.shape[1])
    kernel2 = GPy.kern.RBF(X.shape[1]) + GPy.kern.White(X.shape[1])
    kernel3 = GPy.kern.RBF(X.shape[1]) + GPy.kern.White(X.shape[1])
    kernel4 = GPy.kern.RBF(X.shape[1]) + GPy.kern.White(X.shape[1])

    #Gaussian GP model on clean data
    m1 = GPy.models.GPRegression(X, Y.copy(), kernel=kernel1)
    # optimize
    m1['.*white'].constrain_fixed(1e-5)
    m1.randomize()

    #Gaussian GP model on corrupt data
    m2 = GPy.models.GPRegression(X, Yc.copy(), kernel=kernel2)
    m2['.*white'].constrain_fixed(1e-5)
    m2.randomize()

    #Student t GP model on clean data
    t_distribution = GPy.likelihoods.StudentT(deg_free=deg_free, sigma2=edited_real_sd)
    m3 = GPy.core.GP(X, Y.copy(), kernel3, likelihood=t_distribution, inference_method=laplace_inf)
    m3['.*t_scale2'].constrain_bounded(1e-6, 10.)
    m3['.*white'].constrain_fixed(1e-5)
    m3.randomize()

    #Student t GP model on corrupt data
    m4 = GPy.core.GP(X, Yc.copy(), kernel4, likelihood=t_distribution, inference_method=laplace_inf)
    m4['.*t_scale2'].constrain_bounded(1e-6, 10.)
    m4['.*white'].constrain_fixed(1e-5)
    m4.randomize()
    print m4
    debug=True
    if debug:
        m4.optimize(messages=1)
        pb.plot(m4.X, m4.inference_method.f_hat)
        pb.plot(m4.X, m4.Y, 'rx')
        m4.plot()
        print m4
        return m4

        optimizer='scg'
        print "Clean Gaussian"
        m1.optimize(optimizer, messages=1)
        print "Corrupt Gaussian"
        m2.optimize(optimizer, messages=1)
        print "Clean student t"
        m3.optimize(optimizer, messages=1)
        print "Corrupt student t"
        m4.optimize(optimizer, messages=1)

    if plot:
        plt.figure(1)
        plt.suptitle('Gaussian likelihood')
        ax = plt.subplot(211)
        m1.plot(ax=ax)
        plt.plot(X_full, Y_full)
        plt.ylim(-1.5, 1.5)
        plt.title('Gaussian clean')

        ax = plt.subplot(212)
        m2.plot(ax=ax)
        plt.title('Gaussian corrupt')

        plt.figure(2)
        plt.suptitle('Student-t likelihood')
        m3.plot(ax=ax)
        plt.title('Student-t rasm clean')

        m4.plot(ax=ax)
        plt.title('Student-t rasm corrupt')

    return m1, m2, m3, m4

def boston_example(optimize=True, plot=True):
    raise NotImplementedError("Needs updating")
    import sklearn
    from sklearn.cross_validation import KFold
    optimizer='bfgs'
    messages=0
    data = datasets.boston_housing()
    degrees_freedoms = [3, 5, 8, 10]
    X = data['X'].copy()
    Y = data['Y'].copy()
    X = X-X.mean(axis=0)
    X = X/X.std(axis=0)
    Y = Y-Y.mean()
    Y = Y/Y.std()
    num_folds = 10
    kf = KFold(len(Y), n_folds=num_folds, indices=True)
    num_models = len(degrees_freedoms) + 3 #3 for baseline, gaussian, gaussian laplace approx
    score_folds = np.zeros((num_models, num_folds))
    pred_density = score_folds.copy()

    def rmse(Y, Ystar):
        return np.sqrt(np.mean((Y-Ystar)**2))

    for n, (train, test) in enumerate(kf):
        X_train, X_test, Y_train, Y_test = X[train], X[test], Y[train], Y[test]
        print "Fold {}".format(n)

        noise = 1e-1 #np.exp(-2)
        rbf_len = 0.5
        data_axis_plot = 4
        kernelstu = GPy.kern.RBF(X.shape[1]) + GPy.kern.white(X.shape[1]) + GPy.kern.bias(X.shape[1])
        kernelgp = GPy.kern.RBF(X.shape[1]) + GPy.kern.white(X.shape[1]) + GPy.kern.bias(X.shape[1])

        #Baseline
        score_folds[0, n] = rmse(Y_test, np.mean(Y_train))

        #Gaussian GP
        print "Gauss GP"
        mgp = GPy.models.GPRegression(X_train.copy(), Y_train.copy(), kernel=kernelgp.copy())
        mgp.constrain_fixed('.*white', 1e-5)
        mgp['.*len'] = rbf_len
        mgp['.*noise'] = noise
        print mgp
            mgp.optimize(optimizer=optimizer, messages=messages)
        Y_test_pred = mgp.predict(X_test)
        score_folds[1, n] = rmse(Y_test, Y_test_pred[0])
        pred_density[1, n] = np.mean(mgp.log_predictive_density(X_test, Y_test))
        print mgp
        print pred_density

        print "Gaussian Laplace GP"
        N, D = Y_train.shape
        g_distribution = GPy.likelihoods.noise_model_constructors.gaussian(variance=noise, N=N, D=D)
        g_likelihood = GPy.likelihoods.Laplace(Y_train.copy(), g_distribution)
        mg = GPy.models.GPRegression(X_train.copy(), Y_train.copy(), kernel=kernelstu.copy(), likelihood=g_likelihood)
        mg.constrain_positive('noise_variance')
        mg.constrain_fixed('.*white', 1e-5)
        mg['rbf_len'] = rbf_len
        mg['noise'] = noise
        print mg
            mg.optimize(optimizer=optimizer, messages=messages)
        Y_test_pred = mg.predict(X_test)
        score_folds[2, n] = rmse(Y_test, Y_test_pred[0])
        pred_density[2, n] = np.mean(mg.log_predictive_density(X_test, Y_test))
        print mg

        for stu_num, df in enumerate(degrees_freedoms):
            #Student T
            print "Student-T GP {}df".format(df)
            t_distribution = GPy.likelihoods.noise_model_constructors.student_t(deg_free=df, sigma2=noise)
            stu_t_likelihood = GPy.likelihoods.Laplace(Y_train.copy(), t_distribution)
            mstu_t = GPy.models.GPRegression(X_train.copy(), Y_train.copy(), kernel=kernelstu.copy(), likelihood=stu_t_likelihood)
            mstu_t.constrain_fixed('.*white', 1e-5)
            mstu_t.constrain_bounded('.*t_scale2', 0.0001, 1000)
            mstu_t['rbf_len'] = rbf_len
            mstu_t['.*t_scale2'] = noise
            print mstu_t
            if optimize:
                mstu_t.optimize(optimizer=optimizer, messages=messages)
            Y_test_pred = mstu_t.predict(X_test)
            score_folds[3+stu_num, n] = rmse(Y_test, Y_test_pred[0])
            pred_density[3+stu_num, n] = np.mean(mstu_t.log_predictive_density(X_test, Y_test))
            print pred_density

    if plot:
        plt.scatter(X_test[:, data_axis_plot], Y_test_pred[0])
        plt.scatter(X_test[:, data_axis_plot], Y_test, c='r', marker='x')
        plt.title('GP gauss')

        plt.title('Lap gauss')

        plt.title('Stu t {}df'.format(df))

    print "Average scores: {}".format(np.mean(score_folds, 1))
    print "Average pred density: {}".format(np.mean(pred_density, 1))

    if plot:
        #Plotting
        stu_t_legends = ['Student T, df={}'.format(df) for df in degrees_freedoms]
        legends = ['Baseline', 'Gaussian', 'Laplace Approx Gaussian'] + stu_t_legends

        #Plot boxplots for RMSE density
        fig = plt.figure()
        ax=fig.add_subplot(111)
        plt.title('RMSE')
        bp = ax.boxplot(score_folds.T, notch=0, sym='+', vert=1, whis=1.5)
        plt.setp(bp['boxes'], color='black')
        plt.setp(bp['whiskers'], color='black')
        plt.setp(bp['fliers'], color='red', marker='+')
        xtickNames = plt.setp(ax, xticklabels=legends)
        plt.setp(xtickNames, rotation=45, fontsize=8)
        ax.set_ylabel('RMSE')
        ax.set_xlabel('Distribution')
        #Make grid and put it below boxes
        ax.yaxis.grid(True, linestyle='-', which='major', color='lightgrey',
                alpha=0.5)
        ax.set_axisbelow(True)

        #Plot boxplots for predictive density
        plt.title('Predictive density')
        bp = ax.boxplot(pred_density[1:,:].T, notch=0, sym='+', vert=1, whis=1.5)
        xtickNames = plt.setp(ax, xticklabels=legends[1:])
        ax.set_ylabel('Mean Log probability P(Y*|Y)')
    return mstu_t

#def precipitation_example():
    #import sklearn
    #from sklearn.cross_validation import KFold
    #data = datasets.boston_housing()
    #X = data['X'].copy()
    #Y = data['Y'].copy()
    #X = X-X.mean(axis=0)
    #X = X/X.std(axis=0)
    #Y = Y-Y.mean()
    #Y = Y/Y.std()
    #import ipdb; ipdb.set_trace()  # XXX BREAKPOINT
    #num_folds = 10
    #kf = KFold(len(Y), n_folds=num_folds, indices=True)
    #score_folds = np.zeros((4, num_folds))
    #def rmse(Y, Ystar):
        #return np.sqrt(np.mean((Y-Ystar)**2))
    ##for train, test in kf:
    #for n, (train, test) in enumerate(kf):
        #X_train, X_test, Y_train, Y_test = X[train], X[test], Y[train], Y[test]
        #print "Fold {}".format(n)

# Copyright (c) 2013, 2014 GPy authors (see AUTHORS.txt).

from ..core.mapping import Bijective_mapping

class Linear(Bijective_mapping):
    """
    Mapping based on a linear model.

    .. math::

       f(\mathbf{x}*) = \mathbf{W}\mathbf{x}^* + \mathbf{b}

    """

    def __init__(self, input_dim=1, output_dim=1, name='linear'):
        Bijective_mapping.__init__(self, input_dim=input_dim, output_dim=output_dim, name=name)
        self.W = Param('W',np.array((self.input_dim, self.output_dim)))
        self.bias = Param('bias',np.array(self.output_dim))
        self.link_parameters(self.W, self.bias)

        return np.dot(X,self.W) + self.bias

        V = np.linalg.solve(np.dot(self.W.T, self.W), W.T)
        return np.dot(f-self.bias, V)  

        df_dW = (dL_df[:, :, None]*X[:, None, :]).sum(0).T
        df_dbias = (dL_df.sum(0))
        return np.hstack((df_dW.flatten(), df_dbias))

    def dL_dX(self, partial, X):
        """The gradient of L with respect to the inputs to the mapping, where L is a function that is dependent on the output of the mapping, f."""
        return (partial[:, None, :]*self.W[None, :, :]).sum(2)

import sys
from ...core.parameterization.parameterized import Parameterized
from kernel_slice_operations import KernCallsViaSlicerMeta

class Kern(Parameterized):
    # This adds input slice support. The rather ugly code for slicing can be
    # found in kernel_slice_operations
    __metaclass__ = KernCallsViaSlicerMeta
    _support_GPU=False
    def __init__(self, input_dim, active_dims, name, useGPU=False, *a, **kw):
        """
        The base class for a kernel: a positive definite function
        which forms of a covariance function (kernel).

        input_dim:

            is the number of dimensions to work on. Make sure to give the
            tight dimensionality of inputs.
            You most likely want this to be the integer telling the number of
            input dimensions of the kernel.
            If this is not an integer (!) we will work on the whole input matrix X,
            and not check whether dimensions match or not (!).

        active_dims:

            is the active_dimensions of inputs X we will work on.
            All kernels will get sliced Xes as inputs, if active_dims is not None
            Only positive integers are allowed in active_dims!
            if active_dims is None, slicing is switched off and all X will be passed through as given.

        :param int input_dim: the number of input dimensions to the function
        :param array-like|None active_dims: list of indices on which dimensions this kernel works on, or none if no slicing

        Do not instantiate.
        """
        super(Kern, self).__init__(name=name, *a, **kw)
        self.input_dim = int(input_dim)

        if active_dims is None:
            active_dims = np.arange(input_dim)

        self.active_dims = np.atleast_1d(active_dims).astype(int)

        assert self.active_dims.size == self.input_dim, "input_dim={} does not match len(active_dim)={}, active_dims={}".format(self.input_dim, self.active_dims.size, self.active_dims)

        self._sliced_X = 0
        self.useGPU = self._support_GPU and useGPU
        self._return_psi2_n_flag = ObsAr(np.zeros(1)).astype(bool)

    @property
    def return_psi2_n(self):
        """
        Flag whether to pass back psi2 as NxMxM or MxM, by summing out N.
        """
        return self._return_psi2_n_flag[0]
    @return_psi2_n.setter
    def return_psi2_n(self, val):
        def visit(self):
            if isinstance(self, Kern):
                self._return_psi2_n_flag[0]=val
        self.traverse(visit)

    @Cache_this(limit=20)
    def _slice_X(self, X):
        return X[:, self.active_dims]

        """
        Compute the kernel function.

        :param X: the first set of inputs to the kernel
        :param X2: (optional) the second set of arguments to the kernel. If X2
                   is None, this is passed throgh to the 'part' object, which
                   handLes this as X2 == X.
        """

        """ update the gradients of all parameters when using only the diagonal elements of the covariance matrix"""

    def update_gradients_full(self, dL_dK, X, X2):
        """Set the gradients of all parameters when doing full (N) inference."""

        """
        Set the gradients of all parameters when doing inference with
        uncertain inputs, using expectations of the kernel.

        The esential maths is

        dL_d{theta_i} = dL_dpsi0 * dpsi0_d{theta_i} +
                        dL_dpsi1 * dpsi1_d{theta_i} +
                        dL_dpsi2 * dpsi2_d{theta_i}
        """

        """
        Returns the derivative of the objective wrt Z, using the chain rule
        through the expectation variables.
        """

        """
        Compute the gradients wrt the parameters of the variational
        distruibution q(X), chain-ruling via the expectations of the kernel
        """

    def plot(self, x=None, fignum=None, ax=None, title=None, plot_limits=None, resolution=None, **mpl_kwargs):
        """
        plot this kernel.
        :param x: the value to use for the other kernel argument (kernels are a function of two variables!)
        :param fignum: figure number of the plot
        :param ax: matplotlib axis to plot on
        :param title: the matplotlib title
        :param plot_limits: the range over which to plot the kernel
        :resolution: the resolution of the lines used in plotting
        :mpl_kwargs avalid keyword arguments to pass through to matplotlib (e.g. lw=7)
        """
        from ...plotting.matplot_dep import kernel_plots
        kernel_plots.plot(self, x, fignum, ax, title, plot_limits, resolution, **mpl_kwargs)

    def plot_ARD(self, *args, **kw):
        """
        See :class:`~GPy.plotting.matplot_dep.kernel_plots`
        """
        import sys
        return kernel_plots.plot_ARD(self,*args,**kw)

        """
        Returns the sensitivity for each dimension of this kernel.
        """
        return np.zeros(self.input_dim)

    def __add__(self, other):
        """ Overloading of the '+' operator. for more control, see self.add """
        return self.add(other)

    def __iadd__(self, other):

    def add(self, other, name='add'):
        """
        Add another kernel to this one.

        :param other: the other kernel to be added
        :type other: GPy.kern

        """
        assert isinstance(other, Kern), "only kernels can be added to kernels..."
        from add import Add
        return Add([self, other], name=name)

    def __mul__(self, other):
        """ Here we overload the '*' operator. See self.prod for more information"""
        return self.prod(other)

    def __imul__(self, other):

    def __pow__(self, other):
        """
        Shortcut for tensor `prod`.
        """
        assert np.all(self.active_dims == range(self.input_dim)), "Can only use kernels, which have their input_dims defined from 0"
        assert np.all(other.active_dims == range(other.input_dim)), "Can only use kernels, which have their input_dims defined from 0"
        other.active_dims += self.input_dim

    def prod(self, other, name='mul'):
        """
        Multiply two kernels (either on the same space, or on the tensor
        product of the input space).

        :param tensor: whether or not to use the tensor space (default is false).
        :type tensor: bool

        """
        assert isinstance(other, Kern), "only kernels can be multiplied to kernels..."
        from prod import Prod
        #kernels = []
        #if isinstance(self, Prod): kernels.extend(self.parameters)
        #else: kernels.append(self)
        #if isinstance(other, Prod): kernels.extend(other.parameters)
        #else: kernels.append(other)
        return Prod([self, other], name)

    def _check_input_dim(self, X):
        assert X.shape[1] == self.input_dim, "{} did not specify active_dims and X has wrong shape: X_dim={}, whereas input_dim={}".format(self.name, X.shape[1], self.input_dim)

    def _check_active_dims(self, X):
        assert X.shape[1] >= len(self.active_dims), "At least {} dimensional X needed, X.shape={!s}".format(len(self.active_dims), X.shape)

class CombinationKernel(Kern):
    """
    Abstract super class for combination kernels.
    A combination kernel combines (a list of) kernels and works on those.
    Examples are the HierarchicalKernel or Add and Prod kernels.
    """
    def __init__(self, kernels, name, extra_dims=[]):
        """
        Abstract super class for combination kernels.
        A combination kernel combines (a list of) kernels and works on those.
        Examples are the HierarchicalKernel or Add and Prod kernels.

        :param list kernels: List of kernels to combine (can be only one element)
        :param str name: name of the combination kernel
        :param array-like extra_dims: if needed extra dimensions for the combination kernel to work on
        """
        assert all([isinstance(k, Kern) for k in kernels])
        extra_dims = np.array(extra_dims, dtype=int)
        input_dim, active_dims = self.get_input_dim_active_dims(kernels, extra_dims)
        # initialize the kernel with the full input_dim
        super(CombinationKernel, self).__init__(input_dim, active_dims, name)
        self.extra_dims = extra_dims
        self.link_parameters(*kernels)

    @property
    def parts(self):
        return self.parameters

    def get_input_dim_active_dims(self, kernels, extra_dims = None):
        #active_dims = reduce(np.union1d, (np.r_[x.active_dims] for x in kernels), np.array([], dtype=int))
        #active_dims = np.array(np.concatenate((active_dims, extra_dims if extra_dims is not None else [])), dtype=int)
        input_dim = reduce(max, (k.active_dims.max() for k in kernels)) + 1

        if extra_dims is not None:
            input_dim += extra_dims.size

        active_dims = np.arange(input_dim)
        return input_dim, active_dims

        """
        If summize is true, we want to get the summerized view of the sensitivities,
        otherwise put everything into an array with shape (#kernels, input_dim)
        in the order of appearance of the kernels in the parameterized object.
        """
        raise NotImplementedError("Choose the kernel you want to get the sensitivity for. You need to override the default behaviour for getting the input sensitivity to be able to get the input sensitivity. For sum kernel it is the sum of all sensitivities, TODO: product kernel? Other kernels?, also TODO: shall we return all the sensitivities here in the combination kernel? So we can combine them however we want? This could lead to just plot all the sensitivities here...")

        return

        # As combination kernels cannot always know, what their inner kernels have as input dims, the check will be done inside them, respectively
        return

"""
The package for scikits.cuda initialization

Global variables: initSuccess
providing CUBLAS handle: cublas_handle
"""

gpu_initialized = False
gpu_device = None
gpu_context = None
MPI_enabled = False

try:
    MPI_enabled = True
except:
    pass

try:
    if MPI_enabled and MPI.COMM_WORLD.size>1:
        from .parallel import get_id_within_node
        gpuid = get_id_within_node()
        import pycuda.driver
        pycuda.driver.init()
        if gpuid>=pycuda.driver.Device.count():
            print '['+MPI.Get_processor_name()+'] more processes than the GPU numbers!'
            #MPI.COMM_WORLD.Abort()
            raise
        gpu_device = pycuda.driver.Device(gpuid)
        gpu_context = gpu_device.make_context()
        gpu_initialized = True
    else:
        import pycuda.autoinit
except:
    pass

try:
    from scikits.cuda import cublas
    import scikits.cuda.linalg as culinalg
    culinalg.init()
    cublas_handle = cublas.cublasCreate()
except:
    pass

def closeGPU():
    if gpu_context is not None:
        gpu_context.detach()

# Copyright (c) 2013, 2014 Alan Saul
#
#Parts of this file were influenced by the Matlab GPML framework written by
#Carl Edward Rasmussen & Hannes Nickisch, however all bugs are our own.
#
#The GPML code is released under the FreeBSD License.
#Copyright (c) 2005-2013 Carl Edward Rasmussen & Hannes Nickisch. All rights reserved.
#
#The code and associated documentation is available from
#http://gaussianprocess.org/gpml/code.

from ...util.linalg import mdot, jitchol, dpotrs, dtrtrs, dpotri, symmetrify, pdinv
def warning_on_one_line(message, category, filename, lineno, file=None, line=None):
    return ' %s:%s: %s:%s\n' % (filename, lineno, category.__name__, message)
warnings.formatwarning = warning_on_one_line

class Laplace(LatentFunctionInference):

        """
        Laplace Approximation

        Find the moments \hat{f} and the hessian at this point
        (using Newton-Raphson) of the unnormalised posterior

        """

        self._mode_finding_tolerance = 1e-7
        self._mode_finding_max_iter = 60
        self.bad_fhat = False
        #Store whether it is the first run of the inference so that we can choose whether we need
        #to calculate things or reuse old variables
        self.first_run = True
        self._previous_Ki_fhat = None

        """
        """

        # Compute K

        #Find mode
        if self.bad_fhat or self.first_run:
            Ki_f_init = np.zeros_like(Y)
            first_run = False
        else:
            Ki_f_init = self._previous_Ki_fhat

        f_hat, Ki_fhat = self.rasm_mode(K, Y, likelihood, Ki_f_init, Y_metadata=Y_metadata)
        self.f_hat = f_hat
        self.Ki_fhat =  Ki_fhat
        self.K = K.copy()
        #Compute hessian and other variables at mode
        log_marginal, woodbury_inv, dL_dK, dL_dthetaL = self.mode_computations(f_hat, Ki_fhat, K, Y, likelihood, kern, Y_metadata)

        self._previous_Ki_fhat = Ki_fhat.copy()
        return Posterior(woodbury_vector=Ki_fhat, woodbury_inv=woodbury_inv, K=K), log_marginal, {'dL_dK':dL_dK, 'dL_dthetaL':dL_dthetaL}

    def rasm_mode(self, K, Y, likelihood, Ki_f_init, Y_metadata=None):
        """
        Rasmussen's numerically stable mode finding
        For nomenclature see Rasmussen & Williams 2006
        Influenced by GPML (BSD) code, all errors are our own

        :param K: Covariance matrix evaluated at locations X
        :type K: NxD matrix
        :param Y: The data
        :param likelihood: the likelihood of the latent function value for the given data
        :type likelihood: a GPy.likelihood object
        :param Ki_f_init: the initial guess at the mode
        :type Ki_f_init: np.ndarray
        :param Y_metadata: information about the data, e.g. which likelihood to take from a multi-likelihood object
        :type Y_metadata: np.ndarray | None
        :returns: f_hat, mode on which to make laplace approxmiation
        :rtype: np.ndarray
        """

        Ki_f = Ki_f_init.copy()
        f = np.dot(K, Ki_f)

        #define the objective function (to be maximised)
        def obj(Ki_f, f):
            return -0.5*np.dot(Ki_f.flatten(), f.flatten()) + np.sum(likelihood.logpdf(f, Y, Y_metadata=Y_metadata))

        difference = np.inf
        iteration = 0
        while difference > self._mode_finding_tolerance and iteration < self._mode_finding_max_iter:
            W = -likelihood.d2logpdf_df2(f, Y, Y_metadata=Y_metadata)
            if np.any(np.isnan(W)):
                raise ValueError('One or more element(s) of W is NaN')
            grad = likelihood.dlogpdf_df(f, Y, Y_metadata=Y_metadata)
            if np.any(np.isnan(grad)):
                raise ValueError('One or more element(s) of grad is NaN')

            W_f = W*f

            b = W_f + grad # R+W p46 line 6.
            W12BiW12, _, _ = self._compute_B_statistics(K, W, likelihood.log_concave)
            W12BiW12Kb = np.dot(W12BiW12, np.dot(K, b))

            #Work out the DIRECTION that we want to move in, but don't choose the stepsize yet
            full_step_Ki_f = b - W12BiW12Kb # full_step_Ki_f = a in R&W p46 line 6.
            dKi_f = full_step_Ki_f - Ki_f

            #define an objective for the line search (minimize this one)
            def inner_obj(step_size):
                Ki_f_trial = Ki_f + step_size*dKi_f
                f_trial = np.dot(K, Ki_f_trial)
                return -obj(Ki_f_trial, f_trial)

            #use scipy for the line search, the compute new values of f, Ki_f
            step = optimize.brent(inner_obj, tol=1e-4, maxiter=12)
            Ki_f_new = Ki_f + step*dKi_f
            f_new = np.dot(K, Ki_f_new)

            difference = np.abs(np.sum(f_new - f)) + np.abs(np.sum(Ki_f_new - Ki_f))
            Ki_f = Ki_f_new
            f = f_new
            iteration += 1

        #Warn of bad fits
        if difference > self._mode_finding_tolerance:
            if not self.bad_fhat:
                warnings.warn("Not perfect mode found (f_hat). difference: {}, iteration: {} out of max {}".format(difference, iteration, self._mode_finding_max_iter))
            self.bad_fhat = True
        elif self.bad_fhat:
            self.bad_fhat = False
            warnings.warn("f_hat now fine again. difference: {}, iteration: {} out of max {}".format(difference, iteration, self._mode_finding_max_iter))

        return f, Ki_f

    def mode_computations(self, f_hat, Ki_f, K, Y, likelihood, kern, Y_metadata):
        """
        At the mode, compute the hessian and effective covariance matrix.

        returns: logZ : approximation to the marginal likelihood
                 woodbury_inv : variable required for calculating the approximation to the covariance matrix
                 dL_dthetaL : array of derivatives (1 x num_kernel_params)
                 dL_dthetaL : array of derivatives (1 x num_likelihood_params)
        """
        #At this point get the hessian matrix (or vector as W is diagonal)
        W = -likelihood.d2logpdf_df2(f_hat, Y, Y_metadata=Y_metadata)
        if np.any(np.isnan(W)):
            raise ValueError('One or more element(s) of W is NaN')

        K_Wi_i, L, LiW12 = self._compute_B_statistics(K, W, likelihood.log_concave)

        #compute vital matrices
        C = np.dot(LiW12, K)
        Ki_W_i  = K - C.T.dot(C)

        #compute the log marginal
        log_marginal = -0.5*np.dot(Ki_f.flatten(), f_hat.flatten()) + np.sum(likelihood.logpdf(f_hat, Y, Y_metadata=Y_metadata)) - np.sum(np.log(np.diag(L)))

        # Compute matrices for derivatives
        dW_df = -likelihood.d3logpdf_df3(f_hat, Y, Y_metadata=Y_metadata) # -d3lik_d3fhat
        if np.any(np.isnan(dW_df)):
            raise ValueError('One or more element(s) of dW_df is NaN')

        dL_dfhat = -0.5*(np.diag(Ki_W_i)[:, None]*dW_df) # s2 in R&W p126 line 9.
        #BiK, _ = dpotrs(L, K, lower=1)
        #dL_dfhat = 0.5*np.diag(BiK)[:, None]*dW_df
        I_KW_i = np.eye(Y.shape[0]) - np.dot(K, K_Wi_i)

        ####################
        #  compute dL_dK   #
        if kern.size > 0 and not kern.is_fixed:
            #Explicit
            explicit_part = 0.5*(np.dot(Ki_f, Ki_f.T) - K_Wi_i)

            #Implicit
            implicit_part = np.dot(Ki_f, dL_dfhat.T).dot(I_KW_i)

            dL_dK = explicit_part + implicit_part
        else:
            dL_dK = np.zeros(likelihood.size)

        #compute dL_dthetaL#
        if likelihood.size > 0 and not likelihood.is_fixed:
            dlik_dthetaL, dlik_grad_dthetaL, dlik_hess_dthetaL = likelihood._laplace_gradients(f_hat, Y, Y_metadata=Y_metadata)

            num_params = likelihood.size
            # make space for one derivative for each likelihood parameter
            dL_dthetaL = np.zeros(num_params)
            for thetaL_i in range(num_params):
                #Explicit
                dL_dthetaL_exp = ( np.sum(dlik_dthetaL[thetaL_i])
                                # The + comes from the fact that dlik_hess_dthetaL == -dW_dthetaL
                                + 0.5*np.sum(np.diag(Ki_W_i).flatten()*dlik_hess_dthetaL[:, thetaL_i].flatten())
                                )

                #Implicit
                dfhat_dthetaL = mdot(I_KW_i, K, dlik_grad_dthetaL[:, thetaL_i])
                #dfhat_dthetaL = mdot(Ki_W_i, dlik_grad_dthetaL[:, thetaL_i])
                dL_dthetaL_imp = np.dot(dL_dfhat.T, dfhat_dthetaL)
                dL_dthetaL[thetaL_i] = dL_dthetaL_exp + dL_dthetaL_imp

        else:
            dL_dthetaL = np.zeros(likelihood.size)

        return log_marginal, K_Wi_i, dL_dK, dL_dthetaL

    def _compute_B_statistics(self, K, W, log_concave):
        """
        Rasmussen suggests the use of a numerically stable positive definite matrix B
        Which has a positive diagonal elements and can be easily inverted

        :param K: Prior Covariance matrix evaluated at locations X
        :type K: NxN matrix
        :param W: Negative hessian at a point (diagonal matrix)
        :type W: Vector of diagonal values of Hessian (1xN)
        :returns: (W12BiW12, L_B, Li_W12)
        """
        if not log_concave:
            #print "Under 1e-10: {}".format(np.sum(W < 1e-6))
            W[W<1e-6] = 1e-6
            # NOTE: when setting a parameter inside parameters_changed it will allways come to closed update circles!!!
            #W.__setitem__(W < 1e-6, 1e-6, update=False)  # FIXME-HACK: This is a hack since GPy can't handle negative variances which can occur
                                # If the likelihood is non-log-concave. We wan't to say that there is a negative variance
                                # To cause the posterior to become less certain than the prior and likelihood,
                                # This is a property only held by non-log-concave likelihoods
        #W is diagonal so its sqrt is just the sqrt of the diagonal elements
        W_12 = np.sqrt(W)
        B = np.eye(K.shape[0]) + W_12*K*W_12.T
        L = jitchol(B)

        LiW12, _ = dtrtrs(L, np.diagflat(W_12), lower=1, trans=0)
        K_Wi_i = np.dot(LiW12.T, LiW12) # R = W12BiW12, in R&W p 126, eq 5.25

        #here's a better way to compute the required matrix.
        # you could do the model finding witha backsub, instead of a dot...
        #L2 = L/W_12
        #K_Wi_i_2 , _= dpotri(L2)
        #symmetrify(K_Wi_i_2)

        return K_Wi_i, L, LiW12

class GPCoregionalizedRegression(GP):
    """
    Gaussian Process model for heteroscedastic multioutput regression

    """
    def __init__(self, X_list, Y_list, kernel=None, likelihoods_list=None, name='GPCR',W_rank=1,kernel_name='coreg'):

        #Kernel

        super(GPCoregionalizedRegression, self).__init__(X,Y,kernel,likelihood, Y_metadata={'output_index':self.output_index})

from domains import _POSITIVE,_NEGATIVE, _BOUNDED
import weakref

import sys

_lim_val = 36.0
epsilon = np.finfo(np.float64).resolution

#===============================================================================
# Fixing constants
__fixed__ = "fixed"
FIXED = False
UNFIXED = True

class Transformation(object):
    domain = None
    _instance = None
    def __new__(cls, *args, **kwargs):
        if not cls._instance or cls._instance.__class__ is not cls:
            cls._instance = super(Transformation, cls).__new__(cls, *args, **kwargs)
        return cls._instance
    def f(self, opt_param):
    def finv(self, model_param):
    def gradfactor(self, model_param, dL_dmodel_param):
        """ df(opt_param)_dopt_param evaluated at self.f(opt_param)=model_param, times the gradient dL_dmodel_param,

        i.e.:
        define

        .. math::

            \frac{\frac{\partial L}{\partial f}\left(\left.\partial f(x)}{\partial x}\right|_{x=f^{-1}(f)\right)}
        """
    def initialize(self, f):
        """ produce a sensible initial value for f(x)"""
    def plot(self, xlabel=r'transformed $\theta$', ylabel=r'$\theta$', axes=None, *args,**kw):
        from ...plotting.matplot_dep import base_plots
        x = np.linspace(-8,8)
        base_plots.meanplot(x, self.f(x),axes=axes*args,**kw)
        axes = plt.gca()
        axes.set_xlabel(xlabel)
        axes.set_ylabel(ylabel)
        return self.__class__.__name__

class Logexp(Transformation):
    domain = _POSITIVE
    def f(self, x):
        return np.where(x>_lim_val, x, np.log1p(np.exp(np.clip(x, -_lim_val, _lim_val)))) + epsilon
        #raises overflow warning: return np.where(x>_lim_val, x, np.log(1. + np.exp(x)))
    def finv(self, f):
        return np.where(f>_lim_val, f, np.log(np.exp(f+1e-20) - 1.))
    def gradfactor(self, f, df):
        return np.einsum('i,i->i', df, np.where(f>_lim_val, 1., 1. - np.exp(-f)))
        if np.any(f < 0.):
            print "Warning: changing parameters to satisfy constraints"
        return np.abs(f)
        return '+ve'

class NormalTheta(Transformation):
    _instances = []
    def __new__(cls, mu_indices, var_indices):
        if cls._instances:
            cls._instances[:] = [instance for instance in cls._instances if instance()]
            for instance in cls._instances:
                if np.all(instance().mu_indices==mu_indices, keepdims=False) and np.all(instance().var_indices==var_indices, keepdims=False):
                    return instance()
        o = super(Transformation, cls).__new__(cls, mu_indices, var_indices)
        cls._instances.append(weakref.ref(o))
        return cls._instances[-1]()

    def __init__(self, mu_indices, var_indices):
        self.mu_indices = mu_indices
        self.var_indices = var_indices

    def f(self, theta):
        # In here abs is only a trick to make sure the numerics are ok.
        # The variance will never go below zero, but at initialization we need to make sure
        # that the values are ok
        # Before:
        theta[self.var_indices] = np.abs(-.5/theta[self.var_indices])
        theta[self.mu_indices] *= theta[self.var_indices]
        return theta # which is now {mu, var}

    def finv(self, muvar):
        # before:
        varp = muvar[self.var_indices]
        muvar[self.mu_indices] /= varp
        muvar[self.var_indices] = -.5/varp

        return muvar # which is now {theta1, theta2}

    def gradfactor(self, muvar, dmuvar):
        mu = muvar[self.mu_indices]
        var = muvar[self.var_indices]
        # theta gradients
        # This works and the gradient checks!
        dmuvar[self.mu_indices] *= var
        dmuvar[self.var_indices] *= 2*(var)**2
        dmuvar[self.var_indices] += 2*dmuvar[self.mu_indices]*mu

        return dmuvar # which is now the gradient multiplicator for {theta1, theta2}

        if np.any(f[self.var_indices] < 0.):
            f[self.var_indices] = np.abs(f[self.var_indices])
        return f

        return "theta"

        return [self.mu_indices, self.var_indices]

        self.mu_indices = state[0]
        self.var_indices = state[1]

class NormalNaturalAntti(NormalTheta):
    _logexp = Logexp()

        dmuvar[self.var_indices] *= 2*var**2#np.einsum('i,i,i,i->i', dmuvar[self.var_indices], [2], var, var)

        return dmuvar # which is now the gradient multiplicator

        return f

        return "natantti"

class NormalEta(Transformation):

        theta[self.var_indices] = np.abs(theta[self.var_indices] - theta[self.mu_indices]**2)

        muvar[self.var_indices] += muvar[self.mu_indices]**2
        return muvar # which is now {eta1, eta2}

        # Lets try natural gradients instead: Not working with bfgs... try stochastic!
        dmuvar[self.mu_indices] -= 2*mu*dmuvar[self.var_indices]

        return f

        return "eta"

class NormalNaturalThroughTheta(NormalTheta):

        # This is just eta direction:

        # This is by going through theta fully and then going into eta direction:
        #dmu = dmuvar[self.mu_indices]
        #dmuvar[self.var_indices] += dmu*mu*(var + 4/var)

        return "natgrad"

class NormalNaturalThroughEta(NormalEta):

        return dmuvar

class LogexpNeg(Transformation):
        return np.where(x>_lim_val, -x, -np.log(1. + np.exp(np.clip(x, -np.inf, _lim_val))))
        return np.where(f>_lim_val, 0, np.log(np.exp(-f) - 1.))
        return np.einsum('i,i->i', df, np.where(f>_lim_val, -1, -1 + np.exp(-f)))

class NegativeLogexp(Transformation):
    domain = _NEGATIVE
    logexp = Logexp()
        return -self.logexp.f(x)  # np.log(1. + np.exp(x))
        return self.logexp.finv(-f)  # np.log(np.exp(-f) - 1.)
        return np.einsum('i,i->i', df, -self.logexp.gradfactor(-f))
        return -self.logexp.initialize(f)  # np.abs(f)
        return '-ve'

class LogexpClipped(Logexp):
    max_bound = 1e100
    min_bound = 1e-10
    log_max_bound = np.log(max_bound)
    log_min_bound = np.log(min_bound)
    def __new__(cls, lower=1e-6, *args, **kwargs):
                if instance().lower == lower:
        o = super(Transformation, cls).__new__(cls, lower, *args, **kwargs)
    def __init__(self, lower=1e-6):
        self.lower = lower
        exp = np.exp(np.clip(x, self.log_min_bound, self.log_max_bound))
        f = np.log(1. + exp)
#         if np.isnan(f).any():
#             import ipdb;ipdb.set_trace()
        return np.clip(f, self.min_bound, self.max_bound)
        return np.log(np.exp(f - 1.))
        ef = np.exp(f) # np.clip(f, self.min_bound, self.max_bound))
        gf = (ef - 1.) / ef
        return np.einsum('i,i->i', df, gf) # np.where(f < self.lower, 0, gf)
        return '+ve_c'

class Exponent(Transformation):
    # TODO: can't allow this to go to zero, need to set a lower bound. Similar with negative Exponent below. See old MATLAB code.
        return np.where(x<_lim_val, np.where(x>-_lim_val, np.exp(x), np.exp(-_lim_val)), np.exp(_lim_val))
    def finv(self, x):
        return np.log(x)
        return np.einsum('i,i->i', df, f)

class NegativeExponent(Exponent):
        return -Exponent.f(x)
        return Exponent.finv(-f)
        return -Exponent.initialize(f) #np.abs(f)

class Square(Transformation):
        return x ** 2
        return np.sqrt(x)
        return np.einsum('i,i->i', df, 2 * np.sqrt(f))
        return '+sq'

class Logistic(Transformation):
    domain = _BOUNDED
    def __new__(cls, lower=1e-6, upper=1e-6, *args, **kwargs):
                if instance().lower == lower and instance().upper == upper:
        o = super(Transformation, cls).__new__(cls, lower, upper, *args, **kwargs)
    def __init__(self, lower, upper):
        assert lower < upper
        self.lower, self.upper = float(lower), float(upper)
        self.difference = self.upper - self.lower
        if (x<-300.).any():
            x = x.copy()
            x[x<-300.] = -300.
        return self.lower + self.difference / (1. + np.exp(-x))
        return np.log(np.clip(f - self.lower, 1e-10, np.inf) / np.clip(self.upper - f, 1e-10, np.inf))
        return np.einsum('i,i->i', df, (f - self.lower) * (self.upper - f) / self.difference)
        if np.any(np.logical_or(f < self.lower, f > self.upper)):
        #return np.where(np.logical_or(f < self.lower, f > self.upper), self.f(f * 0.), f)
        #FIXME: Max, zeros_like right?
        return np.where(np.logical_or(f < self.lower, f > self.upper), self.f(np.zeros_like(f)), f)
        return '{},{}'.format(self.lower, self.upper)

# Copyright (c) 2014, James Hensman, Max Zwiessele

class Remapping(Parameterized):
    def mapping(self):
        """
        The return value of this function gives the values which the re-mapped
        parameters should take. Implement in sub-classes.
        """

    def callback(self):

        return self.name

        #ensure all out parameters have the correct value, as specified by our mapping
        index = self._highest_parent_.constraints[self]
        self._highest_parent_.param_array[index] = self.mapping()
        [p.notify_observers(which=self) for p in self.tied_parameters]

class Fix(Remapping):
    pass

class Tie(Parameterized):
    """
    The new parameter tie framework. (under development)
    
    All the parameters tied together get a new parameter inside the *Tie* object. 
    Its value should always be equal to all the tied parameters, and its gradient
    is the sum of all the tied parameters.
    
    =====Implementation Details=====
    The *Tie* object should only exist on the top of param tree (the highest parent).
    
    self.label_buf:
    It uses a label buffer that has the same length as all the parameters (self._highest_parent_.param_array).
    The buffer keeps track of all the tied parameters. All the tied parameters have a label (an interger) higher 
    than 0, and the parameters that have the same label are tied together.
    
    self.buf_index:
    An auxiliary index list for the global index of the tie parameter inside the *Tie* object.
    
    ================================
    
    TODO:
    * EVERYTHING
    
    """
    def __init__(self, name='tie'):
        super(Tie, self).__init__(name)
        self.tied_param = None
        # The buffer keeps track of tie status
        self.label_buf = None
        # The global indices of the 'tied' param
        self.buf_idx = None
        # A boolean array indicating non-tied parameters
        self._tie_ = None
        
    def getTieFlag(self, p=None):
        if self.tied_param is None:
            if self._tie_ is None or self._tie_.size != self._highest_parent_.param_array.size:
                self._tie_ = np.ones((self._highest_parent_.param_array.size,),dtype=np.bool)
        if p is not None:
            return self._tie_[p._highest_parent_._raveled_index_for(p)]
        return self._tie_
    
    def _init_labelBuf(self):
        if self.label_buf is None:
            self.label_buf = np.zeros(self._highest_parent_.param_array.shape, dtype=np.int)
        if self._tie_ is None or self._tie_.size != self._highest_parent_.param_array.size:
            self._tie_ = np.ones((self._highest_parent_.param_array.size,),dtype=np.bool)
            
    def _updateTieFlag(self):
        if self._tie_.size != self.label_buf.size:
        self._tie_[self.label_buf>0] = False
        self._tie_[self.buf_idx] = True

    def add_tied_parameter(self, p, p2=None):
        """
        Tie the list of parameters p together (p2==None) or 
        Tie the list of parameters p with the list of parameters p2 (p2!=None) 
        """
        self._init_labelBuf()
        if p2 is None:
            idx = self._highest_parent_._raveled_index_for(p)
            val = self._sync_val_group(idx)            
            if np.all(self.label_buf[idx]==0):
                # None of p has been tied before.
                tie_idx = self._expandTieParam(1)
                print tie_idx
                tie_id = self.label_buf.max()+1
                self.label_buf[tie_idx] = tie_id
            else:
                b = self.label_buf[idx]
                ids = np.unique(b[b>0])
                tie_id, tie_idx = self._merge_tie_param(ids)
            self._highest_parent_.param_array[tie_idx] = val
            self.label_buf[idx] = tie_id
        else:
            pass
        self._updateTieFlag()
        
    def _merge_tie_param(self, ids):
        """Merge the tie parameters with ids in the list."""
        if len(ids)==1:
            id_final_idx = self.buf_idx[self.label_buf[self.buf_idx]==ids[0]][0]
            return ids[0],id_final_idx
        id_final = ids[0]
        ids_rm = ids[1:]
        label_buf_param = self.label_buf[self.buf_idx]
        idx_param = [np.where(label_buf_param==i)[0][0] for i in ids_rm]
        self._removeTieParam(idx_param)
        [np.put(self.label_buf, np.where(self.label_buf==i), id_final) for i in ids_rm]
        id_final_idx = self.buf_idx[self.label_buf[self.buf_idx]==id_final][0]
        return id_final, id_final_idx
        
    def _sync_val_group(self, idx):
        self._highest_parent_.param_array[idx] = self._highest_parent_.param_array[idx].mean()
        return self._highest_parent_.param_array[idx][0]
        
    def _expandTieParam(self, num):
        """Expand the tie param with the number of *num* parameters"""
            new_buf = np.empty((num,))
        else:
            new_buf = np.empty((self.tied_param.size+num,))
            new_buf[:self.tied_param.size] = self.tied_param.param_array.copy()
            self.remove_parameter(self.tied_param)
        self.tied_param = Param('tied',new_buf)
        self.add_parameter(self.tied_param)
        buf_idx_new = self._highest_parent_._raveled_index_for(self.tied_param)
        self._expand_label_buf(self.buf_idx, buf_idx_new)
        self.buf_idx = buf_idx_new
        return self.buf_idx[-num:]

    def _removeTieParam(self, idx):
        """idx within tied_param"""
        new_buf = np.empty((self.tied_param.size-len(idx),))
        bool_list = np.ones((self.tied_param.size,),dtype=np.bool)
        bool_list[idx] = False
        new_buf[:] = self.tied_param.param_array[bool_list]
        self.remove_parameter(self.tied_param)
        self._shrink_label_buf(self.buf_idx, buf_idx_new, bool_list)
        
    def _expand_label_buf(self, idx_old, idx_new):
        """Expand label buffer accordingly"""
        if idx_old is None:
        else:
            bool_old = np.zeros((self.label_buf.size,),dtype=np.bool)
            bool_old[idx_old] = True
            bool_new = np.zeros((self._highest_parent_.param_array.size,),dtype=np.bool)
            bool_new[idx_new] = True
            label_buf_new = np.zeros(self._highest_parent_.param_array.shape, dtype=np.int)
            label_buf_new[np.logical_not(bool_new)] = self.label_buf[np.logical_not(bool_old)]
            label_buf_new[idx_new[:len(idx_old)]] = self.label_buf[idx_old]
            self.label_buf = label_buf_new

    def _shrink_label_buf(self, idx_old, idx_new, bool_list):
        bool_old = np.zeros((self.label_buf.size,),dtype=np.bool)
        bool_old[idx_old] = True
        bool_new = np.zeros((self._highest_parent_.param_array.size,),dtype=np.bool)
        bool_new[idx_new] = True
        label_buf_new = np.empty(self._highest_parent_.param_array.shape, dtype=np.int)
        label_buf_new[np.logical_not(bool_new)] = self.label_buf[np.logical_not(bool_old)]
        label_buf_new[idx_new] = self.label_buf[idx_old[bool_list]]
        self.label_buf = label_buf_new

    def _check_change(self):
        changed = False
        if self.tied_param is not None:
            for i in xrange(self.tied_param.size):
                b0 = self.label_buf==self.label_buf[self.buf_idx[i]]
                b = self._highest_parent_.param_array[b0]!=self.tied_param[i]
                if b.sum()==0:
                    print 'XXX'
                    continue
                elif b.sum()==1:
                    print '!!!'
                    val = self._highest_parent_.param_array[b0][b][0]
                    self._highest_parent_.param_array[b0] = val
                else:
                    print '@@@'
                    self._highest_parent_.param_array[b0] = self.tied_param[i]
                changed = True
        return changed

        changed = self._check_change()
        if changed:
            self._highest_parent_._trigger_params_changed()
        self.collate_gradient()

    def collate_gradient(self):
            self.tied_param.gradient = 0.
            [np.put(self.tied_param.gradient, i, self._highest_parent_.gradient[self.label_buf==self.label_buf[self.buf_idx[i]]].sum()) 
                for i in xrange(self.tied_param.size)]
    
    def propagate_val(self):
                self._highest_parent_.param_array[self.label_buf==self.label_buf[self.buf_idx[i]]] = self.tied_param[i]

from collections import defaultdict

def intarray_default_factory():
    return np.int_([])

class IntArrayDict(defaultdict):
    def __init__(self, default_factory=None):
        """
        Default will be self._default, if not set otherwise
        """
        defaultdict.__init__(self, intarray_default_factory)

class ArrayList(list):
    """
    List to store ndarray-likes in.
    It will look for 'is' instead of calling __eq__ on each element.
    """
    def __contains__(self, other):
        for el in self:
            if el is other:

    def index(self, item):
        index = 0
            if el is item:
                return index
            index += 1
        raise ValueError, "{} is not in list".format(item)
    pass

class ObserverList(object):
    """
    A list which containts the observables.
    It only holds weak references to observers, such that unbound
    observers dont dangle in memory.
    """
        self._poc = []

    def __getitem__(self, ind):
        p,o,c = self._poc[ind]
        return p, o(), c

    def remove(self, priority, observer, callble):
        """
        Remove one observer, which had priority and callble.
        """
        self.flush()
        for i in range(len(self) - 1, -1, -1):
            p,o,c = self[i]
            if priority==p and observer==o and callble==c:
                del self._poc[i]

        return self._poc.__repr__()

    def add(self, priority, observer, callble):
        """
        Add an observer with priority and callble
        """
        if observer is not None:
            ins = 0
            for pr, _, _ in self:
                if priority > pr:
                    break
                ins += 1
            self._poc.insert(ins, (priority, weakref.ref(observer), callble))

        from . import ObsAr, Param
        from parameter_core import Parameterizable
        ret = []
        curr_p = None
        
        def frmt(o):
            if isinstance(o, ObsAr):
                return 'ObsArr <{}>'.format(hex(id(o)))
            elif isinstance(o, (Param,Parameterizable)):
                return '{}'.format(o.hierarchy_name())
            else:
                return repr(o)                
        for p, o, c in self:
            curr = ''
            if curr_p != p:
                pre = "{!s}: ".format(p)
                curr_pre = pre
            else: curr_pre = " "*len(pre)
            curr_p = p
            curr += curr_pre
            
            ret.append(curr + ", ".join([frmt(o), str(c)]))
            return '\n'.join(ret)

    def flush(self):
        """
        Make sure all weak references, which point to nothing are flushed (deleted)
        """
        self._poc = [(p,o,c) for p,o,c in self._poc if o() is not None]

    def __iter__(self):
        for p, o, c in self._poc:
            yield p, o(), c 

    def __len__(self):
        return self._poc.__len__()

        s = ObserverList()
        for p,o,c in self:
            s.add(p, copy.deepcopy(o, memo), copy.deepcopy(c, memo))
        s.flush()
        return s

        obs = []
            if (getattr(o, c.__name__, None) is not None 
                and not isinstance(o, Cacher)):
                obs.append((p,o,c.__name__))
        return obs

        for p, o, c in state:
            self.add(p,o,getattr(o, c))

    pass

from ..core.gp import GP
from ..core.parameterization.variational import VariationalPosterior

class SparseGPMiniBatch(GP):
    """
'''
Created on 3 Nov 2014

'''

    these.

    """

                 name='sparse gp', Y_metadata=None, normalizer=False,
            else:

        self.missing_data = missing_data

        if stochastic and missing_data:
            self.ninan = ~np.isnan(Y)
            self.stochastics = SparseGPStochastics(self, batchsize)
        elif stochastic and not missing_data:
        elif missing_data:
            self.stochastics = SparseGPMissing(self)
        else:
            self.stochastics = False

            self.Ylist = []
            overall = self.Y_normalized.shape[1]
            m_f = lambda i: "Precomputing Y for missing data: {: >7.2%}".format(float(i+1)/overall)
            message = m_f(-1)
            for d in xrange(overall):
                self.Ylist.append(self.Y_normalized[self.ninan[:, d], d][:, None])
                print ' '*(len(message)+1) + '\r',
                message = m_f(d)
                print message,
            print ''

        """
        This is the standard part, which usually belongs in parameters_changed.

        For automatic handling of subsampling (such as missing_data, stochastics etc.), we need to put this into an inner
        loop, in order to ensure a different handling of gradients etc of different
        subsets of data.

        The dict in current_values will be passed aroung as current_values for
        the rest of the algorithm, so this is the place to store current values,
        such as subsets etc, if necessary.

        If Lm and dL_dKmm can be precomputed (or only need to be computed once)
        pass them in here, so they will be passed to the inference_method.

        subset_indices is a dictionary of indices. you can put the indices however you
        like them into this dictionary for inner use of the indices inside the
        algorithm.
        """
        try:
            posterior, log_marginal_likelihood, grad_dict = self.inference_method.inference(kern, X, Z, likelihood, Y, Y_metadata, Lm=Lm, dL_dKmm=None)
        except:
            posterior, log_marginal_likelihood, grad_dict = self.inference_method.inference(kern, X, Z, likelihood, Y, Y_metadata)
        current_values = {}
        likelihood.update_gradients(grad_dict['dL_dthetaL'])
        current_values['likgrad'] = likelihood.gradient.copy()
        if subset_indices is None:
            subset_indices = {}
            dL_dKmm = grad_dict['dL_dKmm']
            kern.update_gradients_full(dL_dKmm, Z, None)
            current_values['kerngrad'] = kern.gradient.copy()
            kern.update_gradients_expectations(variational_posterior=X,
                                                    Z=Z,
                                                    dL_dpsi0=grad_dict['dL_dpsi0'],
                                                    dL_dpsi1=grad_dict['dL_dpsi1'],
                                                    dL_dpsi2=grad_dict['dL_dpsi2'])
            current_values['kerngrad'] += kern.gradient

            current_values['Zgrad'] = kern.gradients_X(dL_dKmm, Z)
            current_values['Zgrad'] += kern.gradients_Z_expectations(
                               grad_dict['dL_dpsi0'],
                               grad_dict['dL_dpsi1'],
                               grad_dict['dL_dpsi2'],
                               Z=Z,
                               variational_posterior=X)
        else:
            kern.update_gradients_diag(grad_dict['dL_dKdiag'], X)
            kern.update_gradients_full(grad_dict['dL_dKnm'], X, Z)
            kern.update_gradients_full(grad_dict['dL_dKmm'], Z, None)
            current_values['Zgrad'] = kern.gradients_X(grad_dict['dL_dKmm'], Z)
            current_values['Zgrad'] += kern.gradients_X(grad_dict['dL_dKnm'].T, Z, X)
        return posterior, log_marginal_likelihood, grad_dict, current_values, subset_indices

    def _inner_take_over_or_update(self, full_values=None, current_values=None, value_indices=None):
        """
        This is for automatic updates of values in the inner loop of missing
        data handling. Both arguments are dictionaries and the values in
        full_values will be updated by the current_gradients.

        If a key from current_values does not exist in full_values, it will be
        initialized to the value in current_values.

        If there is indices needed for the update, value_indices can be used for
        that. If value_indices has the same key, as current_values, the update
        in full_values will be indexed by the indices in value_indices.

        grads:
            dictionary of standing gradients (you will have to carefully make sure, that
            the ordering is right!). The values in here will be updated such that
            full_values[key] += current_values[key]  forall key in full_gradients.keys()

        gradients:
            dictionary of gradients in the current set of parameters.

        value_indices:
            dictionary holding indices for the update in full_values.
            if the key exists the update rule is:def df(x):
            full_values[key][value_indices[key]] += current_values[key]
        """
        for key in current_values.keys():
            if value_indices is not None and value_indices.has_key(key):
                index = value_indices[key]
            else:
                index = slice(None)
            if full_values.has_key(key):
                full_values[key][index] += current_values[key]
            else:
                full_values[key] = current_values[key]

    def _inner_values_update(self, current_values):
        """
        This exists if there is more to do with the current values.
        It will be called allways in the inner loop, so that
        you can do additional inner updates for the inside of the missing data
        loop etc. This can also be used for stochastic updates, when only working on
        one dimension of the output.
        """
        pass

        """
        """
        self.likelihood.gradient = full_values['likgrad']
        self.kern.gradient = full_values['kerngrad']
        self.Z.gradient = full_values['Zgrad']

        """
        If full_values has indices in values_indices, we might want to initialize
        the full_values differently, so that subsetting is possible.

        Here you can initialize the full_values for the values needed.

        Keep in mind, that if a key does not exist in full_values when updating
        values, it will be set (so e.g. for Z there is no need to initialize Zgrad,
        as there is no subsetting needed. For X in BGPLVM on the other hand we probably need
        to initialize the gradients for the mean and the variance in order to
        have the full gradient for indexing)
        """
        return {}

    def _outer_loop_for_missing_data(self):
        Lm = None
        dL_dKmm = None

        self.full_values = self._outer_init_full_values()

        if self.posterior is None:
            woodbury_inv = np.zeros((self.num_inducing, self.num_inducing, self.output_dim))
            woodbury_vector = np.zeros((self.num_inducing, self.output_dim))
        else:
            woodbury_inv = self.posterior._woodbury_inv
            woodbury_vector = self.posterior._woodbury_vector

        if not self.stochastics:
            m_f = lambda i: "Inference with missing_data: {: >7.2%}".format(float(i+1)/self.output_dim)

        for d in self.stochastics.d:
            ninan = self.ninan[:, d]

            if not self.stochastics:
                print ' '*(len(message)) + '\r',

            posterior, log_marginal_likelihood, \
                grad_dict, current_values, value_indices = self._inner_parameters_changed(
                                self.kern, self.X[ninan],
                                self.Z, self.likelihood,
                                self.Ylist[d], self.Y_metadata,
                                Lm, dL_dKmm,
                                subset_indices=dict(outputs=d, samples=ninan))

            self._inner_take_over_or_update(self.full_values, current_values, value_indices)
            self._inner_values_update(current_values)

            Lm = posterior.K_chol
            woodbury_inv[:, :, d] = posterior.woodbury_inv
            woodbury_vector[:, d:d+1] = posterior.woodbury_vector
            self._log_marginal_likelihood += log_marginal_likelihood
            print ''

            self.posterior = Posterior(woodbury_inv=woodbury_inv, woodbury_vector=woodbury_vector,
                                   K=posterior._K, mean=None, cov=None, K_chol=posterior.K_chol)
        self._outer_values_update(self.full_values)

    def _outer_loop_without_missing_data(self):

        else:

        d = self.stochastics.d
        posterior, log_marginal_likelihood, \
            grad_dict, self.full_values, _ = self._inner_parameters_changed(
                            self.kern, self.X,
                            self.Z, self.likelihood,
                            self.Y_normalized[:, d], self.Y_metadata)
        self.grad_dict = grad_dict

        self._log_marginal_likelihood += log_marginal_likelihood

        woodbury_inv[:, :, d] = posterior.woodbury_inv[:, :, None]
        woodbury_vector[:, d] = posterior.woodbury_vector

            self._outer_loop_for_missing_data()
        elif self.stochastics:
            self._outer_loop_without_missing_data()
        else:
            self.posterior, self._log_marginal_likelihood, self.grad_dict, self.full_values, _ = self._inner_parameters_changed(self.kern, self.X, self.Z, self.likelihood, self.Y_normalized, self.Y_metadata)
            self._outer_values_update(self.full_values)

        """
        """

                var = var
            else:
        else:
            else:

from scipy.special import gammaln, digamma
from ...util.linalg import pdinv
from domains import _REAL, _POSITIVE

class Prior(object):
            cls._instance = super(Prior, cls).__new__(cls, *args, **kwargs)

    def pdf(self, x):
        return np.exp(self.lnpdf(x))

        import sys

        from ...plotting.matplot_dep import priors_plots

        priors_plots.univariate_plot(self)

        return self.__str__()

class Gaussian(Prior):
    """
    Implementation of the univariate Gaussian probability function, coupled with random variables.

    :param mu: mean
    :param sigma: standard deviation

    .. Note:: Bishop 2006 notation is used throughout the code

    """
    domain = _REAL

    def __new__(cls, mu=0, sigma=1):  # Singleton:
                if instance().mu == mu and instance().sigma == sigma:
        o = super(Prior, cls).__new__(cls, mu, sigma)

    def __init__(self, mu, sigma):
        self.mu = float(mu)
        self.sigma = float(sigma)
        self.sigma2 = np.square(self.sigma)
        self.constant = -0.5 * np.log(2 * np.pi * self.sigma2)

        return "N({:.2g}, {:.2g})".format(self.mu, self.sigma)

    def lnpdf(self, x):
        return self.constant - 0.5 * np.square(x - self.mu) / self.sigma2

    def lnpdf_grad(self, x):
        return -(x - self.mu) / self.sigma2

    def rvs(self, n):
        return np.random.randn(n) * self.sigma + self.mu

#     def __getstate__(self):
#         return self.mu, self.sigma
#
#     def __setstate__(self, state):
#         self.mu = state[0]
#         self.sigma = state[1]
#         self.sigma2 = np.square(self.sigma)
#         self.constant = -0.5 * np.log(2 * np.pi * self.sigma2)

class Uniform(Prior):

    def __new__(cls, lower=0, upper=1):  # Singleton:
        o = super(Prior, cls).__new__(cls, lower, upper)

        self.lower = float(lower)
        self.upper = float(upper)

        return "[{:.2g}, {:.2g}]".format(self.lower, self.upper)

        region = (x >= self.lower) * (x <= self.upper)
        return region

        return np.zeros(x.shape)

        return np.random.uniform(self.lower, self.upper, size=n)

#         return self.lower, self.upper
#
#         self.lower = state[0]
#         self.upper = state[1]

class LogGaussian(Gaussian):
    """
    Implementation of the univariate *log*-Gaussian probability function, coupled with random variables.

    """

        return "lnN({:.2g}, {:.2g})".format(self.mu, self.sigma)

        return self.constant - 0.5 * np.square(np.log(x) - self.mu) / self.sigma2 - np.log(x)

        return -((np.log(x) - self.mu) / self.sigma2 + 1.) / x

        return np.exp(np.random.randn(n) * self.sigma + self.mu)

class MultivariateGaussian(Prior):
    """
    Implementation of the multivariate Gaussian probability function, coupled with random variables.

    :param mu: mean (N-dimensional array)
    :param var: covariance matrix (NxN)

    """

    def __new__(cls, mu=0, var=1):  # Singleton:
                if np.all(instance().mu == mu) and np.all(instance().var == var):
        o = super(Prior, cls).__new__(cls, mu, var)

    def __init__(self, mu, var):
        self.mu = np.array(mu).flatten()
        self.var = np.array(var)
        assert len(self.var.shape) == 2
        assert self.var.shape[0] == self.var.shape[1]
        assert self.var.shape[0] == self.mu.size
        self.input_dim = self.mu.size
        self.inv, self.hld = pdinv(self.var)
        self.constant = -0.5 * self.input_dim * np.log(2 * np.pi) - self.hld

    def summary(self):

        d = x - self.mu
        return self.constant - 0.5 * np.sum(d * np.dot(d, self.inv), 1)

        return -np.dot(self.inv, d)

        return np.random.multivariate_normal(self.mu, self.var, n)

        import sys

        from ..plotting.matplot_dep import priors_plots

        priors_plots.multivariate_plot(self)

        return self.mu, self.var

        self.mu = state[0]
        self.var = state[1]

def gamma_from_EV(E, V):
    warnings.warn("use Gamma.from_EV to create Gamma Prior", FutureWarning)
    return Gamma.from_EV(E, V)

class Gamma(Prior):
    """
    Implementation of the Gamma probability function, coupled with random variables.

    :param a: shape parameter
    :param b: rate parameter (warning: it's the *inverse* of the scale)

    """

    def __new__(cls, a=1, b=.5):  # Singleton:
                if instance().a == a and instance().b == b:
        o = super(Prior, cls).__new__(cls, a, b)

    def __init__(self, a, b):
        self.a = float(a)
        self.b = float(b)
        self.constant = -gammaln(self.a) + a * np.log(b)

        return "Ga({:.2g}, {:.2g})".format(self.a, self.b)

        ret = {"E[x]": self.a / self.b, \
               "E[ln x]": digamma(self.a) - np.log(self.b), \
               "var[x]": self.a / self.b / self.b, \
               "Entropy": gammaln(self.a) - (self.a - 1.) * digamma(self.a) - np.log(self.b) + self.a}
        if self.a > 1:
            ret['Mode'] = (self.a - 1.) / self.b
        else:
            ret['mode'] = np.nan
        return ret

        return self.constant + (self.a - 1) * np.log(x) - self.b * x

        return (self.a - 1.) / x - self.b

        return np.random.gamma(scale=1. / self.b, shape=self.a, size=n)

    @staticmethod
    def from_EV(E, V):
        """
        Creates an instance of a Gamma Prior  by specifying the Expected value(s)
        and Variance(s) of the distribution.

        :param E: expected value
        :param V: variance
        """
        a = np.square(E) / V
        b = E / V
        return Gamma(a, b)

        return self.a, self.b

        self.a = state[0]
        self.b = state[1]
        self.constant = -gammaln(self.a) + self.a * np.log(self.b)

class InverseGamma(Gamma):
    """
    Implementation of the inverse-Gamma probability function, coupled with random variables.

    """
    def __new__(cls, a=1, b=.5): # Singleton:

        return "iGa({:.2g}, {:.2g})".format(self.a, self.b)

        return self.constant - (self.a + 1) * np.log(x) - self.b / x

        return -(self.a + 1.) / x + self.b / x ** 2

        return 1. / np.random.gamma(scale=1. / self.b, shape=self.a, size=n)

class DGPLVM_KFDA(Prior):
    """
    Implementation of the Discriminative Gaussian Process Latent Variable function using
    Kernel Fisher Discriminant Analysis by Seung-Jean Kim for implementing Face paper
    by Chaochao Lu.

    :param lambdaa: constant
    :param sigma2: constant

    .. Note:: Surpassing Human-Level Face paper dgplvm implementation

    """
    # _instances = []
    # def __new__(cls, lambdaa, sigma2):  # Singleton:
    #     if cls._instances:
    #         cls._instances[:] = [instance for instance in cls._instances if instance()]
    #         for instance in cls._instances:
    #             if instance().mu == mu and instance().sigma == sigma:
    #                 return instance()
    #     o = super(Prior, cls).__new__(cls, mu, sigma)
    #     cls._instances.append(weakref.ref(o))
    #     return cls._instances[-1]()

    def __init__(self, lambdaa, sigma2, lbl, kern, x_shape):
        """A description for init"""
        self.datanum = lbl.shape[0]
        self.classnum = lbl.shape[1]
        self.lambdaa = lambdaa
        self.sigma2 = sigma2
        self.lbl = lbl
        self.kern = kern
        lst_ni = self.compute_lst_ni()
        self.a = self.compute_a(lst_ni)
        self.A = self.compute_A(lst_ni)
        self.x_shape = x_shape

    def get_class_label(self, y):
        for idx, v in enumerate(y):
            if v == 1:
                return idx
        return -1

    # This function assigns each data point to its own class
    # and returns the dictionary which contains the class name and parameters.
    def compute_cls(self, x):
        cls = {}
        # Appending each data point to its proper class
        for j in xrange(self.datanum):
            class_label = self.get_class_label(self.lbl[j])
            if class_label not in cls:
                cls[class_label] = []
            cls[class_label].append(x[j])
        if len(cls) > 2:
            for i in range(2, self.classnum):
                del cls[i]
        return cls

    def x_reduced(self, cls):
        x1 = cls[0]
        x2 = cls[1]
        x = np.concatenate((x1, x2), axis=0)
        return x

    def compute_lst_ni(self):
        lst_ni = []
        lst_ni1 = []
        lst_ni2 = []
        f1 = (np.where(self.lbl[:, 0] == 1)[0])
        f2 = (np.where(self.lbl[:, 1] == 1)[0])
        for idx in f1:
            lst_ni1.append(idx)
        for idx in f2:
            lst_ni2.append(idx)
        lst_ni.append(len(lst_ni1))
        lst_ni.append(len(lst_ni2))
        return lst_ni

    def compute_a(self, lst_ni):
        a = np.ones((self.datanum, 1))
        count = 0
        for N_i in lst_ni:
            if N_i == lst_ni[0]:
                a[count:count + N_i] = (float(1) / N_i) * a[count]
                count += N_i
            else:
                if N_i == lst_ni[1]:
                    a[count: count + N_i] = -(float(1) / N_i) * a[count]
                    count += N_i
        return a

    def compute_A(self, lst_ni):
        A = np.zeros((self.datanum, self.datanum))
        idx = 0
            B = float(1) / np.sqrt(N_i) * (np.eye(N_i) - ((float(1) / N_i) * np.ones((N_i, N_i))))
            A[idx:idx + N_i, idx:idx + N_i] = B
            idx += N_i
        return A

    # Here log function
        x = x.reshape(self.x_shape)
        K = self.kern.K(x)
        a_trans = np.transpose(self.a)
        paran = self.lambdaa * np.eye(x.shape[0]) + self.A.dot(K).dot(self.A)
        inv_part = pdinv(paran)[0]
        J = a_trans.dot(K).dot(self.a) - a_trans.dot(K).dot(self.A).dot(inv_part).dot(self.A).dot(K).dot(self.a)
        J_star = (1. / self.lambdaa) * J
        return (-1. / self.sigma2) * J_star

    # Here gradient function
        b = self.A.dot(inv_part).dot(self.A).dot(K).dot(self.a)
        a_Minus_b = self.a - b
        a_b_trans = np.transpose(a_Minus_b)
        DJ_star_DK = (1. / self.lambdaa) * (a_Minus_b.dot(a_b_trans))
        DJ_star_DX = self.kern.gradients_X(DJ_star_DK, x)
        return (-1. / self.sigma2) * DJ_star_DX

        return np.random.rand(n)  # A WRONG implementation

        return 'DGPLVM_prior'

    def __getstate___(self):
        return self.lbl, self.lambdaa, self.sigma2, self.kern, self.x_shape

        lbl, lambdaa, sigma2, kern, a, A, x_shape = state

class DGPLVM(Prior):
    """
    Implementation of the Discriminative Gaussian Process Latent Variable model paper, by Raquel.

    .. Note:: DGPLVM for Classification paper implementation

    """
    # def __new__(cls, mu, sigma): # Singleton:

    def __init__(self, sigma2, lbl, x_shape):
        # self.x = x
        self.dim = x_shape[1]

            if v == 1:
                return idx
        return -1

        cls = {}
        return cls

    # This function computes mean of each class. The mean is calculated through each dimension
    def compute_Mi(self, cls):
        M_i = np.zeros((self.classnum, self.dim))
        for i in cls:
            # Mean of each class
            M_i[i] = np.mean(cls[i], axis=0)
        return M_i

    # Adding data points as tuple to the dictionary so that we can access indices
    def compute_indices(self, x):
        data_idx = {}
            if class_label not in data_idx:
                data_idx[class_label] = []
            t = (j, x[j])
            data_idx[class_label].append(t)
        return data_idx

    # Adding indices to the list so we can access whole the indices
    def compute_listIndices(self, data_idx):
        lst_idx = []
        lst_idx_all = []
        for i in data_idx:
            if len(lst_idx) == 0:
                pass
                #Do nothing, because it is the first time list is created so is empty
            else:
                lst_idx = []
            # Here we put indices of each class in to the list called lst_idx_all
            for m in xrange(len(data_idx[i])):
                lst_idx.append(data_idx[i][m][0])
            lst_idx_all.append(lst_idx)
        return lst_idx_all

    # This function calculates between classes variances
    def compute_Sb(self, cls, M_i, M_0):
        Sb = np.zeros((self.dim, self.dim))
            B = (M_i[i] - M_0).reshape(self.dim, 1)
            B_trans = B.transpose()
            Sb += (float(len(cls[i])) / self.datanum) * B.dot(B_trans)
        return Sb

    # This function calculates within classes variances
    def compute_Sw(self, cls, M_i):
        Sw = np.zeros((self.dim, self.dim))
            N_i = float(len(cls[i]))
            W_WT = np.zeros((self.dim, self.dim))
            for xk in cls[i]:
                W = (xk - M_i[i])
                W_WT += np.outer(W, W)
            Sw += (N_i / self.datanum) * ((1. / N_i) * W_WT)
        return Sw

    # Calculating beta and Bi for Sb
    def compute_sig_beta_Bi(self, data_idx, M_i, M_0, lst_idx_all):
        # import pdb
        # pdb.set_trace()
        B_i = np.zeros((self.classnum, self.dim))
        Sig_beta_B_i_all = np.zeros((self.datanum, self.dim))
            # pdb.set_trace()
            # Calculating Bi
            B_i[i] = (M_i[i] - M_0).reshape(1, self.dim)
        for k in xrange(self.datanum):
            for i in data_idx:
                N_i = float(len(data_idx[i]))
                if k in lst_idx_all[i]:
                    beta = (float(1) / N_i) - (float(1) / self.datanum)
                    Sig_beta_B_i_all[k] += float(N_i) / self.datanum * (beta * B_i[i])
                else:
                    beta = -(float(1) / self.datanum)
        Sig_beta_B_i_all = Sig_beta_B_i_all.transpose()
        return Sig_beta_B_i_all

    # Calculating W_j s separately so we can access all the W_j s anytime
    def compute_wj(self, data_idx, M_i):
        W_i = np.zeros((self.datanum, self.dim))
            N_i = float(len(data_idx[i]))
            for tpl in data_idx[i]:
                xj = tpl[1]
                j = tpl[0]
                W_i[j] = (xj - M_i[i])
        return W_i

    # Calculating alpha and Wj for Sw
    def compute_sig_alpha_W(self, data_idx, lst_idx_all, W_i):
        Sig_alpha_W_i = np.zeros((self.datanum, self.dim))
                k = tpl[0]
                for j in lst_idx_all[i]:
                    if k == j:
                        alpha = 1 - (float(1) / N_i)
                        Sig_alpha_W_i[k] += (alpha * W_i[j])
                    else:
                        alpha = 0 - (float(1) / N_i)
        Sig_alpha_W_i = (1. / self.datanum) * np.transpose(Sig_alpha_W_i)
        return Sig_alpha_W_i

    # This function calculates log of our prior
        cls = self.compute_cls(x)
        M_0 = np.mean(x, axis=0)
        M_i = self.compute_Mi(cls)
        Sb = self.compute_Sb(cls, M_i, M_0)
        Sw = self.compute_Sw(cls, M_i)
        # Sb_inv_N = np.linalg.inv(Sb + np.eye(Sb.shape[0]) * (np.diag(Sb).min() * 0.1))
        #Sb_inv_N = np.linalg.inv(Sb+np.eye(Sb.shape[0])*0.1)
        Sb_inv_N = pdinv(Sb+ np.eye(Sb.shape[0]) * (np.diag(Sb).min() * 0.1))[0]
        return (-1 / self.sigma2) * np.trace(Sb_inv_N.dot(Sw))

    # This function calculates derivative of the log of prior function
        data_idx = self.compute_indices(x)
        lst_idx_all = self.compute_listIndices(data_idx)
        Sig_beta_B_i_all = self.compute_sig_beta_Bi(data_idx, M_i, M_0, lst_idx_all)
        W_i = self.compute_wj(data_idx, M_i)
        Sig_alpha_W_i = self.compute_sig_alpha_W(data_idx, lst_idx_all, W_i)

        # Calculating inverse of Sb and its transpose and minus
        # Sb_inv_N = np.linalg.inv(Sb+np.eye(Sb.shape[0])*0.1)
        Sb_inv_N_trans = np.transpose(Sb_inv_N)
        Sb_inv_N_trans_minus = -1 * Sb_inv_N_trans
        Sw_trans = np.transpose(Sw)

        # Calculating DJ/DXk
        DJ_Dxk = 2 * (
            Sb_inv_N_trans_minus.dot(Sw_trans).dot(Sb_inv_N_trans).dot(Sig_beta_B_i_all) + Sb_inv_N_trans.dot(
                Sig_alpha_W_i))
        # Calculating derivative of the log of the prior
        DPx_Dx = ((-1 / self.sigma2) * DJ_Dxk)
        return DPx_Dx.T

    # def frb(self, x):
    #     from functools import partial
    #     from GPy.models import GradientChecker
    #     f = partial(self.lnpdf)
    #     df = partial(self.lnpdf_grad)
    #     grad = GradientChecker(f, df, x, 'X')
    #     grad.checkgrad(verbose=1)

class HalfT(Prior):
    """
    Implementation of the half student t probability function, coupled with random variables.

    :param A: scale parameter
    :param nu: degrees of freedom

    """
    def __new__(cls, A, nu): # Singleton:
                if instance().A == A and instance().nu == nu:
                   return instance()
        o = super(Prior, cls).__new__(cls, A, nu)
    def __init__(self, A, nu):
        self.A = float(A)
        self.nu = float(nu)
        self.constant = gammaln(.5*(self.nu+1.)) - gammaln(.5*self.nu) - .5*np.log(np.pi*self.A*self.nu)

        return "hT({:.2g}, {:.2g})".format(self.A, self.nu)

    def lnpdf(self,theta):
        return (theta>0) * ( self.constant -.5*(self.nu+1) * np.log( 1.+ (1./self.nu) * (theta/self.A)**2 ) )

        #theta = theta if isinstance(theta,np.ndarray) else np.array([theta])
        #lnpdfs = np.zeros_like(theta)
        #theta = np.array([theta])
        #above_zero = theta.flatten()>1e-6
        #v = self.nu
        #sigma2=self.A
        #stop
        #lnpdfs[above_zero] = (+ gammaln((v + 1) * 0.5)
        #    - gammaln(v * 0.5)
        #    - 0.5*np.log(sigma2 * v * np.pi)
        #    - 0.5*(v + 1)*np.log(1 + (1/np.float(v))*((theta[above_zero][0]**2)/sigma2))
        #)
        #return lnpdfs

    def lnpdf_grad(self,theta):
        theta = theta if isinstance(theta,np.ndarray) else np.array([theta])
        grad = np.zeros_like(theta)
        above_zero = theta>1e-6
        v = self.nu
        sigma2=self.A
        grad[above_zero] = -0.5*(v+1)*(2*theta[above_zero])/(v*sigma2 + theta[above_zero][0]**2)

         #return np.random.randn(n) * self.sigma + self.mu
         from scipy.stats import t
         #[np.abs(x) for x in t.rvs(df=4,loc=0,scale=50, size=10000)])
         ret = t.rvs(self.nu,loc=0,scale=self.A, size=n)
         ret[ret<0] = 0
         return ret

# Copyright (c) 2012, James Hensman and Nicolo' Fusi

def plot(model, ax=None, fignum=None, Z_height=None, **kwargs):

    #horrible hack here:
    data = model.likelihood.data.copy()
    model.likelihood.data = model.Y
    GP.plot(model, ax=ax, **kwargs)
    model.likelihood.data = data

    Zu = model.Z * model._Xscale + model._Xoffset
    if model.input_dim==1:
        ax.plot(model.X_batch, model.likelihood.data, 'gx',mew=2)
        if Z_height is None:
            Z_height = ax.get_ylim()[0]
        ax.plot(Zu, np.zeros_like(Zu) + Z_height, 'r|', mew=1.5, markersize=12)

    if model.input_dim==2:
        ax.scatter(model.X[:,0], model.X[:,1], 20., model.Y[:,0], linewidth=0, cmap=pb.cm.jet)  # @UndefinedVariable
        ax.plot(Zu[:,0], Zu[:,1], 'w^')

def plot_traces(model):

    t = np.array(model._param_trace)
    pb.subplot(2,1,1)
    for l,ti in zip(model._get_param_names(),t.T):
        if not l[:3]=='iip':
            pb.plot(ti,label=l)

    pb.subplot(2,1,2)
    pb.plot(np.asarray(model._ll_trace),label='stochastic likelihood')

from ....core.parameterization.parameter_core import Pickleable
from ....core.parameterization import variational
import rbf_psi_comp
import ssrbf_psi_comp
import sslinear_psi_comp
import linear_psi_comp

class PSICOMP_RBF(Pickleable):
    @Cache_this(limit=2, ignore_args=(0,))
        if isinstance(variational_posterior, variational.NormalPosterior):
            return rbf_psi_comp.psicomputations(variance, lengthscale, Z, variational_posterior)
        elif isinstance(variational_posterior, variational.SpikeAndSlabPosterior):
            return ssrbf_psi_comp.psicomputations(variance, lengthscale, Z, variational_posterior)
        else:
            raise ValueError, "unknown distriubtion received for psi-statistics"

    @Cache_this(limit=2, ignore_args=(0,1,2,3))
            return rbf_psi_comp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior)
            return ssrbf_psi_comp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, lengthscale, Z, variational_posterior)
        else:

        pass

class PSICOMP_Linear(Pickleable):

    def psicomputations(self, variance, Z, variational_posterior):
            return linear_psi_comp.psicomputations(variance, Z, variational_posterior)
            return sslinear_psi_comp.psicomputations(variance, Z, variational_posterior)
        else:

    def psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior):
            return linear_psi_comp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior)
            return sslinear_psi_comp.psiDerivativecomputations(dL_dpsi0, dL_dpsi1, dL_dpsi2, variance, Z, variational_posterior)
        else:

        pass

import GPy
import pkgutil
import os
import random
from nose.tools import nottest
import sys

class ExamplesTests(unittest.TestCase):
    def _checkgrad(self, Model):
        self.assertTrue(Model.checkgrad())

    def _model_instance(self, Model):
        self.assertTrue(isinstance(Model, GPy.models))

def model_checkgrads(model):
    model.randomize()
    #NOTE: Step as 1e-4, this should be acceptable for more peaky models
    return model.checkgrad(step=1e-4)

def model_instance(model):
    return isinstance(model, GPy.core.model.Model)

def flatten_nested(lst):
    result = []
    for element in lst:
        if hasattr(element, '__iter__'):
            result.extend(flatten_nested(element))
        else:
            result.append(element)
    return result

@nottest
def test_models():
    optimize=False
    plot=True
    examples_path = os.path.dirname(GPy.examples.__file__)
    # Load modules
    failing_models = {}
    for loader, module_name, is_pkg in pkgutil.iter_modules([examples_path]):
        # Load examples
        module_examples = loader.find_module(module_name).load_module(module_name)
        print "MODULE", module_examples
        print "Before"
        print inspect.getmembers(module_examples, predicate=inspect.isfunction)
        functions = [ func for func in inspect.getmembers(module_examples, predicate=inspect.isfunction) if func[0].startswith('_') is False ][::-1]
        print "After"
        print functions
        for example in functions:
            if example[0] in ['epomeo_gpx']:
                #These are the edge cases that we might want to handle specially
                if example[0] == 'epomeo_gpx' and not GPy.util.datasets.gpxpy_available:
                    print "Skipping as gpxpy is not available to parse GPS"
                    continue

            print "Testing example: ", example[0]
            # Generate model

            try:
                models = [ example[1](optimize=optimize, plot=plot) ]
                #If more than one model returned, flatten them
                models = flatten_nested(models)
                failing_models[example[0]] = "Cannot make model: \n{e}".format(e=e)
            else:
                print models
                model_checkgrads.description = 'test_checkgrads_%s' % example[0]
                try:
                    for model in models:
                        if not model_checkgrads(model):
                            failing_models[model_checkgrads.description] = False
                except Exception as e:
                    failing_models[model_checkgrads.description] = e

                model_instance.description = 'test_instance_%s' % example[0]
                try:
                        if not model_instance(model):
                            failing_models[model_instance.description] = False
                    failing_models[model_instance.description] = e

            #yield model_checkgrads, model
            #yield model_instance, model

        print "Finished checking module {m}".format(m=module_name)
        if len(failing_models.keys()) > 0:
            print "Failing models: "
            print failing_models

    if len(failing_models.keys()) > 0:
        print failing_models
        raise Exception(failing_models)

    # unittest.main()
    test_models()

class Prod(CombinationKernel):
    """
    Computes the product of 2 kernels

    :param k1, k2: the kernels to multiply
    :type k1, k2: Kern
    :param tensor: The kernels are either multiply as functions defined on the same input space (default) or on the product of the input spaces
    :type tensor: Boolean

    """
    def __init__(self, kernels, name='mul'):
        for i, kern in enumerate(kernels[:]):
            if isinstance(kern, Prod):
                del kernels[i]
                    kernels.insert(i, part)
        super(Prod, self).__init__(kernels, name)

        return reduce(np.multiply, (p.K(X, X2) for p in which_parts))

        return reduce(np.multiply, (p.Kdiag(X) for p in which_parts))

        k = self.K(X,X2)*dL_dK
        for p in self.parts:
            p.update_gradients_full(k/p.K(X,X2),X,X2)

        k = self.Kdiag(X)*dL_dKdiag
            p.update_gradients_diag(k/p.Kdiag(X),X)

            target += p.gradients_X(k/p.K(X,X2),X,X2)

            target += p.gradients_X_diag(k/p.Kdiag(X),X)

class ODE_st(Kern):
    """
    kernel resultiong from a first order ODE with OU driving GP

    :param input_dim: the number of input dimension, has to be equal to one
    :param varianceU: variance of the driving GP
    :type varianceU: float
    :param lengthscaleU: lengthscale of the driving GP  (sqrt(3)/lengthscaleU)
    :type lengthscaleU: float
    :param varianceY: 'variance' of the transfer function
    :type varianceY: float
    :param lengthscaleY: 'lengthscale' of the transfer function (1/lengthscaleY)
    :type lengthscaleY: float

    """
    
    def __init__(self, input_dim, a=1.,b=1., c=1.,variance_Yx=3.,variance_Yt=1.5, lengthscale_Yx=1.5, lengthscale_Yt=1.5, active_dims=None, name='ode_st'):
        assert input_dim ==3, "only defined for 3 input dims"
        super(ODE_st, self).__init__(input_dim, active_dims, name)

        self.variance_Yt = Param('variance_Yt', variance_Yt, Logexp())
        self.variance_Yx = Param('variance_Yx', variance_Yx, Logexp())
        self.lengthscale_Yt = Param('lengthscale_Yt', lengthscale_Yt, Logexp())
        self.lengthscale_Yx = Param('lengthscale_Yx', lengthscale_Yx, Logexp())        

        self.a= Param('a', a, Logexp())
        self.b = Param('b', b, Logexp())
        self.c = Param('c', c, Logexp())

        self.add_parameters(self.a, self.b, self.c, self.variance_Yt, self.variance_Yx, self.lengthscale_Yt,self.lengthscale_Yx)

    def K(self, X, X2=None):        
    # model :   -a d^2y/dx^2  + b dy/dt + c * y = U
    # kernel Kyy rbf spatiol temporal
    # vyt Y temporal variance  vyx Y spatiol variance   lyt Y temporal lengthscale   lyx Y spatiol lengthscale
    # kernel Kuu doper( doper(Kyy))
    # a   b    c    lyt   lyx    vyx*vyt
        """Compute the covariance matrix between X and X2."""        
        else:

        tdist = (X[:,0][:,None] - X2[:,0][None,:])**2
        xdist = (X[:,1][:,None] - X2[:,1][None,:])**2

        ttdist = (X[:,0][:,None] - X2[:,0][None,:])
        #rdist = [tdist,xdist]
        #dist = np.abs(X - X2.T)
        vyt = self.variance_Yt
        vyx = self.variance_Yx
        
        lyt=1/(2*self.lengthscale_Yt)
        lyx=1/(2*self.lengthscale_Yx)

        a = self.a ## -a is used in the model, negtive diffusion
        b = self.b
        c = self.c

        kyy = lambda tdist,xdist: np.exp(-lyt*(tdist) -lyx*(xdist))

        k1 = lambda tdist: (2*lyt - 4*lyt**2 * (tdist) )

        k2 = lambda xdist: ( 4*lyx**2 * (xdist)  - 2*lyx )

        k3 = lambda xdist: ( 3*4*lyx**2 - 6*8*xdist*lyx**3 + 16*xdist**2*lyx**4 )

        k4 = lambda ttdist: 2*lyt*(ttdist)

                            K[ss1,ss2] = vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
                            K[ss1,ss2] = (-a*k2(xdist[ss1,ss2]) + b*k4(ttdist[ss1,ss2]) + c)*vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
                            #K[ss1,ss2]=  np.where(  rdist[ss1,ss2]>0 , kuyp(rdist[ss1,ss2]), kuyn(rdist[ss1,ss2] )   )
                            K[ss1,ss2] = ( b**2*k1(tdist[ss1,ss2]) - 2*a*c*k2(xdist[ss1,ss2]) + a**2*k3(xdist[ss1,ss2]) + c**2 )* vyt*vyx* kyy(tdist[ss1,ss2],xdist[ss1,ss2])
                        else:
                            K[ss1,ss2] = (-a*k2(xdist[ss1,ss2]) - b*k4(ttdist[ss1,ss2]) + c)*vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
                            #K[ss1,ss2] = np.where(  rdist[ss1,ss2]>0 , kyup(rdist[ss1,ss2]), kyun(rdist[ss1,ss2] )   )
        
        #stop
        return K

        lyt = 1./(2*self.lengthscale_Yt)
        lyx = 1./(2*self.lengthscale_Yx)

        a = self.a
        b = self.b
        c = self.c

        ## dk^2/dtdt'
        k1 = (2*lyt )*vyt*vyx
        ## dk^2/dx^2
        k2 = ( - 2*lyx )*vyt*vyx
        ## dk^4/dx^2dx'^2
        k3 = ( 4*3*lyx**2 )*vyt*vyx

                if i==0:
                    Kdiag[s1]+= vyt*vyx
                elif i==1:
                    #i=1
                    Kdiag[s1]+= b**2*k1 - 2*a*c*k2 + a**2*k3 + c**2*vyt*vyx
                    #Kdiag[s1]+= Vu*Vy*(k1+k2+k3)
                else:

    #def dK_dtheta(self, dL_dK, X, X2, target):
        else:
        
        a = self.a
        b = self.b
        c = self.c

        rd=tdist.shape[0]

        dka = np.zeros([rd,rd])
        dkb = np.zeros([rd,rd])
        dkc = np.zeros([rd,rd])
        dkYdvart = np.zeros([rd,rd])
        dkYdvarx = np.zeros([rd,rd])
        dkYdlent = np.zeros([rd,rd])
        dkYdlenx = np.zeros([rd,rd])

        #k1 = lambda tdist: (lyt - lyt**2 * (tdist) )
        #k2 = lambda xdist: ( lyx**2 * (xdist)  - lyx )
        #k3 = lambda xdist: ( 3*lyx**2 - 6*xdist*lyx**3 + xdist**2*lyx**4 )
        #k4 = lambda tdist: -lyt*np.sqrt(tdist)

        dkyydlyx = lambda tdist,xdist: kyy(tdist,xdist)*(-xdist)
        dkyydlyt = lambda tdist,xdist: kyy(tdist,xdist)*(-tdist)

        dk1dlyt = lambda tdist: 2. - 4*2.*lyt*tdist
        dk2dlyx = lambda xdist: (4.*2.*lyx*xdist -2.)
        dk3dlyx = lambda xdist: (6.*4.*lyx - 18.*8*xdist*lyx**2 + 4*16*xdist**2*lyx**3)

        dk4dlyt = lambda ttdist: 2*(ttdist)

                            dka[ss1,ss2] = 0
                            dkb[ss1,ss2] = 0
                            dkc[ss1,ss2] = 0
                            dkYdvart[ss1,ss2] = vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
                            dkYdvarx[ss1,ss2] = vyt*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
                            dkYdlenx[ss1,ss2] = vyt*vyx*dkyydlyx(tdist[ss1,ss2],xdist[ss1,ss2])
                            dkYdlent[ss1,ss2] = vyt*vyx*dkyydlyt(tdist[ss1,ss2],xdist[ss1,ss2])
                            dka[ss1,ss2] = -k2(xdist[ss1,ss2])*vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
                            dkb[ss1,ss2] = k4(ttdist[ss1,ss2])*vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
                            dkc[ss1,ss2] = vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
                            #dkYdvart[ss1,ss2] = 0
                            #dkYdvarx[ss1,ss2] = 0
                            #dkYdlent[ss1,ss2] = 0
                            #dkYdlenx[ss1,ss2] = 0
                            dkYdvart[ss1,ss2] = (-a*k2(xdist[ss1,ss2])+b*k4(ttdist[ss1,ss2])+c)*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
                            dkYdvarx[ss1,ss2] = (-a*k2(xdist[ss1,ss2])+b*k4(ttdist[ss1,ss2])+c)*vyt*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
                            dkYdlent[ss1,ss2] = vyt*vyx*dkyydlyt(tdist[ss1,ss2],xdist[ss1,ss2])* (-a*k2(xdist[ss1,ss2])+b*k4(ttdist[ss1,ss2])+c)+\
                            vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])*b*dk4dlyt(ttdist[ss1,ss2])
                            dkYdlenx[ss1,ss2] = vyt*vyx*dkyydlyx(tdist[ss1,ss2],xdist[ss1,ss2])*(-a*k2(xdist[ss1,ss2])+b*k4(ttdist[ss1,ss2])+c)+\
                            vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])*(-a*dk2dlyx(xdist[ss1,ss2]))
                            dka[ss1,ss2] = (2*a*k3(xdist[ss1,ss2]) - 2*c*k2(xdist[ss1,ss2]))*vyt*vyx* kyy(tdist[ss1,ss2],xdist[ss1,ss2])
                            dkb[ss1,ss2] = 2*b*k1(tdist[ss1,ss2])*vyt*vyx* kyy(tdist[ss1,ss2],xdist[ss1,ss2])
                            dkc[ss1,ss2] = (-2*a*k2(xdist[ss1,ss2]) + 2*c )*vyt*vyx* kyy(tdist[ss1,ss2],xdist[ss1,ss2])
                            dkYdvart[ss1,ss2] = ( b**2*k1(tdist[ss1,ss2]) - 2*a*c*k2(xdist[ss1,ss2]) + a**2*k3(xdist[ss1,ss2]) + c**2 )*vyx* kyy(tdist[ss1,ss2],xdist[ss1,ss2])
                            dkYdvarx[ss1,ss2] = ( b**2*k1(tdist[ss1,ss2]) - 2*a*c*k2(xdist[ss1,ss2]) + a**2*k3(xdist[ss1,ss2]) + c**2 )*vyt* kyy(tdist[ss1,ss2],xdist[ss1,ss2])
                            dkYdlent[ss1,ss2] = vyt*vyx*dkyydlyt(tdist[ss1,ss2],xdist[ss1,ss2])*( b**2*k1(tdist[ss1,ss2]) - 2*a*c*k2(xdist[ss1,ss2]) + a**2*k3(xdist[ss1,ss2]) + c**2 ) +\
                            vyx*vyt*kyy(tdist[ss1,ss2],xdist[ss1,ss2])*b**2*dk1dlyt(tdist[ss1,ss2])
                            dkYdlenx[ss1,ss2] = vyt*vyx*dkyydlyx(tdist[ss1,ss2],xdist[ss1,ss2])*( b**2*k1(tdist[ss1,ss2]) - 2*a*c*k2(xdist[ss1,ss2]) + a**2*k3(xdist[ss1,ss2]) + c**2 ) +\
                            vyx*vyt*kyy(tdist[ss1,ss2],xdist[ss1,ss2])* (-2*a*c*dk2dlyx(xdist[ss1,ss2]) + a**2*dk3dlyx(xdist[ss1,ss2]) )
                        else:
                            dkb[ss1,ss2] = -k4(ttdist[ss1,ss2])*vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
                            dkYdvart[ss1,ss2] = (-a*k2(xdist[ss1,ss2])-b*k4(ttdist[ss1,ss2])+c)*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
                            dkYdvarx[ss1,ss2] = (-a*k2(xdist[ss1,ss2])-b*k4(ttdist[ss1,ss2])+c)*vyt*kyy(tdist[ss1,ss2],xdist[ss1,ss2])
                            dkYdlent[ss1,ss2] = vyt*vyx*dkyydlyt(tdist[ss1,ss2],xdist[ss1,ss2])* (-a*k2(xdist[ss1,ss2])-b*k4(ttdist[ss1,ss2])+c)+\
                            vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])*(-1)*b*dk4dlyt(ttdist[ss1,ss2])
                            dkYdlenx[ss1,ss2] = vyt*vyx*dkyydlyx(tdist[ss1,ss2],xdist[ss1,ss2])*(-a*k2(xdist[ss1,ss2])-b*k4(ttdist[ss1,ss2])+c)+\
                            vyt*vyx*kyy(tdist[ss1,ss2],xdist[ss1,ss2])*(-a*dk2dlyx(xdist[ss1,ss2])) 

        self.a.gradient = np.sum(dka * dL_dK)  

        self.b.gradient = np.sum(dkb * dL_dK) 

        self.c.gradient = np.sum(dkc * dL_dK)

        self.variance_Yt.gradient = np.sum(dkYdvart * dL_dK)  # Vy

        self.variance_Yx.gradient = np.sum(dkYdvarx * dL_dK)

        self.lengthscale_Yt.gradient = np.sum(dkYdlent*(-0.5*self.lengthscale_Yt**(-2)) * dL_dK)    #ly np.sum(dktheta2*(-self.lengthscale_Y**(-2)) * dL_dK) 

        self.lengthscale_Yx.gradient =  np.sum(dkYdlenx*(-0.5*self.lengthscale_Yx**(-2)) * dL_dK)

align_subplot_array() (in module GPy.plotting.matplot_dep.base_plots)

align_subplots() (in module GPy.plotting.matplot_dep.base_plots)

removeRightTicks() (in module GPy.plotting.matplot_dep.base_plots)

removeUpperTicks() (in module GPy.plotting.matplot_dep.base_plots)

most_significant_input_dimensions() (in module GPy.plotting.matplot_dep.dim_reduction_plots)

plot_latent() (in module GPy.plotting.matplot_dep.dim_reduction_plots)

plot_magnification() (in module GPy.plotting.matplot_dep.dim_reduction_plots)

plot_steepest_gradient_map() (in module GPy.plotting.matplot_dep.dim_reduction_plots)

plot_optimizer() (in module GPy.plotting.matplot_dep.inference_plots)

plot_sgd_traces() (in module GPy.plotting.matplot_dep.inference_plots)

add_bar_labels() (in module GPy.plotting.matplot_dep.kernel_plots)

plot_mapping() (in module GPy.plotting.matplot_dep.mapping_plots)

asarray() (GPy.plotting.matplot_dep.netpbmfile.NetpbmFile method)

univariate_plot() (in module GPy.plotting.matplot_dep.priors_plots)

plot_inducing() (GPy.plotting.matplot_dep.ssgplvm.SSGPLVM_plot method)

plot_SpikeSlab() (in module GPy.plotting.matplot_dep.variational_plots)

set_image() (GPy.plotting.matplot_dep.visualize.image_show method)

show_sensitivities() (GPy.plotting.matplot_dep.visualize.lvm method)

on_click() (GPy.plotting.matplot_dep.visualize.lvm_dimselect method)

on_leave() (GPy.plotting.matplot_dep.visualize.lvm_dimselect method)

close() (GPy.plotting.matplot_dep.visualize.matplotlib_show method)

draw_edges() (GPy.plotting.matplot_dep.visualize.mocap_data_show method)

draw_vertices() (GPy.plotting.matplot_dep.visualize.mocap_data_show method)

finalize_axes() (GPy.plotting.matplot_dep.visualize.mocap_data_show method)

initialize_axes() (GPy.plotting.matplot_dep.visualize.mocap_data_show method)

modify() (GPy.plotting.matplot_dep.visualize.mocap_data_show method)

process_values() (GPy.plotting.matplot_dep.visualize.mocap_data_show method)

mocap_data_show_vpython (class in GPy.plotting.matplot_dep.visualize)

draw_edges() (GPy.plotting.matplot_dep.visualize.mocap_data_show_vpython method)

modify() (GPy.plotting.matplot_dep.visualize.mocap_data_show_vpython method)

pos_axis() (GPy.plotting.matplot_dep.visualize.mocap_data_show_vpython method)

process_values() (GPy.plotting.matplot_dep.visualize.skeleton_show method)

wrap_around() (GPy.plotting.matplot_dep.visualize.skeleton_show method)

process_values() (GPy.plotting.matplot_dep.visualize.stick_show method)

modify() (GPy.plotting.matplot_dep.visualize.vector_show method)

close() (GPy.plotting.matplot_dep.visualize.vpython_show method)

exact_inference_gradients() (GPy.likelihoods.bernoulli.Bernoulli method)

predictive_variance() (GPy.likelihoods.bernoulli.Bernoulli method)

d2logpdf_dlink2() (GPy.likelihoods.exponential.Exponential method)

d3logpdf_dlink3() (GPy.likelihoods.exponential.Exponential method)

dlogpdf_dlink() (GPy.likelihoods.exponential.Exponential method)

d2logpdf_dlink2_dtheta() (GPy.likelihoods.gaussian.Gaussian method)

d2logpdf_dlink2_dvar() (GPy.likelihoods.gaussian.Gaussian method)

dlogpdf_dlink_dtheta() (GPy.likelihoods.gaussian.Gaussian method)

dlogpdf_link_dtheta() (GPy.likelihoods.gaussian.Gaussian method)

exact_inference_gradients() (GPy.likelihoods.gaussian.Gaussian method)

log_predictive_density() (GPy.likelihoods.gaussian.Gaussian method)

predictive_quantiles() (GPy.likelihoods.gaussian.Gaussian method)

predictive_variance() (GPy.likelihoods.gaussian.Gaussian method)

conditional_mean() (GPy.likelihoods.likelihood.Likelihood method)

conditional_variance() (GPy.likelihoods.likelihood.Likelihood method)

d2logpdf_df2_dtheta() (GPy.likelihoods.likelihood.Likelihood method)

d2logpdf_dlink2() (GPy.likelihoods.likelihood.Likelihood method)

d2logpdf_dlink2_dtheta() (GPy.likelihoods.likelihood.Likelihood method)

d3logpdf_dlink3() (GPy.likelihoods.likelihood.Likelihood method)

dlogpdf_df_dtheta() (GPy.likelihoods.likelihood.Likelihood method)

dlogpdf_dlink_dtheta() (GPy.likelihoods.likelihood.Likelihood method)

dlogpdf_link_dtheta() (GPy.likelihoods.likelihood.Likelihood method)

log_predictive_density() (GPy.likelihoods.likelihood.Likelihood method)

predictive_mean() (GPy.likelihoods.likelihood.Likelihood method)

predictive_quantiles() (GPy.likelihoods.likelihood.Likelihood method)

predictive_values() (GPy.likelihoods.likelihood.Likelihood method)

predictive_variance() (GPy.likelihoods.likelihood.Likelihood method)

update_gradients() (GPy.likelihoods.likelihood.Likelihood method)

variational_expectations() (GPy.likelihoods.likelihood.Likelihood method)

d2transf_df2() (GPy.likelihoods.link_functions.GPTransformation method)

d3transf_df3() (GPy.likelihoods.link_functions.GPTransformation method)

dtransf_df() (GPy.likelihoods.link_functions.GPTransformation method)

transf() (GPy.likelihoods.link_functions.GPTransformation method)

d2transf_df2() (GPy.likelihoods.link_functions.Heaviside method)

d2transf_df2() (GPy.likelihoods.link_functions.Reciprocal method)

d3transf_df3() (GPy.likelihoods.link_functions.Reciprocal method)

exact_inference_gradients() (GPy.likelihoods.mixed_noise.MixedNoise method)

gaussian_variance() (GPy.likelihoods.mixed_noise.MixedNoise method)

predictive_quantiles() (GPy.likelihoods.mixed_noise.MixedNoise method)

predictive_values() (GPy.likelihoods.mixed_noise.MixedNoise method)

predictive_variance() (GPy.likelihoods.mixed_noise.MixedNoise method)

update_gradients() (GPy.likelihoods.mixed_noise.MixedNoise method)

conditional_variance() (GPy.likelihoods.student_t.StudentT method)

d2logpdf_dlink2_dtheta() (GPy.likelihoods.student_t.StudentT method)

d2logpdf_dlink2_dvar() (GPy.likelihoods.student_t.StudentT method)

dlogpdf_dlink_dtheta() (GPy.likelihoods.student_t.StudentT method)

dlogpdf_dlink_dvar() (GPy.likelihoods.student_t.StudentT method)

dlogpdf_link_dtheta() (GPy.likelihoods.student_t.StudentT method)

parameters_changed() (GPy.likelihoods.student_t.StudentT method)

predictive_variance() (GPy.likelihoods.student_t.StudentT method)

new_chain() (GPy.inference.mcmc.samplers.Metropolis_Hastings method)

predict() (GPy.inference.mcmc.samplers.Metropolis_Hastings method)

sample() (GPy.inference.mcmc.samplers.Metropolis_Hastings method)

Async_Optimize (class in GPy.inference.optimization.conjugate_gradient_descent)

CGD (class in GPy.inference.optimization.conjugate_gradient_descent)

opt() (GPy.inference.optimization.conjugate_gradient_descent.CGD method)

opt_async() (GPy.inference.optimization.conjugate_gradient_descent.CGD method)

opt_name (GPy.inference.optimization.conjugate_gradient_descent.CGD attribute)

FletcherReeves (class in GPy.inference.optimization.gradient_descent_update_rules)

GDUpdateRule (class in GPy.inference.optimization.gradient_descent_update_rules)

PolakRibiere (class in GPy.inference.optimization.gradient_descent_update_rules)

opt() (GPy.inference.optimization.optimization.Optimizer method)

plot() (GPy.inference.optimization.optimization.Optimizer method)

run() (GPy.inference.optimization.optimization.Optimizer method)

get_optimizer() (in module GPy.inference.optimization.optimization)

opt() (GPy.inference.optimization.optimization.opt_lbfgsb method)

opt() (GPy.inference.optimization.optimization.opt_simplex method)

SparseGPMissing (class in GPy.inference.optimization.stochastics)

SparseGPStochastics (class in GPy.inference.optimization.stochastics)

reset() (GPy.inference.optimization.stochastics.SparseGPStochastics method)

StochasticStorage (class in GPy.inference.optimization.stochastics)

reset() (GPy.inference.optimization.stochastics.StochasticStorage method)

test_index_view() (GPy.testing.index_operations_tests.Test method)

test_indexview_remove() (GPy.testing.index_operations_tests.Test method)

test_shift_left() (GPy.testing.index_operations_tests.Test method)

test_shift_right() (GPy.testing.index_operations_tests.Test method)

test_view_of_view() (GPy.testing.index_operations_tests.Test method)

genData() (GPy.testing.inference_tests.InferenceXTestCase method)

test_inferenceX_BGPLVM() (GPy.testing.inference_tests.InferenceXTestCase method)

test_inferenceX_GPLVM() (GPy.testing.inference_tests.InferenceXTestCase method)

setUp() (GPy.testing.kernel_tests.Coregionalize_weave_test method)

test_nonsym() (GPy.testing.kernel_tests.Coregionalize_weave_test method)

test_sym() (GPy.testing.kernel_tests.Coregionalize_weave_test method)

parameters_changed() (GPy.testing.kernel_tests.Kern_check_dK_dX method)

parameters_changed() (GPy.testing.kernel_tests.Kern_check_dK_dtheta method)

log_likelihood() (GPy.testing.kernel_tests.Kern_check_dKdiag_dX method)

parameters_changed() (GPy.testing.kernel_tests.Kern_check_dKdiag_dX method)

log_likelihood() (GPy.testing.kernel_tests.Kern_check_dKdiag_dtheta method)

parameters_changed() (GPy.testing.kernel_tests.Kern_check_dKdiag_dtheta method)

is_positive_semi_definite() (GPy.testing.kernel_tests.Kern_check_model method)

log_likelihood() (GPy.testing.kernel_tests.Kern_check_model method)

KernelGradientTestsContinuous (class in GPy.testing.kernel_tests)

setUp() (GPy.testing.kernel_tests.KernelGradientTestsContinuous method)

test_Add() (GPy.testing.kernel_tests.KernelGradientTestsContinuous method)

test_Add_dims() (GPy.testing.kernel_tests.KernelGradientTestsContinuous method)

test_Linear() (GPy.testing.kernel_tests.KernelGradientTestsContinuous method)

test_Matern32() (GPy.testing.kernel_tests.KernelGradientTestsContinuous method)

test_Matern52() (GPy.testing.kernel_tests.KernelGradientTestsContinuous method)

test_Prod() (GPy.testing.kernel_tests.KernelGradientTestsContinuous method)

test_Prod2() (GPy.testing.kernel_tests.KernelGradientTestsContinuous method)

test_Prod3() (GPy.testing.kernel_tests.KernelGradientTestsContinuous method)

test_RBF() (GPy.testing.kernel_tests.KernelGradientTestsContinuous method)

setUp() (GPy.testing.kernel_tests.KernelTestsMiscellaneous method)

test_which_parts() (GPy.testing.kernel_tests.KernelTestsMiscellaneous method)

setUp() (GPy.testing.kernel_tests.KernelTestsNonContinuous method)

test_Hierarchical() (GPy.testing.kernel_tests.KernelTestsNonContinuous method)

test_ODE_UY() (GPy.testing.kernel_tests.KernelTestsNonContinuous method)

check_kernel_gradient_functions() (in module GPy.testing.kernel_tests)

test_laplace_log_likelihood() (GPy.testing.likelihood_tests.LaplaceTests method)

t_d2logpdf2_df2_dparams() (GPy.testing.likelihood_tests.TestNoiseModels method)

t_d2logpdf_df2() (GPy.testing.likelihood_tests.TestNoiseModels method)

t_d2logpdf_dlink2() (GPy.testing.likelihood_tests.TestNoiseModels method)

t_d3logpdf_df3() (GPy.testing.likelihood_tests.TestNoiseModels method)

t_d3logpdf_dlink3() (GPy.testing.likelihood_tests.TestNoiseModels method)

t_dlogpdf_df() (GPy.testing.likelihood_tests.TestNoiseModels method)

t_dlogpdf_df_dparams() (GPy.testing.likelihood_tests.TestNoiseModels method)

t_dlogpdf_dlink() (GPy.testing.likelihood_tests.TestNoiseModels method)

t_dlogpdf_dlink_dparams() (GPy.testing.likelihood_tests.TestNoiseModels method)

t_dlogpdf_dparams() (GPy.testing.likelihood_tests.TestNoiseModels method)

t_dlogpdf_link_dparams() (GPy.testing.likelihood_tests.TestNoiseModels method)

t_ep_fit_rbf_white() (GPy.testing.likelihood_tests.TestNoiseModels method)

t_laplace_fit_rbf_white() (GPy.testing.likelihood_tests.TestNoiseModels method)

t_logpdf() (GPy.testing.likelihood_tests.TestNoiseModels method)

tearDown() (GPy.testing.likelihood_tests.TestNoiseModels method)

test_scale2_models() (GPy.testing.likelihood_tests.TestNoiseModels method)

test_GPRegression_bias_kern_1D() (GPy.testing.model_tests.GradientTests method)

test_GPRegression_bias_kern_2D() (GPy.testing.model_tests.GradientTests method)

test_GPRegression_matern32_1D() (GPy.testing.model_tests.GradientTests method)

test_GPRegression_matern32_2D() (GPy.testing.model_tests.GradientTests method)

test_GPRegression_matern52_1D() (GPy.testing.model_tests.GradientTests method)

test_GPRegression_matern52_2D() (GPy.testing.model_tests.GradientTests method)

test_GPRegression_mlp_1d() (GPy.testing.model_tests.GradientTests method)

test_GPRegression_rbf_1d() (GPy.testing.model_tests.GradientTests method)

test_GPRegression_rbf_2D() (GPy.testing.model_tests.GradientTests method)

test_GPRegression_rbf_ARD_2D() (GPy.testing.model_tests.GradientTests method)

test_GP_EP_probit() (GPy.testing.model_tests.GradientTests method)

test_generalized_FITC() (GPy.testing.model_tests.GradientTests method)

test_gp_kronecker_gaussian() (GPy.testing.model_tests.GradientTests method)

test_multioutput_regression_1D() (GPy.testing.model_tests.GradientTests method)

test_sparse_EP_DTC_probit() (GPy.testing.model_tests.GradientTests method)

test_likelihood_replicate() (GPy.testing.model_tests.MiscTests method)

test_likelihood_replicate_kern() (GPy.testing.model_tests.MiscTests method)

test_likelihood_set() (GPy.testing.model_tests.MiscTests method)

test_model_optimize() (GPy.testing.model_tests.MiscTests method)

test_model_set_params() (GPy.testing.model_tests.MiscTests method)

test_sparse_raw_predict() (GPy.testing.model_tests.MiscTests method)

parameters_changed() (GPy.testing.observable_tests.ParamTestParent method)

parent_changed_count (GPy.testing.observable_tests.ParamTestParent attribute)

parameters_changed() (GPy.testing.observable_tests.ParameterizedTest method)

params_changed_count (GPy.testing.observable_tests.ParameterizedTest attribute)

test_priority_notify() (GPy.testing.observable_tests.Test method)

test_init() (GPy.testing.parameterized_tests.ArrayCoreTest method)

test_slice() (GPy.testing.parameterized_tests.ArrayCoreTest method)

setUp() (GPy.testing.parameterized_tests.ParameterizedTest method)

test_add_parameter() (GPy.testing.parameterized_tests.ParameterizedTest method)

test_constraints() (GPy.testing.parameterized_tests.ParameterizedTest method)

test_fix_unfix() (GPy.testing.parameterized_tests.ParameterizedTest method)

test_fixes() (GPy.testing.parameterized_tests.ParameterizedTest method)

test_printing() (GPy.testing.parameterized_tests.ParameterizedTest method)

test_randomize() (GPy.testing.parameterized_tests.ParameterizedTest method)

test_updates() (GPy.testing.parameterized_tests.ParameterizedTest method)

assertArrayListEquals() (GPy.testing.pickle_tests.ListDictTestCase method)

assertListDictEquals() (GPy.testing.pickle_tests.ListDictTestCase method)

test_parameter_index_operations() (GPy.testing.pickle_tests.Test method)

test_fixed_domain_check() (GPy.testing.prior_tests.PriorTests method)

test_fixed_domain_check1() (GPy.testing.prior_tests.PriorTests method)

test_incompatibility() (GPy.testing.prior_tests.PriorTests method)

test_set_gaussian_for_reals() (GPy.testing.prior_tests.PriorTests method)

do_test_latents() (GPy.models.bayesian_gplvm.BayesianGPLVM method)

get_X_gradients() (GPy.models.bayesian_gplvm.BayesianGPLVM method)

parameters_changed() (GPy.models.bayesian_gplvm.BayesianGPLVM method)

plot_steepest_gradient_map() (GPy.models.bayesian_gplvm.BayesianGPLVM method)

set_X_gradients() (GPy.models.bayesian_gplvm.BayesianGPLVM method)

BayesianGPLVMMiniBatch (class in GPy.models.bayesian_gplvm_minibatch)

dmu_dX() (GPy.models.bayesian_gplvm_minibatch.BayesianGPLVMMiniBatch method)

dmu_dXnew() (GPy.models.bayesian_gplvm_minibatch.BayesianGPLVMMiniBatch method)

latent_cost_and_grad() (in module GPy.models.bayesian_gplvm_minibatch)

GPCoregionalizedRegression (class in GPy.models.gp_coregionalized_regression)

GPHeteroscedasticRegression (class in GPy.models.gp_heteroscedastic_regression)

GPKroneckerGaussianRegression (class in GPy.models.gp_kronecker_gaussian_regression)

GPVariationalGaussianApproximation (class in GPy.models.gp_var_gauss)

predict() (GPy.models.gp_var_gauss.GPVariationalGaussianApproximation method)

log_likelihood() (GPy.models.gradient_checker.GradientChecker method)

SparseGPClassification (class in GPy.models.sparse_gp_classification)

SparseGPCoregionalizedRegression (class in GPy.models.sparse_gp_coregionalized_regression)

has_uncertain_inputs() (GPy.models.sparse_gp_minibatch.SparseGPMiniBatch method)

parameters_changed() (GPy.models.sparse_gp_minibatch.SparseGPMiniBatch method)

parameters_changed() (GPy.models.sparse_gp_regression.SparseGPRegression method)

SparseGPRegressionUncertainInput (class in GPy.models.sparse_gp_regression)

parameters_changed() (GPy.models.sparse_gplvm.SparseGPLVM method)

warping_function_gradients() (GPy.models.warped_gp.WarpedGP method)

AxisChangedController (class in GPy.plotting.matplot_dep.latent_space_visualizations.controllers.axis_event_controller)

AxisEventController (class in GPy.plotting.matplot_dep.latent_space_visualizations.controllers.axis_event_controller)

ImAnnotateController (class in GPy.plotting.matplot_dep.latent_space_visualizations.controllers.imshow_controller)

ImshowController (class in GPy.plotting.matplot_dep.latent_space_visualizations.controllers.imshow_controller)

psiDerivativecomputations() (in module GPy.kern._src.psi_comp.linear_psi_comp)

psicomputations() (in module GPy.kern._src.psi_comp.linear_psi_comp)

psiDerivativecomputations() (in module GPy.kern._src.psi_comp.rbf_psi_comp)

psicomputations() (in module GPy.kern._src.psi_comp.rbf_psi_comp)

PSICOMP_RBF_GPU (class in GPy.kern._src.psi_comp.rbf_psi_gpucomp)

get_dimensions() (GPy.kern._src.psi_comp.rbf_psi_gpucomp.PSICOMP_RBF_GPU method)

sync_params() (GPy.kern._src.psi_comp.rbf_psi_gpucomp.PSICOMP_RBF_GPU method)

psiDerivativecomputations() (in module GPy.kern._src.psi_comp.sslinear_psi_comp)

psicomputations() (in module GPy.kern._src.psi_comp.sslinear_psi_comp)

psiDerivativecomputations() (in module GPy.kern._src.psi_comp.ssrbf_psi_comp)

PSICOMP_SSRBF_GPU (class in GPy.kern._src.psi_comp.ssrbf_psi_gpucomp)

psiDerivativecomputations (GPy.kern._src.psi_comp.PSICOMP_Linear attribute)

psicomputations (GPy.kern._src.psi_comp.PSICOMP_Linear attribute)

psiDerivativecomputations (GPy.kern._src.psi_comp.PSICOMP_RBF attribute)

ParameterIndexOperations (class in GPy.core.parameterization.index_operations)

ParameterIndexOperationsView (class in GPy.core.parameterization.index_operations)

combine_indices() (in module GPy.core.parameterization.index_operations)

extract_properties_to_index() (in module GPy.core.parameterization.index_operations)

index_empty() (in module GPy.core.parameterization.index_operations)

remove_indices() (in module GPy.core.parameterization.index_operations)

index() (GPy.core.parameterization.lists_and_dicts.ArrayList method)

IntArrayDict (class in GPy.core.parameterization.lists_and_dicts)

ObserverList (class in GPy.core.parameterization.lists_and_dicts)

add() (GPy.core.parameterization.lists_and_dicts.ObserverList method)

flush() (GPy.core.parameterization.lists_and_dicts.ObserverList method)

remove() (GPy.core.parameterization.lists_and_dicts.ObserverList method)

intarray_default_factory() (in module GPy.core.parameterization.lists_and_dicts)

copy() (GPy.core.parameterization.observable_array.ObsAr method)

values (GPy.core.parameterization.observable_array.ObsAr attribute)

flattened_parameters (GPy.core.parameterization.param.Param attribute)

parameter_names() (GPy.core.parameterization.param.Param method)

parameter_shapes (GPy.core.parameterization.param.Param attribute)

checkgrad() (GPy.core.parameterization.param.ParamConcatenation method)

constrain() (GPy.core.parameterization.param.ParamConcatenation method)

constrain_bounded() (GPy.core.parameterization.param.ParamConcatenation method)

constrain_fixed() (GPy.core.parameterization.param.ParamConcatenation method)

constrain_negative() (GPy.core.parameterization.param.ParamConcatenation method)

constrain_positive() (GPy.core.parameterization.param.ParamConcatenation method)

fix() (GPy.core.parameterization.param.ParamConcatenation method)

unconstrain() (GPy.core.parameterization.param.ParamConcatenation method)

unconstrain_fixed() (GPy.core.parameterization.param.ParamConcatenation method)

unfix() (GPy.core.parameterization.param.ParamConcatenation method)

untie() (GPy.core.parameterization.param.ParamConcatenation method)

update_all_params() (GPy.core.parameterization.param.ParamConcatenation method)

values() (GPy.core.parameterization.param.ParamConcatenation method)

Gradcheckable (class in GPy.core.parameterization.parameter_core)

checkgrad() (GPy.core.parameterization.parameter_core.Gradcheckable method)

constrain() (GPy.core.parameterization.parameter_core.Indexable method)

constrain_bounded() (GPy.core.parameterization.parameter_core.Indexable method)

constrain_fixed() (GPy.core.parameterization.parameter_core.Indexable method)

constrain_negative() (GPy.core.parameterization.parameter_core.Indexable method)

constrain_positive() (GPy.core.parameterization.parameter_core.Indexable method)

fix() (GPy.core.parameterization.parameter_core.Indexable method)

is_fixed (GPy.core.parameterization.parameter_core.Indexable attribute)

log_prior() (GPy.core.parameterization.parameter_core.Indexable method)

set_prior() (GPy.core.parameterization.parameter_core.Indexable method)

tie_together() (GPy.core.parameterization.parameter_core.Indexable method)

unconstrain() (GPy.core.parameterization.parameter_core.Indexable method)

unconstrain_fixed() (GPy.core.parameterization.parameter_core.Indexable method)

unfix() (GPy.core.parameterization.parameter_core.Indexable method)

unset_priors() (GPy.core.parameterization.parameter_core.Indexable method)

hierarchy_name() (GPy.core.parameterization.parameter_core.Nameable method)

name (GPy.core.parameterization.parameter_core.Nameable attribute)

OptimizationHandlable (class in GPy.core.parameterization.parameter_core)

Parameterizable (class in GPy.core.parameterization.parameter_core)

gradient (GPy.core.parameterization.parameter_core.Parameterizable attribute)

num_params (GPy.core.parameterization.parameter_core.Parameterizable attribute)

param_array (GPy.core.parameterization.parameter_core.Parameterizable attribute)

save() (GPy.core.parameterization.parameter_core.Parameterizable method)

traverse() (GPy.core.parameterization.parameter_core.Parameterizable method)

has_parent() (GPy.core.parameterization.parameter_core.Parentable method)

copy() (GPy.core.parameterization.parameter_core.Pickleable method)

pickle() (GPy.core.parameterization.parameter_core.Pickleable method)

adjust_name_for_printing() (in module GPy.core.parameterization.parameter_core)

Parameterized (class in GPy.core.parameterization.parameterized)

add_parameter() (GPy.core.parameterization.parameterized.Parameterized method)

build_pydot() (GPy.core.parameterization.parameterized.Parameterized method)

copy() (GPy.core.parameterization.parameterized.Parameterized method)

link_parameter() (GPy.core.parameterization.parameterized.Parameterized method)

link_parameters() (GPy.core.parameterization.parameterized.Parameterized method)

ParametersChangedMeta (class in GPy.core.parameterization.parameterized)

compute_indices() (GPy.core.parameterization.priors.DGPLVM method)

compute_listIndices() (GPy.core.parameterization.priors.DGPLVM method)

compute_sig_alpha_W() (GPy.core.parameterization.priors.DGPLVM method)

compute_sig_beta_Bi() (GPy.core.parameterization.priors.DGPLVM method)

get_class_label() (GPy.core.parameterization.priors.DGPLVM method)

compute_A() (GPy.core.parameterization.priors.DGPLVM_KFDA method)

compute_a() (GPy.core.parameterization.priors.DGPLVM_KFDA method)

compute_cls() (GPy.core.parameterization.priors.DGPLVM_KFDA method)

compute_lst_ni() (GPy.core.parameterization.priors.DGPLVM_KFDA method)

get_class_label() (GPy.core.parameterization.priors.DGPLVM_KFDA method)

lnpdf_grad() (GPy.core.parameterization.priors.DGPLVM_KFDA method)

x_reduced() (GPy.core.parameterization.priors.DGPLVM_KFDA method)

from_EV() (GPy.core.parameterization.priors.Gamma static method)

domain (GPy.core.parameterization.priors.InverseGamma attribute)

lnpdf_grad() (GPy.core.parameterization.priors.InverseGamma method)

lnpdf_grad() (GPy.core.parameterization.priors.LogGaussian method)

MultivariateGaussian (class in GPy.core.parameterization.priors)

domain (GPy.core.parameterization.priors.MultivariateGaussian attribute)

lnpdf() (GPy.core.parameterization.priors.MultivariateGaussian method)

lnpdf_grad() (GPy.core.parameterization.priors.MultivariateGaussian method)

pdf() (GPy.core.parameterization.priors.MultivariateGaussian method)

plot() (GPy.core.parameterization.priors.MultivariateGaussian method)

rvs() (GPy.core.parameterization.priors.MultivariateGaussian method)

summary() (GPy.core.parameterization.priors.MultivariateGaussian method)

Remapping (class in GPy.core.parameterization.ties_and_remappings)

callback() (GPy.core.parameterization.ties_and_remappings.Remapping method)

mapping() (GPy.core.parameterization.ties_and_remappings.Remapping method)

add_tied_parameter() (GPy.core.parameterization.ties_and_remappings.Tie method)

collate_gradient() (GPy.core.parameterization.ties_and_remappings.Tie method)

getTieFlag() (GPy.core.parameterization.ties_and_remappings.Tie method)

parameters_changed() (GPy.core.parameterization.ties_and_remappings.Tie method)

propagate_val() (GPy.core.parameterization.ties_and_remappings.Tie method)

domain (GPy.core.parameterization.transformations.Exponent attribute)

finv() (GPy.core.parameterization.transformations.Exponent method)

gradfactor() (GPy.core.parameterization.transformations.Exponent method)

initialize() (GPy.core.parameterization.transformations.Exponent method)

domain (GPy.core.parameterization.transformations.Logexp attribute)

finv() (GPy.core.parameterization.transformations.Logexp method)

gradfactor() (GPy.core.parameterization.transformations.Logexp method)

initialize() (GPy.core.parameterization.transformations.Logexp method)

LogexpClipped (class in GPy.core.parameterization.transformations)

domain (GPy.core.parameterization.transformations.LogexpClipped attribute)

f() (GPy.core.parameterization.transformations.LogexpClipped method)

finv() (GPy.core.parameterization.transformations.LogexpClipped method)

gradfactor() (GPy.core.parameterization.transformations.LogexpClipped method)

initialize() (GPy.core.parameterization.transformations.LogexpClipped method)

max_bound (GPy.core.parameterization.transformations.LogexpClipped attribute)

min_bound (GPy.core.parameterization.transformations.LogexpClipped attribute)

domain (GPy.core.parameterization.transformations.LogexpNeg attribute)

f() (GPy.core.parameterization.transformations.LogexpNeg method)

finv() (GPy.core.parameterization.transformations.LogexpNeg method)

gradfactor() (GPy.core.parameterization.transformations.LogexpNeg method)

initialize() (GPy.core.parameterization.transformations.LogexpNeg method)

domain (GPy.core.parameterization.transformations.Logistic attribute)

finv() (GPy.core.parameterization.transformations.Logistic method)

gradfactor() (GPy.core.parameterization.transformations.Logistic method)

initialize() (GPy.core.parameterization.transformations.Logistic method)

NegativeExponent (class in GPy.core.parameterization.transformations)

domain (GPy.core.parameterization.transformations.NegativeExponent attribute)

f() (GPy.core.parameterization.transformations.NegativeExponent method)

finv() (GPy.core.parameterization.transformations.NegativeExponent method)

gradfactor() (GPy.core.parameterization.transformations.NegativeExponent method)

initialize() (GPy.core.parameterization.transformations.NegativeExponent method)

NegativeLogexp (class in GPy.core.parameterization.transformations)

domain (GPy.core.parameterization.transformations.NegativeLogexp attribute)

f() (GPy.core.parameterization.transformations.NegativeLogexp method)

finv() (GPy.core.parameterization.transformations.NegativeLogexp method)

gradfactor() (GPy.core.parameterization.transformations.NegativeLogexp method)

initialize() (GPy.core.parameterization.transformations.NegativeLogexp method)

logexp (GPy.core.parameterization.transformations.NegativeLogexp attribute)

f() (GPy.core.parameterization.transformations.NormalEta method)

finv() (GPy.core.parameterization.transformations.NormalEta method)

gradfactor() (GPy.core.parameterization.transformations.NormalEta method)

initialize() (GPy.core.parameterization.transformations.NormalEta method)

NormalNaturalAntti (class in GPy.core.parameterization.transformations)

NormalNaturalThroughEta (class in GPy.core.parameterization.transformations)

NormalNaturalThroughTheta (class in GPy.core.parameterization.transformations)

NormalTheta (class in GPy.core.parameterization.transformations)

f() (GPy.core.parameterization.transformations.NormalTheta method)

finv() (GPy.core.parameterization.transformations.NormalTheta method)

gradfactor() (GPy.core.parameterization.transformations.NormalTheta method)

initialize() (GPy.core.parameterization.transformations.NormalTheta method)

domain (GPy.core.parameterization.transformations.Square attribute)

finv() (GPy.core.parameterization.transformations.Square method)

gradfactor() (GPy.core.parameterization.transformations.Square method)

initialize() (GPy.core.parameterization.transformations.Square method)

Transformation (class in GPy.core.parameterization.transformations)

domain (GPy.core.parameterization.transformations.Transformation attribute)

f() (GPy.core.parameterization.transformations.Transformation method)

finv() (GPy.core.parameterization.transformations.Transformation method)

gradfactor() (GPy.core.parameterization.transformations.Transformation method)

initialize() (GPy.core.parameterization.transformations.Transformation method)

plot() (GPy.core.parameterization.transformations.Transformation method)

NormalPosterior (class in GPy.core.parameterization.variational)

plot() (GPy.core.parameterization.variational.NormalPosterior method)

KL_divergence() (GPy.core.parameterization.variational.NormalPrior method)

update_gradients_KL() (GPy.core.parameterization.variational.NormalPrior method)

SpikeAndSlabPosterior (class in GPy.core.parameterization.variational)

plot() (GPy.core.parameterization.variational.SpikeAndSlabPosterior method)

SpikeAndSlabPrior (class in GPy.core.parameterization.variational)

KL_divergence() (GPy.core.parameterization.variational.SpikeAndSlabPrior method)

VariationalPosterior (class in GPy.core.parameterization.variational)

VariationalPrior (class in GPy.core.parameterization.variational)

KL_divergence() (GPy.core.parameterization.variational.VariationalPrior method)

inference() (GPy.inference.latent_function_inference.dtc.DTC method)

inference() (GPy.inference.latent_function_inference.dtc.vDTC method)

ExactGaussianInference (class in GPy.inference.latent_function_inference.exact_gaussian_inference)

EP (class in GPy.inference.latent_function_inference.expectation_propagation)

EPDTC (class in GPy.inference.latent_function_inference.expectation_propagation_dtc)

const_jitter (GPy.inference.latent_function_inference.fitc.FITC attribute)

inference() (GPy.inference.latent_function_inference.fitc.FITC method)

InferenceX (class in GPy.inference.latent_function_inference.inferenceX)

infer_newX() (in module GPy.inference.latent_function_inference.inferenceX)

Laplace (class in GPy.inference.latent_function_inference.laplace)

inference() (GPy.inference.latent_function_inference.laplace.Laplace method)

rasm_mode() (GPy.inference.latent_function_inference.laplace.Laplace method)

warning_on_one_line() (in module GPy.inference.latent_function_inference.laplace)

Posterior (class in GPy.inference.latent_function_inference.posterior)

K_chol (GPy.inference.latent_function_inference.posterior.Posterior attribute)

mean (GPy.inference.latent_function_inference.posterior.Posterior attribute)

VarDTC (class in GPy.inference.latent_function_inference.var_dtc)

const_jitter (GPy.inference.latent_function_inference.var_dtc.VarDTC attribute)

get_VVTfactor() (GPy.inference.latent_function_inference.var_dtc.VarDTC method)

inference() (GPy.inference.latent_function_inference.var_dtc.VarDTC method)

set_limit() (GPy.inference.latent_function_inference.var_dtc.VarDTC method)

VarDTC_minibatch (class in GPy.inference.latent_function_inference.var_dtc_parallel)

update_gradients() (in module GPy.inference.latent_function_inference.var_dtc_parallel)

update_gradients_sparsegp() (in module GPy.inference.latent_function_inference.var_dtc_parallel)

InferenceMethodList (class in GPy.inference.latent_function_inference)

LatentFunctionInference (class in GPy.inference.latent_function_inference)

della_gatta_TRP63_gene_expression() (in module GPy.util.datasets)

set_rotation_matrices() (GPy.util.mocap.acclaim_skeleton method)

fgrad_y() (GPy.util.warping_functions.TanhWarpingFunction method)

fgrad_y_psi() (GPy.util.warping_functions.TanhWarpingFunction method)

f_inv() (GPy.util.warping_functions.TanhWarpingFunction_d method)

fgrad_y() (GPy.util.warping_functions.TanhWarpingFunction_d method)

fgrad_y_psi() (GPy.util.warping_functions.TanhWarpingFunction_d method)

fgrad_y_psi() (GPy.util.warping_functions.WarpingFunction method)

sparse_toy_linear_1d_classification() (in module GPy.examples.classification)

toy_linear_1d_classification() (in module GPy.examples.classification)

toy_linear_1d_classification_laplace() (in module GPy.examples.classification)

bcgplvm_linear_stick() (in module GPy.examples.dimensionality_reduction)

bcgplvm_stick() (in module GPy.examples.dimensionality_reduction)

bgplvm_simulation() (in module GPy.examples.dimensionality_reduction)

bgplvm_simulation_missing_data() (in module GPy.examples.dimensionality_reduction)

bgplvm_test_model() (in module GPy.examples.dimensionality_reduction)

brendan_faces() (in module GPy.examples.dimensionality_reduction)

gplvm_oil_100() (in module GPy.examples.dimensionality_reduction)

mrd_simulation() (in module GPy.examples.dimensionality_reduction)

mrd_simulation_missing_data() (in module GPy.examples.dimensionality_reduction)

olivetti_faces() (in module GPy.examples.dimensionality_reduction)

robot_wireless() (in module GPy.examples.dimensionality_reduction)

sparse_gplvm_oil() (in module GPy.examples.dimensionality_reduction)

ssgplvm_simulation() (in module GPy.examples.dimensionality_reduction)

ssgplvm_simulation_linear() (in module GPy.examples.dimensionality_reduction)

stick_bgplvm() (in module GPy.examples.dimensionality_reduction)

toy_poisson_rbf_1d_laplace() (in module GPy.examples.regression)

uncertain_inputs_sparse_regression() (in module GPy.examples.regression)

parameters_changed() (GPy.core.sparse_gp_mpi.SparseGP_MPI method)

update_gradients_full() (GPy.kern._src.brownian.Brownian method)

gradients_X() (GPy.kern._src.coregionalize.Coregionalize method)

gradients_X_diag() (GPy.kern._src.coregionalize.Coregionalize method)

parameters_changed() (GPy.kern._src.coregionalize.Coregionalize method)

update_gradients_diag() (GPy.kern._src.coregionalize.Coregionalize method)

update_gradients_full() (GPy.kern._src.coregionalize.Coregionalize method)

gradients_X() (GPy.kern._src.independent_outputs.Hierarchical method)

update_gradients_full() (GPy.kern._src.independent_outputs.Hierarchical method)

K() (GPy.kern._src.independent_outputs.IndependentOutputs method)

Kdiag() (GPy.kern._src.independent_outputs.IndependentOutputs method)

gradients_X() (GPy.kern._src.independent_outputs.IndependentOutputs method)

gradients_X_diag() (GPy.kern._src.independent_outputs.IndependentOutputs method)

get_input_dim_active_dims() (GPy.kern._src.kern.CombinationKernel method)

input_sensitivity() (GPy.kern._src.kern.CombinationKernel method)

update_gradients_expectations() (GPy.kern._src.kern.Kern method)

KernCallsViaSlicerMeta (class in GPy.kern._src.kernel_slice_operations)

gradients_qX_expectations() (GPy.kern._src.linear.Linear method)

update_gradients_expectations() (GPy.kern._src.linear.Linear method)

update_gradients_diag() (GPy.kern._src.linear.LinearFull method)

update_gradients_full() (GPy.kern._src.linear.LinearFull method)

Gram_matrix() (GPy.kern._src.periodic.PeriodicExponential method)

parameters_changed() (GPy.kern._src.periodic.PeriodicExponential method)

update_gradients_full() (GPy.kern._src.periodic.PeriodicExponential method)

parameters_changed() (GPy.kern._src.periodic.PeriodicMatern32 method)

update_gradients_full() (GPy.kern._src.periodic.PeriodicMatern32 method)

parameters_changed() (GPy.kern._src.periodic.PeriodicMatern52 method)

update_gradients_full() (GPy.kern._src.periodic.PeriodicMatern52 method)

update_gradients_diag() (GPy.kern._src.splitKern.DiffGenomeKern method)

update_gradients_full() (GPy.kern._src.splitKern.DiffGenomeKern method)

update_gradients_diag() (GPy.kern._src.splitKern.SplitKern method)

update_gradients_full() (GPy.kern._src.splitKern.SplitKern method)

update_gradients_diag() (GPy.kern._src.splitKern.SplitKern_cross method)

update_gradients_full() (GPy.kern._src.splitKern.SplitKern_cross method)

update_gradients_expectations() (GPy.kern._src.static.Bias method)

update_gradients_expectations() (GPy.kern._src.static.Fixed method)

gradients_qX_expectations() (GPy.kern._src.static.Static method)

update_gradients_expectations() (GPy.kern._src.static.White method)

update_gradients_diag() (GPy.kern._src.stationary.RatQuad method)

update_gradients_full() (GPy.kern._src.stationary.RatQuad method)

gradients_X_weave() (GPy.kern._src.stationary.Stationary method)

input_sensitivity() (GPy.kern._src.stationary.Stationary method)

update_gradients_diag() (GPy.kern._src.stationary.Stationary method)

update_gradients_full() (GPy.kern._src.stationary.Stationary method)

weave_lengthscale_grads() (GPy.kern._src.stationary.Stationary method)

gradients_X_diag() (GPy.kern._src.trunclinear.TruncLinear method)

input_sensitivity() (GPy.kern._src.trunclinear.TruncLinear method)

update_gradients_diag() (GPy.kern._src.trunclinear.TruncLinear method)

update_gradients_full() (GPy.kern._src.trunclinear.TruncLinear method)

gradients_X() (GPy.kern._src.trunclinear.TruncLinear_inf method)

gradients_X_diag() (GPy.kern._src.trunclinear.TruncLinear_inf method)

input_sensitivity() (GPy.kern._src.trunclinear.TruncLinear_inf method)

update_gradients_diag() (GPy.kern._src.trunclinear.TruncLinear_inf method)

update_gradients_full() (GPy.kern._src.trunclinear.TruncLinear_inf method)