In [1]: 
from __future__ import division
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
In [2]: 
#Plotting functions
delta = 0.1
    
def plot_surface(f, x_min, x_max, y_min, y_max):
    
    x = np.arange(x_min, x_max + delta, delta)
    y = np.arange(y_min, y_max + delta, delta)
    X1, X2 = np.meshgrid(x, y)
    Z = np.array([f(z) for z in zip(X1.ravel(), X2.ravel())])
    Z = Z.reshape(len(x), len(y))
    plt.contourf(X1, X2, Z, 10, cmap=matplotlib.cm.bone, origin='lower', alpha=0.85)
    plt.colorbar()
    
def show_descent(f, x_hist,f_hist):
    plt.subplot(1,2,1)
    X1,X2 = zip(*[(x[0], x[1]) for x in x_hist])
    
    plot_surface(f, min(X1+X2)-1, max(X1+X2)+1, min(X1+X2)-1, max(X1+X2)+1)
    plt.plot(X1, X2, 'o-', markersize=10)
    
    plt.subplot(1,2,2)
    plt.plot(f_hist, 'rx-')
    plt.show()
In [3]: 
def stepsize(x,p,f,t,c, mode):
    '''chooses a stepsize.
        different methods:  const/sqrt(t)
                            const/t
                            linesearch
    '''                      
    if mode=="c/sqrt(t)":
        return c/np.sqrt(t+1)
    elif mode=="c":
        return c
    elif mode=="c/t":
        return c/(t+1)
    elif mode=="linesearch":
        alpha=1.0
        epsilon=10**(-6)
        gamma = 0.5
        fx = f(x)
        print "alpha", alpha
        while f(x+alpha*p) > f(x) - epsilon and alpha > epsilon:
            alpha *= gamma
            print "alpha", alpha
        
        return alpha
    else:
        assert 0, "not supported stepsize mode "+ mode
The following is the gradient descent code as discussed in tutorial.
In [4]: 
def gradientDescent(x_init, f, g, stepsize_rule='c/sqrt(t)', c=0.1 ,max_iters=1000, eps=10**(-6)):
    '''minimizes function f with gradient g'''
    x = x_init
    x_hist = [x]
    f_hist = [f(x)]
    convergence_hist = []
    print "iter",-1,'x=',x, 'f(x)=', f_hist[-1]
    for t in xrange(0,max_iters):

        
        p = -g(x) #choose a direction
        alpha = stepsize(x, p, f, t, c, stepsize_rule) #choose a stepsize
        
        if alpha == 0: #error handling
            break
        
        print "iter",t,'x=',x, 'f(x)=', f(x), 'p=', p, 'alpha=', alpha             
        x = x + alpha*p #go!
        
        x_hist.append(x) #logging x
        f_hist.append(f(x)) #logging function value
        
        if np.linalg.norm(x - x_hist[-2]) < eps: #check convergence
            break
            
    return x_hist, f_hist
Initialize a function and it's gradient:
In [5]: 
f = lambda x: x[0]**2 + x[1]**2 + x[0]*x[1] + 3*x[0] + x[1]
g = lambda x: np.array([2*x[0] + x[1] + 3, 2*x[1] + x[0] + 1])
x_init = np.array([3,5])
In [6]: 
#First look at a constant stepsize (change c to take larger or smaller steps)
x_hist, f_hist = gradientDescent(x_init, f, g, stepsize_rule="c",c=0.1)
show_descent(f,x_hist,f_hist)

iter -1 x= [3 5] f(x)= 63
iter 0 x= [3 5] f(x)= 63 p= [-14 -14] alpha= 0.1
iter 1 x= [ 1.6  3.6] f(x)= 29.68 p= [-9.8 -9.8] alpha= 0.1
iter 2 x= [ 0.62  2.62] f(x)= 13.3532 p= [-6.86 -6.86] alpha= 0.1
iter 3 x= [-0.066  1.934] f(x)= 5.353068 p= [-4.802 -4.802] alpha= 0.1
iter 4 x= [-0.5462  1.4538] f(x)= 1.43300332 p= [-3.3614 -3.3614] alpha= 0.1
iter 5 x= [-0.88234  1.11766] f(x)= -0.4878283732 p= [-2.35298 -2.35298] alpha= 0.1
iter 6 x= [-1.117638  0.882362] f(x)= -1.42903590287 p= [-1.647086 -1.647086] alpha= 0.1
iter 7 x= [-1.2823466  0.7176534] f(x)= -1.89022759241 p= [-1.1529602 -1.1529602] alpha= 0.1
iter 8 x= [-1.39764262  0.60235738] f(x)= -2.11621152028 p= [-0.80707214 -0.80707214] alpha= 0.1
iter 9 x= [-1.47834983  0.52165017] f(x)= -2.22694364494 p= [-0.5649505 -0.5649505] alpha= 0.1
iter 10 x= [-1.53484488  0.46515512] f(x)= -2.28120238602 p= [-0.39546535 -0.39546535] alpha= 0.1
iter 11 x= [-1.57439142  0.42560858] f(x)= -2.30778916915 p= [-0.27682574 -0.27682574] alpha= 0.1
iter 12 x= [-1.60207399  0.39792601] f(x)= -2.32081669288 p= [-0.19377802 -0.19377802] alpha= 0.1
iter 13 x= [-1.6214518  0.3785482] f(x)= -2.32720017951 p= [-0.13564461 -0.13564461] alpha= 0.1
iter 14 x= [-1.63501626  0.36498374] f(x)= -2.33032808796 p= [-0.09495123 -0.09495123] alpha= 0.1
iter 15 x= [-1.64451138  0.35548862] f(x)= -2.3318607631 p= [-0.06646586 -0.06646586] alpha= 0.1
iter 16 x= [-1.65115797  0.34884203] f(x)= -2.33261177392 p= [-0.0465261 -0.0465261] alpha= 0.1
iter 17 x= [-1.65581058  0.34418942] f(x)= -2.33297976922 p= [-0.03256827 -0.03256827] alpha= 0.1
iter 18 x= [-1.6590674  0.3409326] f(x)= -2.33316008692 p= [-0.02279779 -0.02279779] alpha= 0.1
iter 19 x= [-1.66134718  0.33865282] f(x)= -2.33324844259 p= [-0.01595845 -0.01595845] alpha= 0.1
iter 20 x= [-1.66294303  0.33705697] f(x)= -2.33329173687 p= [-0.01117092 -0.01117092] alpha= 0.1
iter 21 x= [-1.66406012  0.33593988] f(x)= -2.33331295107 p= [-0.00781964 -0.00781964] alpha= 0.1
iter 22 x= [-1.66484208  0.33515792] f(x)= -2.33332334602 p= [-0.00547375 -0.00547375] alpha= 0.1
iter 23 x= [-1.66538946  0.33461054] f(x)= -2.33332843955 p= [-0.00383162 -0.00383162] alpha= 0.1
iter 24 x= [-1.66577262  0.33422738] f(x)= -2.33333093538 p= [-0.00268214 -0.00268214] alpha= 0.1
iter 25 x= [-1.66604083  0.33395917] f(x)= -2.33333215834 p= [-0.0018775 -0.0018775] alpha= 0.1
iter 26 x= [-1.66622858  0.33377142] f(x)= -2.33333275758 p= [-0.00131425 -0.00131425] alpha= 0.1
iter 27 x= [-1.66636001  0.33363999] f(x)= -2.33333305122 p= [-0.00091997 -0.00091997] alpha= 0.1
iter 28 x= [-1.66645201  0.33354799] f(x)= -2.3333331951 p= [-0.00064398 -0.00064398] alpha= 0.1
iter 29 x= [-1.6665164  0.3334836] f(x)= -2.3333332656 p= [-0.00045079 -0.00045079] alpha= 0.1
iter 30 x= [-1.66656148  0.33343852] f(x)= -2.33333330014 p= [-0.00031555 -0.00031555] alpha= 0.1
iter 31 x= [-1.66659304  0.33340696] f(x)= -2.33333331707 p= [-0.00022089 -0.00022089] alpha= 0.1
iter 32 x= [-1.66661513  0.33338487] f(x)= -2.33333332536 p= [-0.00015462 -0.00015462] alpha= 0.1
iter 33 x= [-1.66663059  0.33336941] f(x)= -2.33333332943 p= [-0.00010823 -0.00010823] alpha= 0.1
iter 34 x= [-1.66664141  0.33335859] f(x)= -2.33333333142 p= [ -7.57637385e-05  -7.57637385e-05] alpha= 0.1
iter 35 x= [-1.66664899  0.33335101] f(x)= -2.3333333324 p= [ -5.30346169e-05  -5.30346169e-05] alpha= 0.1
iter 36 x= [-1.66665429  0.33334571] f(x)= -2.33333333287 p= [ -3.71242318e-05  -3.71242318e-05] alpha= 0.1
iter 37 x= [-1.666658  0.333342] f(x)= -2.33333333311 p= [ -2.59869623e-05  -2.59869623e-05] alpha= 0.1
iter 38 x= [-1.6666606  0.3333394] f(x)= -2.33333333322 p= [ -1.81908736e-05  -1.81908736e-05] alpha= 0.1
iter 39 x= [-1.66666242  0.33333758] f(x)= -2.33333333328 p= [ -1.27336115e-05  -1.27336115e-05] alpha= 0.1
iter 40 x= [-1.6666637  0.3333363] f(x)= -2.33333333331 p= [ -8.91352807e-06  -8.91352807e-06] alpha= 0.1
iter 41 x= [-1.66666459  0.33333541] f(x)= -2.33333333332 p= [ -6.23946965e-06  -6.23946965e-06] alpha= 0.1
In [7]: 
#Decaying stepsize 1/t
x_hist, f_hist = gradientDescent(x_init, f, g, stepsize_rule="c/t",c=1)
show_descent(f,x_hist,f_hist)

iter -1 x= [3 5] f(x)= 63
iter 0 x= [3 5] f(x)= 63 p= [-14 -14] alpha= 1.0
iter 1 x= [-11.  -9.] f(x)= 259.0 p= [ 28.  28.] alpha= 0.5
iter 2 x= [ 3.  5.] f(x)= 63.0 p= [-14. -14.] alpha= 0.333333333333
iter 3 x= [-1.66666667  0.33333333] f(x)= -2.33333333333 p= [ -1.77635684e-15  -1.77635684e-15] alpha= 0.25
In [8]: 
#Decaying stepsize of 1/sqrt(t)
x_hist, f_hist = gradientDescent(x_init, f, g, stepsize_rule="c/sqrt(t)",c=0.1)
show_descent(f,x_hist,f_hist)

iter -1 x= [3 5] f(x)= 63
iter 0 x= [3 5] f(x)= 63 p= [-14 -14] alpha= 0.1
iter 1 x= [ 1.6  3.6] f(x)= 29.68 p= [-9.8 -9.8] alpha= 0.0707106781187
iter 2 x= [ 0.90703535  2.90703535] f(x)= 17.538492947 p= [-7.72110606 -7.72110606] alpha= 0.057735026919
iter 3 x= [ 0.46125709  2.46125709] f(x)= 11.250845184 p= [-6.38377126 -6.38377126] alpha= 0.05
iter 4 x= [ 0.14206852  2.14206852] f(x)= 7.48123564547 p= [-5.42620557 -5.42620557] alpha= 0.04472135955
iter 5 x= [-0.10059877  1.89940123] f(x)= 5.02437267831 p= [-4.6982037 -4.6982037] alpha= 0.0408248290464
iter 6 x= [-0.29240213  1.70759787] f(x)= 3.33247572789 p= [-4.12279361 -4.12279361] alpha= 0.0377964473009
iter 7 x= [-0.44822908  1.55177092] f(x)= 2.1204371228 p= [-3.65531276 -3.65531276] alpha= 0.0353553390593
iter 8 x= [-0.5774639  1.4225361] f(x)= 1.22575465302 p= [-3.26760829 -3.26760829] alpha= 0.0333333333333
iter 9 x= [-0.68638418  1.31361582] f(x)= 0.549527935609 p= [-2.94084746 -2.94084746] alpha= 0.0316227766017
iter 10 x= [-0.77938194  1.22061806] f(x)= 0.0284892197411 p= [-2.66185418 -2.66185418] alpha= 0.0301511344578
iter 11 x= [-0.85963986  1.14036014] f(x)= -0.379456553731 p= [-2.42108041 -2.42108041] alpha= 0.0288675134595
iter 12 x= [-0.92953044  1.07046956] f(x)= -0.703223863284 p= [-2.21140869 -2.21140869] alpha= 0.0277350098113
iter 13 x= [-0.99086388  1.00913612] f(x)= -0.963205102901 p= [-2.02740837 -2.02740837] alpha= 0.0267261241912
iter 14 x= [-1.04504865  0.95495135] f(x)= -1.17410643917 p= [-1.86485406 -1.86485406] alpha= 0.0258198889747
iter 15 x= [-1.09319897  0.90680103] f(x)= -1.34673773603 p= [-1.72040309 -1.72040309] alpha= 0.025
iter 16 x= [-1.13620905  0.86379095] f(x)= -1.48917747539 p= [-1.59137286 -1.59137286] alpha= 0.0242535625036
iter 17 x= [-1.17480551  0.82519449] f(x)= -1.60755113615 p= [-1.47558347 -1.47558347] alpha= 0.0235702260396
iter 18 x= [-1.20958534  0.79041466] f(x)= -1.70656332782 p= [-1.37124397 -1.37124397] alpha= 0.0229415733871
iter 19 x= [-1.24104384  0.75895616] f(x)= -1.78986895774 p= [-1.27686848 -1.27686848] alpha= 0.022360679775
iter 20 x= [-1.26959549  0.73040451] f(x)= -1.86033676528 p= [-1.19121354 -1.19121354] alpha= 0.0218217890236
iter 21 x= [-1.2955899  0.7044101] f(x)= -1.92023942503 p= [-1.11323031 -1.11323031] alpha= 0.0213200716356
iter 22 x= [-1.31932405  0.68067595] f(x)= -1.97139264571 p= [-1.04202786 -1.04202786] alpha= 0.0208514414057
iter 23 x= [-1.34105183  0.65894817] f(x)= -2.0152582663 p= [-0.97684451 -0.97684451] alpha= 0.0204124145232
iter 24 x= [-1.36099158  0.63900842] f(x)= -2.05302156551 p= [-0.91702525 -0.91702525] alpha= 0.02
iter 25 x= [-1.37933209  0.62066791] f(x)= -2.08564985528 p= [-0.86200373 -0.86200373] alpha= 0.0196116135138
iter 26 x= [-1.39623737  0.60376263] f(x)= -2.11393732527 p= [-0.81128788 -0.81128788] alpha= 0.019245008973
iter 27 x= [-1.41185062  0.58814938] f(x)= -2.13853967411 p= [-0.76444815 -0.76444815] alpha= 0.0188982236505
iter 28 x= [-1.42629733  0.57370267] f(x)= -2.16000107646 p= [-0.72110802 -0.72110802] alpha= 0.0185695338177
iter 29 x= [-1.43968797  0.56031203] f(x)= -2.178775344 p= [-0.6809361 -0.6809361] alpha= 0.0182574185835
iter 30 x= [-1.4521201  0.5478799] f(x)= -2.19524264947 p= [-0.64363969 -0.64363969] alpha= 0.0179605302027
iter 31 x= [-1.46368021  0.53631979] f(x)= -2.20972283242 p= [-0.60895936 -0.60895936] alpha= 0.0176776695297
iter 32 x= [-1.4744452  0.5255548] f(x)= -2.2224860514 p= [-0.57666441 -0.57666441] alpha= 0.0174077655956
iter 33 x= [-1.48448363  0.51551637] f(x)= -2.23376136164 p= [-0.5465491 -0.5465491] alpha= 0.0171498585143
iter 34 x= [-1.49385687  0.50614313] f(x)= -2.24374366013 p= [-0.51842938 -0.51842938] alpha= 0.0169030850946
iter 35 x= [-1.50261993  0.49738007] f(x)= -2.2525993379 p= [-0.49214021 -0.49214021] alpha= 0.0166666666667
iter 36 x= [-1.51082227  0.48917773] f(x)= -2.26047090246 p= [-0.4675332 -0.4675332] alpha= 0.0164398987305
iter 37 x= [-1.51850847  0.48149153] f(x)= -2.26748077543 p= [-0.4444746 -0.4444746] alpha= 0.0162221421131
iter 38 x= [-1.5257188  0.4742812] f(x)= -2.27373442616 p= [-0.42284361 -0.42284361] alpha= 0.0160128153805
iter 39 x= [-1.53248971  0.46751029] f(x)= -2.279322968 p= [-0.40253086 -0.40253086] alpha= 0.0158113883008
iter 40 x= [-1.53885428  0.46114572] f(x)= -2.28432531783 p= [-0.38343715 -0.38343715] alpha= 0.0156173761889
iter 41 x= [-1.54484257  0.45515743] f(x)= -2.28880999895 p= [-0.3654723 -0.3654723] alpha= 0.0154303349962
iter 42 x= [-1.55048193  0.44951807] f(x)= -2.29283665159 p= [-0.34855422 -0.34855422] alpha= 0.0152498570333
iter 43 x= [-1.55579733  0.44420267] f(x)= -2.29645730274 p= [-0.33260802 -0.33260802] alpha= 0.0150755672289
iter 44 x= [-1.56081158  0.43918842] f(x)= -2.29971743696 p= [-0.31756525 -0.31756525] alpha= 0.01490711985
iter 45 x= [-1.56554557  0.43445443] f(x)= -2.30265690235 p= [-0.3033633 -0.3033633] alpha= 0.0147441956155
iter 46 x= [-1.57001841  0.42998159] f(x)= -2.30531067904 p= [-0.28994476 -0.28994476] alpha= 0.0145864991498
iter 47 x= [-1.57424769  0.42575231] f(x)= -2.30770953317 p= [-0.27725692 -0.27725692] alpha= 0.0144337567297
iter 48 x= [-1.57824955  0.42175045] f(x)= -2.30988057474 p= [-0.26525134 -0.26525134] alpha= 0.0142857142857
iter 49 x= [-1.58203886  0.41796114] f(x)= -2.31184773469 p= [-0.25388343 -0.25388343] alpha= 0.0141421356237
iter 50 x= [-1.58562931  0.41437069] f(x)= -2.31363217411 p= [-0.24311207 -0.24311207] alpha= 0.0140028008403
iter 51 x= [-1.58903356  0.41096644] f(x)= -2.31525263582 p= [-0.23289932 -0.23289932] alpha= 0.0138675049056
iter 52 x= [-1.59226329  0.40773671] f(x)= -2.31672574727 p= [-0.22321012 -0.22321012] alpha= 0.0137360563949
iter 53 x= [-1.59532932  0.40467068] f(x)= -2.31806628214 p= [-0.21401204 -0.21401204] alpha= 0.0136082763488
iter 54 x= [-1.59824165  0.40175835] f(x)= -2.31928738657 p= [-0.20527504 -0.20527504] alpha= 0.0134839972493
iter 55 x= [-1.60100958  0.39899042] f(x)= -2.32040077534 p= [-0.19697125 -0.19697125] alpha= 0.0133630620956
iter 56 x= [-1.60364172  0.39635828] f(x)= -2.32141690232 p= [-0.18907483 -0.18907483] alpha= 0.0132453235707
iter 57 x= [-1.60614608  0.39385392] f(x)= -2.32234510881 p= [-0.18156176 -0.18156176] alpha= 0.013130643286
iter 58 x= [-1.6085301  0.3914699] f(x)= -2.32319375286 p= [-0.17440969 -0.17440969] alpha= 0.0130188910981
iter 59 x= [-1.61080072  0.38919928] f(x)= -2.32397032226 p= [-0.16759783 -0.16759783] alpha= 0.0129099444874
iter 60 x= [-1.6129644  0.3870356] f(x)= -2.32468153347 p= [-0.1611068 -0.1611068] alpha= 0.0128036879933
iter 61 x= [-1.61502716  0.38497284] f(x)= -2.32533341819 p= [-0.15491851 -0.15491851] alpha= 0.0127000127
iter 62 x= [-1.61699463  0.38300537] f(x)= -2.32593139956 p= [-0.14901611 -0.14901611] alpha= 0.012598815767
iter 63 x= [-1.61887206  0.38112794] f(x)= -2.32648035896 p= [-0.14338383 -0.14338383] alpha= 0.0125
iter 64 x= [-1.62066435  0.37933565] f(x)= -2.32698469505 p= [-0.13800694 -0.13800694] alpha= 0.0124034734589
iter 65 x= [-1.62237612  0.37762388] f(x)= -2.32744837562 p= [-0.13287164 -0.13287164] alpha= 0.0123091490979
iter 66 x= [-1.62401166  0.37598834] f(x)= -2.32787498361 p= [-0.12796503 -0.12796503] alpha= 0.0122169444356
iter 67 x= [-1.625575  0.374425] f(x)= -2.32826775763 p= [-0.12327501 -0.12327501] alpha= 0.0121267812518
iter 68 x= [-1.62706993  0.37293007] f(x)= -2.32862962796 p= [-0.11879022 -0.11879022] alpha= 0.0120385853086
iter 69 x= [-1.62849999  0.37150001] f(x)= -2.32896324845 p= [-0.11450002 -0.11450002] alpha= 0.0119522860933
iter 70 x= [-1.62986853  0.37013147] f(x)= -2.3292710248 p= [-0.11039441 -0.11039441] alpha= 0.0118678165819
iter 71 x= [-1.63117867  0.36882133] f(x)= -2.32955513979 p= [-0.10646399 -0.10646399] alpha= 0.0117851130198
iter 72 x= [-1.63243336  0.36756664] f(x)= -2.32981757567 p= [-0.10269992 -0.10269992] alpha= 0.0117041147196
iter 73 x= [-1.63363537  0.36636463] f(x)= -2.33006013416 p= [-0.09909388 -0.09909388] alpha= 0.0116247638744
iter 74 x= [-1.63478732  0.36521268] f(x)= -2.33028445425 p= [-0.09563805 -0.09563805] alpha= 0.0115470053838
iter 75 x= [-1.63589165  0.36410835] f(x)= -2.33049202813 p= [-0.09232505 -0.09232505] alpha= 0.0114707866935
iter 76 x= [-1.63695069  0.36304931] f(x)= -2.33068421546 p= [-0.08914793 -0.08914793] alpha= 0.011396057646
iter 77 x= [-1.63796662  0.36203338] f(x)= -2.33086225609 p= [-0.08610013 -0.08610013] alpha= 0.0113227703414
iter 78 x= [-1.63894152  0.36105848] f(x)= -2.33102728149 p= [-0.08317545 -0.08317545] alpha= 0.0112508790093
iter 79 x= [-1.63987731  0.36012269] f(x)= -2.331180325 p= [-0.08036806 -0.08036806] alpha= 0.0111803398875
iter 80 x= [-1.64077586  0.35922414] f(x)= -2.33132233106 p= [-0.07767243 -0.07767243] alpha= 0.0111111111111
iter 81 x= [-1.64163888  0.35836112] f(x)= -2.33145416343 p= [-0.07508335 -0.07508335] alpha= 0.0110431526075
iter 82 x= [-1.64246804  0.35753196] f(x)= -2.33157661269 p= [-0.07259588 -0.07259588] alpha= 0.010976425999
iter 83 x= [-1.64326488  0.35673512] f(x)= -2.33169040289 p= [-0.07020535 -0.07020535] alpha= 0.0109108945118
iter 84 x= [-1.64403089  0.35596911] f(x)= -2.33179619766 p= [-0.06790734 -0.06790734] alpha= 0.0108465228909
iter 85 x= [-1.64476744  0.35523256] f(x)= -2.33189460556 p= [-0.06569767 -0.06569767] alpha= 0.0107832773203
iter 86 x= [-1.64547588  0.35452412] f(x)= -2.33198618512 p= [-0.06357236 -0.06357236] alpha= 0.0107211253484
iter 87 x= [-1.64615745  0.35384255] f(x)= -2.33207144919 p= [-0.06152766 -0.06152766] alpha= 0.0106600358178
iter 88 x= [-1.64681334  0.35318666] f(x)= -2.33215086901 p= [-0.05955999 -0.05955999] alpha= 0.0105999788001
iter 89 x= [-1.64744467  0.35255533] f(x)= -2.33222487784 p= [-0.05766599 -0.05766599] alpha= 0.0105409255339
iter 90 x= [-1.64805252  0.35194748] f(x)= -2.33229387426 p= [-0.05584243 -0.05584243] alpha= 0.0104828483672
iter 91 x= [-1.64863791  0.35136209] f(x)= -2.33235822518 p= [-0.05408627 -0.05408627] alpha= 0.0104257207029
iter 92 x= [-1.6492018  0.3507982] f(x)= -2.3324182685 p= [-0.0523946 -0.0523946] alpha= 0.0103695169473
iter 93 x= [-1.64974511  0.35025489] f(x)= -2.33247431563 p= [-0.05076468 -0.05076468] alpha= 0.0103142124626
iter 94 x= [-1.6502687  0.3497313] f(x)= -2.33252665372 p= [-0.04919389 -0.04919389] alpha= 0.0102597835209
iter 95 x= [-1.65077342  0.34922658] f(x)= -2.33257554764 p= [-0.04767973 -0.04767973] alpha= 0.0102062072616
iter 96 x= [-1.65126005  0.34873995] f(x)= -2.33262124193 p= [-0.04621985 -0.04621985] alpha= 0.0101534616513
iter 97 x= [-1.65172934  0.34827066] f(x)= -2.33266396238 p= [-0.04481197 -0.04481197] alpha= 0.0101015254455
iter 98 x= [-1.65218201  0.34781799] f(x)= -2.33270391766 p= [-0.04345396 -0.04345396] alpha= 0.0100503781526
iter 99 x= [-1.65261874  0.34738126] f(x)= -2.33274130065 p= [-0.04214378 -0.04214378] alpha= 0.01
iter 100 x= [-1.65304018  0.34695982] f(x)= -2.33277628979 p= [-0.04087946 -0.04087946] alpha= 0.0099503719021
iter 101 x= [-1.65344694  0.34655306] f(x)= -2.33280905015 p= [-0.03965917 -0.03965917] alpha= 0.00990147542977
iter 102 x= [-1.65383963  0.34616037]
In [9]: 
#backtracking linesearch
x_hist, f_hist = gradientDescent(x_init, f, g, stepsize_rule="linesearch",c=0.1)
show_descent(f,x_hist,f_hist)

iter -1 x= [3 5] f(x)= 63
alpha 1.0
alpha 0.5
iter 0 x= [3 5] f(x)= 63 p= [-14 -14] alpha= 0.5
alpha 1.0
alpha 0.5
iter 1 x= [-4. -2.] f(x)= 14.0 p= [ 7.  7.] alpha= 0.5
alpha 1.0
alpha 0.5
iter 2 x= [-0.5  1.5] f(x)= 1.75 p= [-3.5 -3.5] alpha= 0.5
alpha 1.0
alpha 0.5
iter 3 x= [-2.25 -0.25] f(x)= -1.3125 p= [ 1.75  1.75] alpha= 0.5
alpha 1.0
alpha 0.5
iter 4 x= [-1.375  0.625] f(x)= -2.078125 p= [-0.875 -0.875] alpha= 0.5
alpha 1.0
alpha 0.5
iter 5 x= [-1.8125  0.1875] f(x)= -2.26953125 p= [ 0.4375  0.4375] alpha= 0.5
alpha 1.0
alpha 0.5
iter 6 x= [-1.59375  0.40625] f(x)= -2.3173828125 p= [-0.21875 -0.21875] alpha= 0.5
alpha 1.0
alpha 0.5
iter 7 x= [-1.703125  0.296875] f(x)= -2.32934570312 p= [ 0.109375  0.109375] alpha= 0.5
alpha 1.0
alpha 0.5
iter 8 x= [-1.6484375  0.3515625] f(x)= -2.33233642578 p= [-0.0546875 -0.0546875] alpha= 0.5
alpha 1.0
alpha 0.5
iter 9 x= [-1.67578125  0.32421875] f(x)= -2.33308410645 p= [ 0.02734375  0.02734375] alpha= 0.5
alpha 1.0
alpha 0.5
iter 10 x= [-1.66210938  0.33789062] f(x)= -2.33327102661 p= [-0.01367188 -0.01367188] alpha= 0.5
alpha 1.0
alpha 0.5
iter 11 x= [-1.66894531  0.33105469] f(x)= -2.33331775665 p= [ 0.00683594  0.00683594] alpha= 0.5
alpha 1.0
alpha 0.5
iter 12 x= [-1.66552734  0.33447266] f(x)= -2.33332943916 p= [-0.00341797 -0.00341797] alpha= 0.5
alpha 1.0
alpha 0.5
alpha 0.25
alpha 0.125
alpha 0.0625
alpha 0.03125
alpha 0.015625
alpha 0.0078125
alpha 0.00390625
alpha 0.001953125
alpha 0.0009765625
alpha 0.00048828125
alpha 0.000244140625
alpha 0.0001220703125
alpha 6.103515625e-05
alpha 3.0517578125e-05
alpha 1.52587890625e-05
alpha 7.62939453125e-06
alpha 3.81469726562e-06
alpha 1.90734863281e-06
alpha 9.53674316406e-07
iter 13 x= [-1.66723633  0.33276367] f(x)= -2.33333235979 p= [ 0.00170898  0.00170898] alpha= 9.53674316406e-07
In [9]: 















In []: