import numpy as np
a = np.ones(10)
b = np.ones(10) * 2
c = b * a # when the programs execute to c = b * a, 
        #it runs the actual computation.
d = c + 1  

from IPython.display import Image
Image(filename='./pics/comp_graph.png', height=400)
# image from the mxnet documentation 

import theano.tensor as T
from theano import function

# in theano each symbolic variable must be assigned a tensor (shape) type and data type
A = T.dscalar('A') # default of dtype is floatX
print ("A.dtype:", A.dtype)
B = T.dscalar('B')
C = T.mul(B, A)
D = C + 1
f = function(inputs=[A, B], outputs=D) # compile a function

f(1,2)

# implicit compilation with theano
Z = T.add(A+B) * 3.
Z.eval({A:2., B:5.})

import tensorflow as tf

# if no graph is specified the default graph is used
# tf.placeholder for defining symbolic variables 
A = tf.placeholder(name="A", dtype=tf.float32)
B = tf.placeholder(name="B", dtype=tf.float32)
C = tf.multiply(B, A, name="multiplication")
D = C + 1

with tf.Session() as sess:
    result = sess.run([D], feed_dict={A:1., B:[2.]})
    print(result)

tf.get_variable?

g = tf.get_default_graph()
[op.name for op in g.get_operations()]

# write the definition of the graph to the log file
writer = tf.summary.FileWriter("./tmp/graph_logs", 
                                tf.get_default_graph())
writer.flush()

with tf.variable_scope("foo"):
    with tf.variable_scope("bar", reuse=None):
        q = tf.get_variable(name="q", shape=[1])
assert q.name == "foo/bar/q:0"

with tf.name_scope("foo"):
    with tf.variable_scope("bar", reuse=None):
        p = tf.Variable(name='p', initial_value=[1])
        u = tf.get_variable(name='u', shape=[1]) # the foo scope is ignored
    
print (p.name)
print (u.name)

x = 2
k = 10
result = 1
# compute x**k by a python loop
for i in range(k):
    result = result * x
print (result)

import theano
import theano.tensor as T

k = T.iscalar("k")
A = T.iscalar("A")

# Symbolic description of the result
result, updates = theano.scan(fn=lambda prior_result, A: prior_result 
                                                          * A,
                              outputs_info=T.ones_like(A),
                              non_sequences=A,
                              n_steps=k)

# Note that the result is not a matrix, but a 3D tensor containing the value of A**k for each step. 
# We want the last value (after k steps) so we compile a function to return just that.

# We only care about A**k, but scan has provided us with A**1 through A**k.
# Discard the values that we don't care about. Scan is smart enough to
# notice this and not waste memory saving them.
final_result = result[-1]

# compiled function that returns A**k
power = theano.function(inputs=[A, k], 
                        outputs=final_result, updates=updates)


# note A is a T.vector, as input a np-vector or a python list is OK:
print (power(2, 10))

# Example compute x**k by a python loop
k = 10
x = 2
out = tf.scan(lambda previous_output, current_input: 
              previous_output * current_input, 
              tf.fill([k], x), initializer=tf.constant(1)) 

with tf.Session() as sess:
    sess.run(tf.initialize_all_variables())
    print(sess.run(out[-1]))  

# example cummulative sum
elems = tf.Variable([1.0, 2.0, 2.0, 2.0])
elems = tf.identity(elems)
initializer = tf.constant(0.0)
out = tf.scan(lambda previous_output, current_input: previous_output + current_input, 
              elems, initializer=initializer)

with tf.Session() as sess:
    sess.run(tf.initialize_all_variables())
    print(sess.run(out))

#tf.scan?

from IPython.display import Image
Image(filename='./pics/comp_graph_fold.png', height=400)
# from the mx net documentation

import operator
def combine_dicts(a, b, op=operator.add):
    x = (list(a.items()) + list(b.items()) +
        [(k, op(a[k], b[k])) for k in set(b) & set(a)])
    return {x[i][0]: x[i][1] for i in range(0, len(x))}

a = {'a': 2, 'b':3, 'c':4}
b = {'a': 5, 'c':6, 'x':7}
combine_dicts(a, b)

A = {'a':-1.3, 'b':-4, 'c':3}
B = {'b':3, 'c':4, 'd':5}
combine_dicts(A,B)

class Scalar(object) :
    """Simple Scalar object that support autodiff."""
    
    def __init__(self, value, name=None):
        self.value = value
        if name:
            self.grad = lambda g : {name : g}
        else:
            self.grad = lambda g : {}
            
    def __add__(self, other):
        assert isinstance(other, Scalar)
        # forward pass: addition is simple:
        ret = Scalar(self.value + other.value)
        
        #  
        def grad(g):
            x = self.grad(g)
            x = combine_dicts(x, other.grad(g))
            return x
        ret.grad = grad
        
        return ret

    def __mul__(self, other):
        assert isinstance(other, Scalar)
        ret = Scalar(self.value * other.value)
        
        #
        def grad(g):
            x = self.grad(g * other.value)
            x = combine_dicts(x, other.grad(g * self.value))
            return x
        ret.grad = grad
        return ret
    
    def square(self):
        ret = Scalar(self.value**2)
        
        #
        def grad(g):
            x = self.grad(2.0 * self.value * g) 
            return x
        ret.grad = grad
        return ret      

a = Scalar(1., 'a')
b = Scalar(2., 'b')
c = b * a 
d = c + Scalar(1.)
print (d.value)

# backpropagtion of the value 1 through the graph:
print ("derivatives of d:", d.grad(1))

# generate some train data
x_min = -10.
x_max = 10.
m = 10

x = np.random.uniform(x_min, x_max, m)
a = 10.
c = 5.
y_noise_sigma = 3.
y = a + c * x + np.random.randn(len(x)) * y_noise_sigma

%matplotlib inline
import matplotlib.pyplot as plt
plt.plot(x, y, "bo")
plt.xlabel("x")
plt.ylabel("y")

# Univariate Linear Regression

# train data
x_values = x
y_values = y

# start values
b_value=15.; w_value=2.

assert x_values.shape[0] == y_values.shape[0]
nb_examples = x_values.shape[0]

def linear_model_with_data(b_value, w_value, x_values, y_values):
    b = Scalar(b_value, name='b')
    w = Scalar(w_value, name='w')
    cost = Scalar(0.)
    h = []
    
    for x, y in zip(x_values, y_values):
        h_ = b + w * Scalar(x)
        diff = h_ + Scalar(-1.) * Scalar(y) 
        cost = cost + diff.square()
        h.append(h_)
    cost = cost * Scalar(1./nb_examples)
    return h, cost

def get_linear_model(x_values, y_values):
    return lambda b_value, w_value: linear_model_with_data(
        b_value, w_value, x_values, y_values)
        

linear_model = get_linear_model(x_values, y_values)

h, cost = linear_model(b_value, w_value)
#print ([i.value for i in h] )
print (cost.value)

# Gradient Descent 

learning_rate = 0.01
n_iterations = 200

costs = np.ndarray([n_iterations])
for i in range(n_iterations):
    deltas = cost.grad(1.)
    #print ("derivatives of cost:", cost.grad(1.))
    b_value = b_value - learning_rate * deltas['b']
    w_value = w_value - learning_rate * deltas['w']
    h, cost = linear_model(b_value, w_value)
    costs[i] = cost.value
    
#print ([i.value for i in h] ) 
print (b_value, w_value)
print (cost.value)

plt.plot(range(n_iterations), costs)
plt.xlabel("Iterations")
plt.ylabel("Cost")

plt.plot(x,y, "bo")

xx = np.array((-10.,10))
yy = b_value + w_value * xx
plt.plot(xx, yy, 'r-')
plt.xlabel("x")
plt.ylabel("y")

# forward pass
theta = np.random.randn(6, 9)
x = np.random.randn(9, 4)
mprod = np.dot(theta, x) # shape [6, 4]

# gradient backproped down to mprop  
grad_mprod = np.random.randn(*mprod.shape) # same shape as mprop

# Gradient calculation:
# shape of grad_theta has to be [6, 4]
grad_theta = np.dot(grad_mprod, x.T) #.T gives the transpose of the matrix
# shape of grad_x has to be [9,4]
grad_x = np.dot(theta.T, grad_mprod)

from IPython.display import Image
Image(filename='pics/comp_graph_backward.png', height=400)

# in theano each symbolic variable must be assigned a tensor (shape) type and data type
A = T.dscalar('A') # default of dtype is floatX
print ("A.dtype:", A.dtype)
B = T.dscalar('B')
C = T.mul(B, A)
D = C + 1
f = function(inputs=[A, B], outputs=D) # compile a function

dD_dA, dD_dB = T.grad(D, wrt=[A, B])
from theano import pp
print (pp(dD_dA)) # print out the gradient prior to optimization
print (pp(dD_dB))

grad_a = function(inputs=[A,B], outputs=dD_dA)
grad_b = function(inputs=[A,B], outputs=dD_dB)

grad_a(1,2)

grad_b(1,2)

# if no graph is specified the default graph is used

A = tf.placeholder(name="A", dtype=tf.float32)
B = tf.placeholder(name="B", dtype=tf.float32)
C = tf.multiply(B, A, name="multiplication")
D = C + 1

help(tf.gradients)

grad = tf.gradients(D, [A, B])
#print (grad)
with tf.Session() as sess:
    #sess.run(tf.initialize_all_variables())
    print(sess.run(grad, feed_dict={A:5., B:3.}))

v = tf.Variable(0., name="v")
c = tf.constant(0.1)
new_value = tf.add(v, c)
update = tf.assign(v, new_value)
init_op = tf.initialize_all_variables()
with tf.Session() as sess:
    sess.run(init_op)
    print(sess.run(v))
    for _ in range(3):
        sess.run(update)
        print(sess.run(v))