Function approximation with Neural Network - Loss 0

Question

I'm trying to create a NN to approximate functions (sine, cos, custom...) but I'm struggling with the format, I don't want to use input-label, but rather, input-output. How do I change it?

I'm following this tutorial

import tensorflow as tf
import random
from math import sin
import numpy as np


n_nodes_hl1 = 500
n_nodes_hl2 = 500
n_nodes_hl3 = 500

n_inputs = 1 # CHANGES HERE
n_outputs = 1 #CHANGES HERE
batch_size = 100

x = tf.placeholder('float', [None, n_inputs]) #CHANGES HERE
y = tf.placeholder('float', [None, n_outputs]) #CHANGES HERE

def neural_network_model(data):
    hidden_layer_1 = {'weights':tf.Variable(tf.random_normal([n_inputs, n_nodes_hl1])),
                      'biases': tf.Variable(tf.random_normal([n_nodes_hl1]))} #CHANGES HERE

    hidden_layer_2 = {'weights':tf.Variable(tf.random_normal([n_nodes_hl1, n_nodes_hl2])),
                      'biases': tf.Variable(tf.random_normal([n_nodes_hl2]))}

    hidden_layer_3 = {'weights':tf.Variable(tf.random_normal([n_nodes_hl2, n_nodes_hl3])),
                      'biases': tf.Variable(tf.random_normal([n_nodes_hl2]))}

    output_layer = {'weights':tf.Variable(tf.random_normal([n_nodes_hl3, n_outputs])),
                    'biases': tf.Variable(tf.random_normal([n_outputs]))} #CHANGES HERE


    l1 = tf.add(tf.matmul(data, hidden_layer_1['weights']), hidden_layer_1['biases'])
    l1 = tf.nn.relu(l1)


    l2 = tf.add(tf.matmul(l1, hidden_layer_2['weights']), hidden_layer_2['biases'])
    l2 = tf.nn.relu(l2)

    l3 = tf.add(tf.matmul(l2, hidden_layer_3['weights']), hidden_layer_3['biases'])
    l3 = tf.nn.relu(l3)

    output = tf.add(tf.matmul(l3, output_layer['weights']), output_layer['biases'])

    return output

def train_neural_network(x):
    prediction = neural_network_model(x)
    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=y))

    optmizer = tf.train.AdamOptimizer().minimize(cost)

    epochs = 10

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())

        for epoch in range(epochs): 
            loss = 0
            for _ in range(batch_size^2): #CHANGES HERE
                batch_x, batch_y = generate_input_output(batch_size) #CHANGES HERE
                a, c = sess.run([optmizer, cost], feed_dict={x: batch_x, y:batch_y})
                loss += c
            print("Epoch:", epoch+1, "out of", epochs, "- Loss:", loss)

        correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y,1))
        accuracy = tf.reduce_mean(tf.cast(correct, 'float'))

        test_x, test_y = generate_input_output(batch_size) #CHANGES HERE

        print('Accuracy', accuracy.eval({x:test_x, y:test_y}))

def desired_function(x): #CHANGES HERE
    return sin(x)

def generate_input_output(batch_size): #CHANGES HERE
    batch_x = [random.uniform(-10, 10) for _ in range(batch_size)]
    batch_y = [desired_function(x) for x in batch_x]
    batch_x = np.reshape(batch_x, (-1, 1))
    batch_y = np.reshape(batch_y, (-1, 1))
    return batch_x, batch_y

train_neural_network(x)

Answer 1

Have not tried it myself, so there might be more things you need to change to get the model to run, but you will definitely want to change this line:

cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=y))

to something more like:

cost = tf.reduce_sum(tf.square(prediction - y))

Basically, your cost function is much simpler in this case...you just want to reduce the sum of the squared difference between the network's output and the expected y value.