I am trying to create the simplest perception there is with just one neuron (neuron takes 2 input values and * them by weights then + bias and activate the SUM of them with (1 / (1 + Math.exp(-x))) sigmoid function ) and train it via backpropagation (get an error by subtracting expected value from the output I gain, find gradient and weight delta by which we multiply the difference between wight and input value), but after the first iteration, my weights get really close to 0 and start producing a sigmoid(0) which is 0.5 (it always produces values from 0.48 to 0.52 or close.
class Neuron {constructor(){
this.inputs = [1,1];
this.inputWeights = [(Math.random()*2)-1,(Math.random()*2)-1];
this.bias = 0.1;
this.activate = () => {
if(this.inputs.length !== this.inputWeights.length)return "Wrong input length";
let sum = 0;
for(var n = 0; n < this.inputs.length;n++){
sum = sum + (this.inputs[n]*this.inputWeights[n]);
}
sum = sum + this.bias;
//return sigmoid activated value
let activated_output = (1 / (1 + Math.exp(-sum)));
return activated_output;
};
this.error = (predicted,desired) => {
let error = predicted - desired;
let gradient = predicted * (1-predicted);
let weights_delta = error * gradient;
return weights_delta;
};
this.changeWeights = (weights_delta) => {
let info = this.inputWeights[0];
for(var n = 0; n < this.inputWeights.length; n++){
this.inputWeights[n] = (this.inputWeights[n] - this.inputs[n]) * weights_delta * learning_rate;
}
return "first weight changed from " + info + " to " + this.inputWeights[0];
}
}}
var testNeuron = new Neuron();
var learning_rate = 0.05;
var dataset = [
{ inputs: [1,0], outputs: [1] },
{ inputs: [0,1], outputs: [0] },
{ inputs: [0.5,0.1], outputs: [1] },
{ inputs: [0.1,0.9], outputs: [0] }];
//train
var train = (iterations, data) => {
for(var i = 0; i < iterations; i++){
for(var n = 0; n < data.length; n++){
testNeuron.inputs = data[n].inputs;
console.log(testNeuron.changeWeights(testNeuron.error(testNeuron.activate() ,
data[n].outputs[0])));
}
}
}
train(10,dataset);
Here is all the code, I tryed it with and without biases but i feel my math is definitely wrong but I could not figure out where because I'm a noob..halp sirs
The biggest error was that I didn't use any bias input, and did not adjust weights for it. If we don't use bias then a simple input like 0,0 will ALWAYS return 0 and there is no way to adjust weights for output to change.
Second, if we looking at simple Perceptron we should be using a threshold function and not a sigmoid.(although sigmoid is possible but imo slower in this example) Thresh hold function is a simple function that returns 0 if output is negative and 1 if it's positive. My redone and working code looks like this, increasing training iteration results in a decrease of error just like it should, Thank you
class Perceptron{constructor(){
//bias , input1, input2
this.inputs = [1,0,0];
this.inputWeights = [(Math.random()*2)-1,(Math.random()*2)-1,(Math.random()*2)-1];
this.output = 0;
this.desiredOutput = 0;
}//perceptron methods
activate = () => {
let sum = 0;
for(var n = 0; n < this.inputs.length; n++){
sum += this.inputs[n] * this.inputWeights[n];
};
this.output = sum < 0 ? 0 : 1;
this.desiredOutput == this.output ? console.log("Correct answer") : console.log("Incorrect answer");
};
propagate = () => {
let error = this.desiredOutput - this.output;
for(var m = 0; m < this.inputs.length; m++){
let delta = error * this.inputs[m];
this.inputWeights[m] = this.inputWeights[m] + (delta * learningRate);
}
};
}
let learningRate = 0.1;
var train = (iterations) => {
for(var x = 0; x < iterations; x++){
for(var y = 0; y < dataset.length; y++){
perception.inputs = [1,dataset[y][0],dataset[y][1]];
perception.desiredOutput = dataset[y][2];
perception.activate();
perception.propagate();
}
}
}
var perception = new Perceptron();
//[input1 , input2 , desiredOutput]
var dataset = [
[0,0,1],
[1,1,0],
[0.1,0.3,1],
[1.5,1.8,0]
];
train(100);