Simple/Single layer perceptron algorithm doesn't work

Question

I am trying to grasp the ideas behind neural networks (fully) so I'm starting out by creating my own simple perceptron algorithm.

Here is my code (in JavaScript):

var lr = 0.1;//learning rate

//Initiate the weights randomly
function initWeights(weights, trainingSets){
    for(var i=0; i<trainingSets[0].in.length; i++){
        weights[i] = Math.random()*2 - 1;
    }
    weights.push(Math.random()*2 - 1); //b
}


//Return the raw activation value for a giving trainingSet
function getSum(weights, trainingSet){
    var sum = 0;
    for(var i=0; i < trainingSet.in.length; i++){
        sum += weights[i]*trainingSet.in[i];
    }
    sum += 1 * weights[weights.length-1];
    return sum;
}

//Activation function
function activate(value){
    return (value >= 0)? 1 : 0;
}

function train(weights, trainingSets){
    var error = 0;
    for(var i=0; i<trainingSets.length; i++){
        var currentSet = trainingSets[i];
        var activationValue = getSum(weights, currentSet);
        var error = currentSet.out - activate(activationValue);
        error += error;
        for(var j=0; j<weights.length-1; j++){
            var deltaW = error * lr * currentSet.in[j];
            weights[j] += deltaW;
        }
        weights[weights.length-1] += error * lr * 1;
    }
    return error/(weights.length);
}

var inp = [
    {in:[1,1], out:1},
    {in:[0,0], out:0},
    {in:[0,1], out:0},
];
var w = [];
initWeights(w, inp);
//for(var j = 0; j < inp.length; j++){
    var error = 1;
    while(error >= 0.01){
        error = train(w, inp);
    }
//}
console.log("===")
var tester = {in:[1,0], out: NaN};
console.log(getSum(w, tester)) //should be negative
console.log("y=("+w[1]+"*x+"+w[2]+")/"+w[1])

The results aren't consistent, (I'm using an AND algorithm to learn by).
The plot should look like:

But usually looks like this:

I'm sure that I'm missing something small here,
Thanks in advance.

Answer 1

There are at least three problems with your code:

• You are redeclaring the error variable, it was first meant to be a summarized error, then you declare it again as a per-output-neuron error, which leads to the lose of information reagarding the whole process
• Your stopping criterion is bad - it should be a mean absolute value of errors, not just sum of errors - consider simple network, which classyfies one training example of label 0 as 1, it will result in negative error in your code, so training stops, even though it is far from being over

• It is not true, that after training with

  var inp = [
    {in:[1,1], out:1},
    {in:[0,0], out:0},
    {in:[0,1], out:0},
  ];
  

  you will get f( [1,0] ) == 0, this is not how perceptron works. It will simply find such a line in the 2 dimensioal plane, that [1,1] is on its one side, and [0,0] and [0,1] on the other. There is no guarantee, that [1,0] lies on the same side as [0,0] and [0,1], and this is expected behaviour. With provided data, there is no reason for perceptron to not use the vertical line with x=0.5, which perfectly separates your data, but f( [1,0] ) == 1. Your training data does not "define" and operation, just a simple set of rules, which are obeyed by infinite number of classifiers.

  function train(weights, trainingSets){
  var error = 0;
  for(var i=0; i<trainingSets.length; i++){
      var currentSet = trainingSets[i];
      var activationValue = getSum(weights, currentSet);
      var error_current = currentSet.out - activate(activationValue);
      error += Math.abs( error_current );
      for(var j=0; j<weights.length-1; j++){
          var deltaW = error_current * lr * currentSet.in[j];
          weights[j] += deltaW;
      }
      weights[weights.length-1] += error_current * lr * 1;
  }
  return error/(weights.length);
  }
  

as stated in the comment, if you train your network with values for points (1,0), (0,1) and (1,1) it will infer value for (0,0) by itself

var inp = [
    {in:[1,1], out:1},
    {in:[0,1], out:0},
    {in:[1,0], out:0},
];

var w = [];
initWeights(w, inp);
//for(var j = 0; j < inp.length; j++){
    var error = 1;
    while(error >= 0.01){
        error = train(w, inp);
    }
//}
console.log("===")

var test = [
    {in:[1,1], out:1},
    {in:[0,0], out:0},
    {in:[0,1], out:0},
    {in:[1,0], out:0},
];

for(var i=0; i<test.length; ++i){
 console.log(test[i].in + " out: " +test[i].out + " nn: " + activate(getSum(w, test[i]) ) );
}

produces

1,1 out: 1 nn: 1 
0,0 out: 0 nn: 0 
0,1 out: 0 nn: 0 
1,0 out: 0 nn: 0