%% ANNs

%% Create Data
numEx = 200;
meanP = 6;
meanN = 2;
x1P = (randn(1,numEx) + meanP)';
x2P = (randn(1,numEx) + meanP)';
x1N = (randn(1,numEx) + meanN)';
x2N = (randn(1,numEx) + meanN)';

figure;
plot(x1P, x2P, 'bo', 'linewidth', 3, 'markersize', 10);
hold on;
plot(x1N, x2N, 'ro', 'linewidth', 3, 'markersize', 10);

% for the discr func
xmin = min(min(x1P), min(x1N)) - 1;
xmax = max(max(x1P), max(x1N)) + 1;
theXs = xmin:0.1:xmax;


% Positive is 1, negative is 0
w_ini=randn(3,1);

% Binary activ func + squared error cost (aka perceptron rule)
% -------------------------------------------------------------------
w=w_ini;
countNeg = 1;
countPos = 1;
for i=1:5*numEx
    % Positive if even, negative if odd
    if rem(i, 2)
        % Negative
        trial = [1; x1N(countNeg); x2N(countNeg)];
        countNeg = rem(countNeg, numEx) + 1;
        target = 0;
    else
        % Positive
        trial = [1; x1P(countPos); x2P(countPos)];
        countPos = rem(countPos, numEx) + 1;
        target = 1;
    end
    z = w'*trial;
    y = z > 0;
    w = w + (target - y)*trial;
end

% Plot discriminating function
theYs = -w(2)/w(3)*theXs - w(1)/w(3);
plot(theXs, theYs, 'g', 'linewidth', 3);


% Linear activ func + squared error cost (aka perceptron rule)
% -------------------------------------------------------------------
w=w_ini;
countNeg = 1;
countPos = 1;
learnRate = 0.01;
for i=1:5*numEx
    % Positive if even, negative if odd
    if rem(i, 2)
        % Negative
        trial = [1; x1N(countNeg); x2N(countNeg)];
        countNeg = rem(countNeg, numEx) + 1;
        target = -1;
    else
        % Positive
        trial = [1; x1P(countPos); x2P(countPos)];
        countPos = rem(countPos, numEx) + 1;
        target = 1;
    end
    z = w'*trial;
    y = z;
    w = w + learnRate*(target - y)*trial;
end

% Plot discriminating function
theYs = -w(2)/w(3)*theXs - w(1)/w(3);
plot(theXs, theYs, 'm', 'linewidth', 3);


% Logit activ func + squared error cost 
% -------------------------------------
w=w_ini;
countNeg = 1;
countPos = 1;
learnRate = 0.05;
for i=1:5*numEx
    % Positive if even, negative if odd
    if rem(i, 2)
        % Negative
        trial = [1; x1N(countNeg); x2N(countNeg)];
        countNeg = rem(countNeg, numEx) + 1;
        target = -1;
    else
        % Positive
        trial = [1; x1P(countPos); x2P(countPos)];
        countPos = rem(countPos, numEx) + 1;
        target = 1;
    end
    z = w'*trial;
    y = 1/(1 + exp(-z));
    w = w + learnRate*(target - y)*(1 - y)*y*trial;
end

% Plot discriminating function
theYs = -w(2)/w(3)*theXs - w(1)/w(3);
plot(theXs, theYs, 'c', 'linewidth', 3);





hold off;

legend('Positive instances','Negative instances','Binary + Squared error','Linear + Squared error','Logit + Squared error');
