%% Logistic Regression vs Bayes optimal decision boundary

%% Initialization
clear all; close all; clc

%% Create Data
%  The first two columns contains the point coordinates
%  and the third column contains the label.

meanP = 6;
meanN = 2;
x1P = (randn(1,100) + meanP)';
x2P = (randn(1,100) + meanP)';
x1N = (randn(1,100) + meanN)';
x2N = (randn(1,100) + meanN)';
X = [x1P x2P; x1N x2N];
y = [ones(100,1); zeros(100,1)];

%  Setup the data matrix appropriately, and add ones for the intercept term
[m, n] = size(X);

% Add intercept term to x and X_test
X = [ones(m, 1) X];

% Initialize fitting parameters
initial_theta = zeros(n + 1, 1);


%% ============= Part 3: Optimizing using fminunc  =============
%  In this exercise, you will use a built-in function (fminunc) to find the
%  optimal parameters theta.

%  Set options for fminunc
options = optimset('GradObj', 'on', 'MaxIter', 400);

%  Run fminunc to obtain the optimal theta
%  This function will return theta and the cost 
[theta, cost] = ...
	fminunc(@(t)(costFunction(t, X, y)), initial_theta, options);

% Print theta to screen
fprintf('Cost at theta found by fminunc: %f\n', cost);
fprintf('theta: \n');
fprintf(' %f \n', theta);

thetaLR = theta;

plotData(X(:,2:3), y);
hold on;

% Bayes optimal
thetaBO(1) = -0.5 * [meanP - meanN; meanP - meanN]' * [meanP + meanN; meanP + meanN];
thetaBO(2) = meanP - meanN;
thetaBO(3) = meanP - meanN;
plotMyDecisionBoundary(thetaBO, X, y, 'Bayes Optimal', 'k');

% Plot Boundary
plotMyDecisionBoundary(thetaLR, X, y, 'Decision Boundary', 'g');

% Labels and Legend
legend('Positive', 'Negative', 'Bayes Optimal', 'Decision Boundary')
xlabel('Feature 1')
ylabel('Feature 2')
title('Logistic Regression')

hold off;

fprintf('\nProgram paused. Press enter to continue.\n');
pause;

