%CLEVALF Classifier evaluation (feature size curve)
%
% E = CLEVALF(A,CLASSF,FEATSIZES,TRAINSIZE,NREPS,S,TESTFUN)
%
% INPUT
% A Training dataset.
% CLASSF The untrained classifier to be tested.
% FEATSIZES Vector of feature sizes (default: all sizes)
% TRAINSIZE Number of objects/fraction of training set size (see GENDAT)
% or generator mapping.
% NREPS Number of repetitions (default: 1)
% S Independent test dataset (optional)
% TESTFUN Mapping,evaluation function (default classification error)
%
% OUTPUT
% E Structure with results
% See PLOTE for a description
%
% DESCRIPTION
% Generates at random for all feature sizes stored in FEATSIZES training
% sets of the given TRAINSIZE out of the dataset A. See GENDAT for the
% interpretation of TRAINSIZE. These are used for training the untrained
% classifier CLASSF. The result is tested by all unused ojects of A, or,
% if given, by the test dataset S. This is repeated N times. If no testset
% is given and if LEARNSIZE is not given or empty, the training set is
% bootstrapped. If a testset is given, the default training set size is
% the entire training set. Default FEATSIZES: all feature sizes.
% The mean erors are stored in E. The observed standard deviations are
% stored in S. The default test routine is classification error estimation
% by TESTC([],'crisp').
%
% See CLEVALFS for how to construct feature curves in addition with
% automatic feature extraction / selection.
%
% This function uses the RAND random generator and thereby reproduces only
% if its seed is saved and reset.
%
% SEE ALSO (<a href="http://37steps.com/prtools">PRTools Guide</a>)
% MAPPINGS, DATASETS, CLEVAL, CLEVALFS, TESTC, PLOTE, GENDAT
% Copyright: R.P.W. Duin, duin@ph.tn.tudelft.nl
% Faculty of Applied Sciences, Delft University of Technology
% P.O. Box 5046, 2600 GA Delft, The Netherlands
% $Id: clevalf.m,v 1.8 2008/07/03 09:05:50 duin Exp $
function e = clevalf(a,classf,featsizes,learnsize,n,Tset,testfun)
[m,k] = size(a);
if (nargin < 7) | isempty(testfun)
testfun = testc([],'crisp');
end;
if nargin < 6, Tset = []; end
if nargin < 5, n = 1; end;
if nargin < 4 | isempty(learnsize), learnsize = [0.5]; end
if nargin < 3 | isempty(featsizes), featsizes = [1:k]; end
if isdataset(classf) & ismapping(a) % correct for old order
dd = a; a = classf; classf = {dd};
end
if isdataset(classf) & iscell(a) & ismapping(a{1}) % correct for old order
dd = a; a = classf; classf = dd;
end
if ~iscell(classf), classf = {classf}; end
isdataset(a);
ismapping(classf{1});
a = setprior(a,getprior(a));
if ~isempty(Tset), isdataset(Tset); T = Tset; end
[m,k,c] = getsize(a);
featsizes(find(featsizes > k)) = [];
featsizes = featsizes(:)';
if ~ismapping(learnsize) && (length(learnsize) > 1 & length(learnsize) ~= c)
error('Learnsize should be scalar or a vector with length equal to the class size')
end
r = length(classf(:));
e.error = zeros(r,length(featsizes));
e.std = zeros(r,length(featsizes));
e.xvalues = featsizes;
e.n = n;
datname = getname(a);
if ~isempty(datname)
e.title = ['Feature curve for ' getname(a)];
end
e.xlabel= 'Feature size';
if n > 1
e.ylabel= ['Averaged error (' num2str(n) ' experiments)'];
else
e.ylabel = 'Error';
end
if featsizes(end)/featsizes(1) > 20
e.plot = 'semilogx';
else
e.plot = 'plot';
end
e.names = [];
s1 = sprintf('clevalf: %i classifiers: ',r);
prwaitbar(r,s1);
e1 = zeros(n,length(featsizes));
seed = randreset;
% loop over all classifiers
for q = 1:r
isuntrained(classf{q});
name = getname(classf{q});
prwaitbar(r,q,[s1 name]);
e.names = char(e.names,name);
e1 = zeros(n,length(featsizes));
randreset(seed); % take care that classifiers use same training set
seed2 = seed;
s2 = sprintf('clevalf: %i repetitions: ',n);
prwaitbar(n,s2);
for i = 1:n
prwaitbar(n,i,[s2 int2str(i)]);
randreset(seed2);
if isempty(Tset)
[b,T] = gendat(a,learnsize);
elseif ~isempty(learnsize)
b = gendat(a,learnsize);
else
b = a;
end
seed2 = randreset;
nfeatsizes = length(featsizes);
s3 = sprintf('clevalf: %i feature sizes: ',nfeatsizes);
prwaitbar(nfeatsizes,s2);
for j=1:nfeatsizes
f = featsizes(j);
prwaitbar(nfeatsizes,j,[s3 int2str(j) ' (' int2str(f) ')']);
e1(i,j) = T(:,1:f)*(b(:,1:f)*classf{q})*testfun;
end
prwaitbar(0)
end
prwaitbar(0)
e.error(q,:) = mean(e1,1);
if n == 1
e.std(q,:) = zeros(1,size(e.std,2));
else
e.std(q,:) = std(e1)/sqrt(n);
end
end
prwaitbar(0)
e.names(1,:) = [];
return