% MODSELC Model selection
%
% V = MODSELC(A,W,N,NREP)
% V = A*(W*MODSELC([],N,NREP))
%
% INPUT
% A Dataset used for training base classifiers and/or selection
% B Dataset used for testing (executing) the selector
% W Set of trained or untrained base classifiers
% N Number of crossvalidations, default 10
% NREP Number of crossvalidation repetitions
%
% OUTPUT
% V Selected classidfer
%
% DESCRIPTION
% This routine selects out of a set of given classifiers stored in W the
% best one on the basis of N-fold crossvalidation (see CROSSVAL), which
% might be repeated NREP times. If W contains a set of already trained
% classifiers, N and NREP are neglected and just the best classifier
% according to the evaluation set A is returned.
%
% This routine can be considered as a classifier combiner based on global
% selection. See DCSC for local, dynamic selection.
%
% SEE ALSO (<a href="http://37steps.com/prtools">PRTools Guide</a>)
% DATASETS, MAPPINGS, STACKED, DCSC, CLASSC, TESTD, LABELD
% Copyright: R.P.W. Duin, r.p.w.duin@37steps.com
% Faculty EWI, Delft University of Technology
% P.O. Box 5031, 2600 GA Delft, The Netherlands
function OUT = modselc(par1,par2,par3,par4)
name = 'Classifier Model Selection';
DefaultN = 10;
DefaultNrep = 1;
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% Empty Call MODSELC, MODSELC([],N,NREP)
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
if nargin < 1 | isempty(par1)
if nargin > 1 & (ismapping(par2) | iscell(par2))
if nargin < 4 | isempty(par4), par4 = DefaultNrep; end
if nargin < 3 | isempty(par3), par3 = DefaultN; end
OUT = prmapping(mfilename,'untrained',{par2,par3,par4});
else
if nargin < 3 | isempty(par3), par3 = DefaultNrep; end
if nargin < 2 | isempty(par2), par2 = DefaultN; end
% If there are no inputs, return an untrained mapping.
% (PRTools transfers the latter into the first)
OUT = prmapping(mfilename,'combiner',{par2,par3});
end
OUT = setname(OUT,name);
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% Storing Base Classifiers: W*MODSELC, MODSELC(W,N,NREP)
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
elseif ismapping(par1)
if nargin < 3 | isempty(par3), par3 = DefaultNrep; end
if nargin < 2 | isempty(par2), par2 = DefaultN; end
% call like OUT = MODSELC(W,N) or OUT = W*MODSELC([],N)
% store trained or untrained base classifiers,
% ready for later training of the combiner
BaseClassf = par1;
if ~isparallel(BaseClassf) & ~isstacked(BaseClassf)
error('Parallel or stacked set of base classifiers expected');
end
OUT = prmapping(mfilename,'untrained',{BaseClassf,par2,par3});
OUT = setname(OUT,name);
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% Selection (and training base classifiers if needed):
% A*(W*MODSELC), A*(W*MODSELC([],N,NREP)), MODSELC(A,W,N,NREP)
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
elseif isdataset(par1) & ismapping(par2) & ...
(isuntrained(par2) | (istrained(par2) & (isstacked(par2) | isparallel(par2))))
% call like OUT = MODSELC(TrainSet,W,N,NREP) or TrainSet*(W*MODSELC([],N,NREP))
% (PRTools transfers the latter into the first)
% W is a set of trained or untrained set classifiers.
if nargin < 4 | isempty(par4), par4 = DefaultNrep; end
if nargin < 3 | isempty(par3), par3 = DefaultN; end
TrainSet = par1;
BaseClassf = par2;
N = par3;
Nrep = par4;
isvaldfile(TrainSet,1,2); % at least one object per class, 2 classes
TrainSet = setprior(TrainSet,getprior(TrainSet,0)); % avoid many warnings
if ~(isstacked(BaseClassf) | isparallel(BaseClassf))
if iscombiner(BaseClassf)
error('No base classifiers found')
end
Data = getdata(BaseClassf);
BaseClassf = Data.BaseClassf; % base classifiers were already stored
end
if isuntrained(BaseClassf) % base classifiers untrained, crossval needed
randstate = randreset; % makes routine reproducing
if isparallel(BaseClassf)
BaseClassf = getdata(BaseClassf);
tsize = BaseClassf{end};
if ismapping(tsize)
error(['Training of parallel combined untrained classifier not possible.' ...
newline 'Feature sizes should be stored in the classifier first.'])
end
csize = cumsum([0 tsize(:)']);
n = length(BaseClassf)-1;
e = zeros(1,n);
rstate = randreset;
s = sprintf('Crossvalidating %i base classifiers: ',n);
prwaitbar(n,s)
for j=1:n
prwaitbar(n,j,[s getname(BaseClassf{j})]);
randreset(1);
e(j) = crossval(TrainSet(:,csize(j)+1:csize(j+1)),BaseClassf{j},N,Nrep);
end
prwaitbar(0);
[ee,k] = min(e);
J = [csize(k)+1:csize(k+1)];
OUT = TrainSet(:,J)*BaseClassf{k};
OUT = featsel(csize(end),J)*OUT;
else
if isstacked(BaseClassf)
BaseClassf = getdata(BaseClassf);
end
randreset(1);
e = crossval(TrainSet,BaseClassf,N,Nrep);
[ee,k] = min(e);
OUT = TrainSet*BaseClassf{k};
end
randreset(randstate);
else % base classifiers are trained, check label lists and find best one
BaseClassf = getdata(BaseClassf);
n = length(BaseClassf);
for j=1:n
if ~isequal(getlabels(BaseClassf{j}),getlablist(TrainSet))
error('Training set and base classifiers should deal with same labels')
end
end
e = testc(TrainSet,BaseClassf);
[ee,k] = min([e{:}]);
OUT = BaseClassf{k};
end
else
error('Illegal input');
end