Home > analysis > WBCperformance_4.m

WBCperformance_4

PURPOSE ^

Evaluate the performance of a tracking algorithm

SYNOPSIS ^

function S = WBCperformance_4(iFolder,tracksR)

DESCRIPTION ^

Evaluate the performance of a tracking algorithm

Based on RMSE, PTP, and TL
Add the detection performance as well

INPUT
 iFolder    - the dataset folder name
 tracksR    - the mx4 track matrix
              [ trackID frameID Col Row ]

OUTPUT
 S      - a structure contains:
           Name: name of the dataset
           GT      - the GT information of that dataset
           Eden    - the Eden Jscore method structure
           Kalman  - the Single Hypothesis method
           Rich    - the Multiple Hypotheses method
           GT, Eden, Kalman, and Rich contains the same structure
   Rich - a structure element contains:
           Dataset - name of the dataset
           TPR     - True Positive Rate (Detection)
           PPV     - Precision (Detection)
           FDR     - False Discovery Rate (Detection)
           TP      - number of True Positives / frame
           FP      - number of False Positives / frame
           FN      - number of False Negatives / frame
           RSMEx   - RMSE array
           tracks  - tracks array
           IDs     - All IDs of tracked cells
           nTR     - number of tracked cells
           nFR     - number of frames
           RMSE    - Root Means Square Error
           PTP     - Percentage of Tracked Positions
           TL      - Track length
           VD      - Velocity Distribution
           CC      - structure for Colliding Cells
           NC      - structure for Non-colliding Cells

SEE ALSO
 convertFrames2Array, classifyCollision, get_velocityDistribution,
 NKTdetectionEvaluation

EXAMPLE
 load _trackMH;
 S = WBCperformance3('clpsaline1bl#3',tracksR);

Written by Rich Nguyen (rich.uncc@gmail.com)
Version 3.0, Nov 2009.

--------------------------------------------------------------------------

CROSS-REFERENCE INFORMATION ^

This function calls: This function is called by:

SOURCE CODE ^

0001 function S = WBCperformance_4(iFolder,tracksR)
0002 %Evaluate the performance of a tracking algorithm
0003 %
0004 %Based on RMSE, PTP, and TL
0005 %Add the detection performance as well
0006 %
0007 %INPUT
0008 % iFolder    - the dataset folder name
0009 % tracksR    - the mx4 track matrix
0010 %              [ trackID frameID Col Row ]
0011 %
0012 %OUTPUT
0013 % S      - a structure contains:
0014 %           Name: name of the dataset
0015 %           GT      - the GT information of that dataset
0016 %           Eden    - the Eden Jscore method structure
0017 %           Kalman  - the Single Hypothesis method
0018 %           Rich    - the Multiple Hypotheses method
0019 %           GT, Eden, Kalman, and Rich contains the same structure
0020 %   Rich - a structure element contains:
0021 %           Dataset - name of the dataset
0022 %           TPR     - True Positive Rate (Detection)
0023 %           PPV     - Precision (Detection)
0024 %           FDR     - False Discovery Rate (Detection)
0025 %           TP      - number of True Positives / frame
0026 %           FP      - number of False Positives / frame
0027 %           FN      - number of False Negatives / frame
0028 %           RSMEx   - RMSE array
0029 %           tracks  - tracks array
0030 %           IDs     - All IDs of tracked cells
0031 %           nTR     - number of tracked cells
0032 %           nFR     - number of frames
0033 %           RMSE    - Root Means Square Error
0034 %           PTP     - Percentage of Tracked Positions
0035 %           TL      - Track length
0036 %           VD      - Velocity Distribution
0037 %           CC      - structure for Colliding Cells
0038 %           NC      - structure for Non-colliding Cells
0039 %
0040 %SEE ALSO
0041 % convertFrames2Array, classifyCollision, get_velocityDistribution,
0042 % NKTdetectionEvaluation
0043 %
0044 %EXAMPLE
0045 % load _trackMH;
0046 % S = WBCperformance3('clpsaline1bl#3',tracksR);
0047 %
0048 %Written by Rich Nguyen (rich.uncc@gmail.com)
0049 %Version 3.0, Nov 2009.
0050 %
0051 %--------------------------------------------------------------------------
0052 
0053 %% Init
0054 MAX_TRACK_LENGTH = 2;
0055 iPath = '/wbc/Data/';
0056 warning off;
0057 iPath= strcat(iPath,iFolder);
0058 cd(strcat(iPath));
0059 
0060 load ../_param/_parameters.mat
0061 if strcmp(iFolder,'F2') || strcmp(iFolder,'F3')
0062     P.tCollision_Distance = 25;
0063 end
0064 load _frames;
0065 framesR = convertFrames2Array(frames);
0066 load _tracksGT;
0067 % tracksGT nX4   [ trackID frameID Col Row ]
0068 load _Counter;
0069 
0070 %% Remove invalid position
0071 tracksGT(tracksGT(:,2)==999,:)=[];
0072 tr_ids = unique(tracksR(:,1));
0073 
0074 for id = 1: length(tr_ids)
0075     track = tracksR(tracksR(:,1)==tr_ids(id),:);
0076     if size(track,1) <= MAX_TRACK_LENGTH;
0077         tracksR(tracksR(:,1)==tr_ids(id),:) = [];
0078     end
0079 end
0080 tracksGap = zeros(0,4);
0081 
0082 
0083 
0084 S  = struct;
0085 RMSE = [];
0086 lengthsTR = [];
0087 tracksGT = round(tracksGT);
0088 frGT = unique(tracksGT(:,2));
0089 frAU = unique(tracksR(:,2));
0090 
0091 % Make sure that tracksGT and tracksR have the same frame number.
0092 frDEL = setdiff(frGT,frAU);
0093 for fr = 1: length(frDEL)
0094     tracksGT(tracksGT(:,2)==frDEL(fr),:) =[];
0095 end
0096 
0097 frDEL = setdiff(frAU,frGT);
0098 for fr = 1: length(frDEL)
0099     tracksR(tracksR(:,2)==frDEL(fr),:) =[];
0100 end
0101 
0102 %% Add Available Column
0103 availableR  = ones(size(tracksR,1),1);
0104 tracksR = horzcat(tracksR, availableR);
0105 
0106 availableGT  = ones(size(tracksGT,1),1);
0107 tracksGT = horzcat(tracksGT, availableGT);
0108 %% Calculate the performance factors for ALL DATA
0109 tracksGT_id = unique(tracksGT(:,1));
0110 n_tracksGT  = length(tracksGT_id);
0111 for tr = 1 : n_tracksGT
0112     trackGT = tracksGT(tracksGT(:,1)==tracksGT_id(tr),:);
0113     % compare to all automatic tracks
0114     % only compare to the one that are available
0115     tracksR_id = unique(tracksR(tracksR(:,5)==1 ,1));
0116 %     tracksR_id = unique(tracksR(:,1));
0117     n_tracksR  =  length(tracksR_id);
0118     maxOverlapFrame = 0;
0119     selectedFr       = [];
0120     for trR = 1 : n_tracksR
0121         trackR = tracksR(tracksR(:,1) == tracksR_id(trR),:);
0122         % Calculate how many overlapping frame between GT and R
0123         startFr = max( min(trackGT(:,2)), min(trackR(:,2)) );
0124         endFr   = min( max(trackGT(:,2)), max(trackR(:,2)) );
0125         % Trim extra frame
0126         trimR = trackR (trackR (:,2) >= startFr & trackR (:,2) <= endFr,:);
0127         trimGT= trackGT(trackGT(:,2) >= startFr & trackGT(:,2) <= endFr,:);
0128         % Build distance vector
0129         distanceV = zeros(size(trimR,1),1);
0130         for p = 1: size(trimR,1)
0131            posR = trimR(p,:);
0132            posGT= trimGT(trimGT(:,2)== posR(2),:);
0133            if ~isempty(posR) && ~isempty(posGT)
0134                 distanceV(p) = sqrt( (posR(3) - posGT(3) )^2 + (posR(4) - posGT(4) )^2 );
0135            end
0136         end       
0137         overlapFr = distanceV < P.tEval_Distance;
0138         if sum(overlapFr) > maxOverlapFrame
0139             maxOverlapFrame = sum(overlapFr);
0140             selectedFr       = trimR(overlapFr,:);
0141             selectedDis      = distanceV(overlapFr);
0142            
0143         end
0144     end
0145     if ~isempty(selectedFr)
0146 
0147         % Filling the frame gap if any
0148         tracksGap = zeros(0,5);
0149         % check each positions
0150         for po = 1: size(selectedFr,1)-1
0151             % if not consecutive
0152             if selectedFr(po+1,2) ~= selectedFr(po,2)+1
0153                 % find out how many frame is missing
0154                 nGap = selectedFr(po+1,2) - selectedFr(po,2);
0155                 % filling in the gap
0156                 xPos = linspace(selectedFr(po,3),selectedFr(po+1,3),nGap+1);
0157                 yPos = linspace(selectedFr(po,4),selectedFr(po+1,4),nGap+1);
0158                 tracksGap = vertcat(tracksGap, ...
0159                     horzcat(selectedFr(po,1)*ones(nGap-1,1), ...
0160                     (selectedFr(po,2)+1:1:selectedFr(po+1,2)-1)', ...
0161                     xPos(2:end-1)',...
0162                     yPos(2:end-1)',ones(nGap-1,1)));
0163                 
0164             end
0165         end
0166         selectedFr= vertcat(selectedFr, tracksGap);
0167         selectedDis = vertcat(selectedDis, zeros(size(selectedFr,1)-size(selectedDis,1),1) );
0168         % Check off the selected track
0169         tracksR(tracksR(:,1)== selectedFr(1),5) = 0;
0170         RMSE = [RMSE; tracksGT_id(tr)*ones(size(selectedFr,1),1), ...
0171             selectedFr(:,1),...
0172             selectedFr(:,2),...
0173             selectedDis ];
0174     end
0175 end
0176 
0177 RMSEx = [];
0178 if (~isempty(RMSE))
0179     ids= unique(RMSE(:,1));
0180     for id = 1: length(ids)
0181         tr = ids(id);
0182         track = RMSE(RMSE(:,1)== tr,:);
0183         
0184         idx = unique(track(:,2));
0185         % Get the maximum correspondence among the automatic track
0186         % this is to ensure 1:1 correspondence
0187         maxlength = 0;
0188         track_AUTO = [];
0189         for id2 = 1: length(idx)
0190             temp = track(track(:,2) == idx(id2),:);
0191             if length(temp) > maxlength
0192                 maxlength = length(temp);
0193                 track_AUTO = temp;
0194             end
0195         end
0196         
0197         lengthTR = size(track_AUTO,1);
0198         if lengthTR > 2
0199             lengthsTR = [lengthsTR; lengthTR ];
0200         end
0201         RMSEx  = [RMSEx; track_AUTO];
0202     end
0203     
0204     % PTP
0205     PTP = size(RMSEx,1) / size(tracksGT,1) ;
0206     % TL
0207     TL = mean(lengthsTR);
0208     % RMSE
0209     mRMSE = mean(RMSEx(:,end));
0210     
0211     
0212     %% Calculate the performance factors for COLLIDING CELLS
0213     
0214     tracks_C = [];
0215     RMSEx_C = [];
0216     lengthsTR_C = [];
0217     tracksGT_C = [];
0218     tracksGT_C2 = [];
0219     
0220     RMSEx_NC = RMSEx;
0221     tracks_NC = tracksR;
0222     lengthsTR_NC = [];
0223     tracksGT_NC = tracksGT;
0224     tracksGT_NC2 = tracksGT;
0225     
0226     collidingCells = classifyCollision(tracksGT,P);
0227     fr_Cx = [collidingCells.Pairs(:,1:2); [collidingCells.Pairs(:,1) collidingCells.Pairs(:,3)] ];
0228     
0229     for id = 1: length(collidingCells.Ids)
0230         
0231         fr_C = unique(fr_Cx(fr_Cx(:,2)== collidingCells.Ids(id),1));
0232         fr_C = unique([fr_C - 2 ; fr_C - 1 ; fr_C ; fr_C + 1 ; fr_C + 2] );
0233         fr_C = unique([fr_C - 1 ; fr_C ; fr_C + 1] );
0234         
0235         for id_fr = 1: length(fr_C)
0236             fr = fr_C(id_fr);
0237             rmse_C = RMSEx(RMSEx(:,1) == collidingCells.Ids(id) & RMSEx(:,3) == fr,:);
0238             RMSEx_C = [RMSEx_C; rmse_C];
0239             RMSEx_NC(RMSEx_NC(:,1) == collidingCells.Ids(id) & RMSEx_NC(:,3) == fr,:)=[];
0240             %             RMSEx_NC(RMSEx_NC(:,1) == collidingCells.Ids(id),:)=[];
0241             
0242             trackGT_C = tracksGT(tracksGT(:,1) == collidingCells.Ids(id) & tracksGT(:,2) == fr,:);
0243             tracksGT_C = [tracksGT_C; trackGT_C];
0244             tracksGT_NC(tracksGT_NC(:,1) == collidingCells.Ids(id) & tracksGT_NC(:,2) == fr,:)=[];
0245             %             tracksGT_NC(tracksGT_NC(:,1) == collidingCells.Ids(id),:)=[];
0246         end
0247         
0248     end
0249     
0250     idx = unique(RMSEx_C(:,2));
0251     for id  = 1: length(idx)
0252         track_C = tracksR(tracksR(:,1) == idx(id),:);
0253         tracks_C = [tracks_C; track_C];
0254         tracks_NC(tracks_NC(:,1) == idx(id),:) = [];
0255         
0256         lengthTR_C = size(track_C,1);
0257         lengthsTR_C = [lengthsTR_C; lengthTR_C];
0258     end
0259     
0260     fr_C2x = unique(collidingCells.Pairs(:,1));
0261     for id_fr = 1: length(fr_C2x)
0262         fr = fr_C2x(id_fr);
0263         trackGT_C2 = tracksGT(tracksGT(:,2) == fr,:);
0264         tracksGT_C2 = [tracksGT_C2; trackGT_C2];
0265         tracksGT_NC2(tracksGT_NC2(:,2) == fr,:)=[];
0266     end
0267     
0268     % PTP
0269     PTP_C = size(RMSEx_C,1) / size(tracksGT_C,1) ;
0270     % TL
0271     TL_C = mean(lengthsTR_C);
0272     % RMSE
0273     mRMSE_C = mean(RMSEx_C(:,end));
0274     
0275     
0276     %% Calculate the performance factors for NON-COLLIDING CELL
0277     idx = unique(RMSEx_NC(:,2));
0278     for id  = 1: length(idx)
0279         track_NC = tracksR(tracksR(:,1) == idx(id),:);
0280         lengthTR_NC = size(track_NC,1);
0281         lengthsTR_NC = [lengthsTR_NC; lengthTR_NC];
0282     end
0283     
0284     
0285     % PTP
0286     PTP_NC = size(RMSEx_NC,1) / size(tracksGT_NC,1) ;
0287     % TL
0288     TL_NC = mean(lengthsTR_NC);
0289     % RMSE
0290     mRMSE_NC = mean(RMSEx_NC(:,end));
0291             
0292     %% Velocity distribution
0293     V = get_velocityDistribution(tracksR);
0294     V_C = get_velocityDistribution(tracks_C);
0295     V_NC = get_velocityDistribution(tracks_NC);
0296     
0297     %% Run detection evaluation
0298     [d_specs] = NKTdetectionEvaluation(iFolder,framesR,tracksGT,0,P);
0299     [d_specs_C] = NKTdetectionEvaluation(iFolder,framesR,tracksGT_C2,0,P);
0300     [d_specs_NC] = NKTdetectionEvaluation(iFolder,framesR,tracksGT_NC2,0,P);
0301     
0302     %% Construct S
0303     S.Dataset = iFolder;
0304     S.TPR = mean(d_specs(:,2));
0305     S.PPV = mean(d_specs(:,3));
0306     S.FDR = mean(d_specs(:,4));
0307     S.TP = mean(d_specs(:,5));
0308     S.FP = mean(d_specs(:,6));
0309     S.FN = mean(d_specs(:,7));
0310     
0311     S.RMSEx = RMSEx;
0312     S.tracks = tracksR;
0313     S.tracksGT = tracksGT;
0314     S.frames = framesR;
0315     S.IDs = unique(tracksR(:,1))';
0316     S.nTR = length(unique(tracksR(:,1)));
0317     S.nFR = length(unique(RMSEx(:,3)));
0318     
0319     S.RMSE = mRMSE;
0320     S.PTP = PTP;
0321     S.TL = TL;
0322     S.VD = [mean(V(:,1)) std(V(:,1))];
0323     
0324     
0325     S.CC.TPR = mean(d_specs_C(:,2));
0326     S.CC.PPV = mean(d_specs_C(:,3));
0327     S.CC.FDR = mean(d_specs_C(:,4));
0328     S.CC.TP = mean(d_specs_C(:,5));
0329     S.CC.FP = mean(d_specs_C(:,6));
0330     S.CC.FN = mean(d_specs_C(:,7));
0331     
0332     S.CC.RMSEx = RMSEx_C;
0333     S.CC.Cases = collidingCells.Pairs;
0334     if ~isempty(tracks_C)
0335         S.CC.tracks = tracks_C;
0336         S.CC.tracksGT = tracksGT_C;
0337         S.CC.IDs = unique(tracks_C(:,1))';
0338         S.CC.nTR = length(unique(tracks_C(:,1)));
0339         S.CC.nFR = length(unique(RMSEx_C(:,3)));
0340         
0341         S.CC.RMSE = mRMSE_C;
0342         S.CC.PTP = PTP_C;
0343         S.CC.TL = TL_C;
0344         S.CC.VD = [mean(V_C(:,1)) std(V_C(:,1))];
0345     else
0346         S.CC.tracks = [];
0347         S.CC.tracksGT = tracksGT_CC;
0348         S.CC.IDs = -1;
0349         S.CC.nTR = 0;
0350         S.CC.nFR = 0;
0351         
0352         S.CC.RMSE = 0;
0353         S.CC.PTP = 0;
0354         S.CC.TL = 0;
0355         S.CC.VD = [0 0];
0356     end
0357     
0358     
0359    
0360     S.NC.TPR = mean(d_specs_NC(:,2));
0361     S.NC.PPV = mean(d_specs_NC(:,3));
0362     S.NC.FDR = mean(d_specs_NC(:,4));
0363     
0364     S.NC.TP = mean(d_specs_NC(:,5));
0365     S.NC.FP = mean(d_specs_NC(:,6));
0366     S.NC.FN = mean(d_specs_NC(:,7));
0367     
0368     S.NC.RMSEx = RMSEx_NC;
0369     
0370     S.NC.tracks = tracks_NC;
0371     S.NC.tracksGT = tracksGT_NC;
0372     S.NC.IDs = unique(tracks_NC(:,1))';
0373     S.NC.nTR = length(unique(tracks_NC(:,1)));
0374     S.NC.nFR = length(unique(RMSEx_NC(:,3)));
0375     
0376     S.NC.RMSE = mRMSE_NC;
0377     S.NC.PTP = PTP_NC;
0378     S.NC.TL = TL_NC;
0379     S.NC.VD = [mean(V_NC(:,1)) std(V_NC(:,1))];
0380     
0381     S.frames = framesR;
0382     S.Counter = Counter;
0383 end
0384 %     save _performance.mat S

Generated on Thu 17-Mar-2011 14:45:51 by m2html © 2005