Home > analysis > WBCperformance3.m

WBCperformance3

PURPOSE ^

Evaluate the performance of a tracking algorithm

SYNOPSIS ^

function S = WBCperformance3(iFolder,tracksR)

DESCRIPTION ^

Evaluate the performance of a tracking algorithm

Based on RMSE, PTP, and TL
Add the detection performance as well

INPUT
 iFolder    - the dataset folder name
 tracksR    - the mx4 track matrix
              [ trackID frameID Col Row ]

OUTPUT
 S      - a structure contains:
           Name: name of the dataset
           GT      - the GT information of that dataset
           Eden    - the Eden Jscore method structure
           Kalman  - the Single Hypothesis method
           Rich    - the Multiple Hypotheses method
           GT, Eden, Kalman, and Rich contains the same structure
   Rich - a structure element contains:
           Dataset - name of the dataset
           TPR     - True Positive Rate (Detection)
           PPV     - Precision (Detection)
           FDR     - False Discovery Rate (Detection)
           TP      - number of True Positives / frame
           FP      - number of False Positives / frame
           FN      - number of False Negatives / frame
           RSMEx   - RMSE array
           tracks  - tracks array
           IDs     - All IDs of tracked cells
           nTR     - number of tracked cells
           nFR     - number of frames
           RMSE    - Root Means Square Error
           PTP     - Percentage of Tracked Positions
           TL      - Track length
           VD      - Velocity Distribution
           CC      - structure for Colliding Cells
           NC      - structure for Non-colliding Cells

SEE ALSO
 convertFrames2Array, classifyCollision, get_velocityDistribution,
 NKTdetectionEvaluation

EXAMPLE
 load _trackMH;
 S = WBCperformance3('clpsaline1bl#3',tracksR);

Written by Rich Nguyen (rich.uncc@gmail.com)
Version 3.0, Nov 2009.

--------------------------------------------------------------------------

CROSS-REFERENCE INFORMATION ^

This function calls: This function is called by:

SOURCE CODE ^

0001 function S = WBCperformance3(iFolder,tracksR)
0002 %Evaluate the performance of a tracking algorithm
0003 %
0004 %Based on RMSE, PTP, and TL
0005 %Add the detection performance as well
0006 %
0007 %INPUT
0008 % iFolder    - the dataset folder name
0009 % tracksR    - the mx4 track matrix
0010 %              [ trackID frameID Col Row ]
0011 %
0012 %OUTPUT
0013 % S      - a structure contains:
0014 %           Name: name of the dataset
0015 %           GT      - the GT information of that dataset
0016 %           Eden    - the Eden Jscore method structure
0017 %           Kalman  - the Single Hypothesis method
0018 %           Rich    - the Multiple Hypotheses method
0019 %           GT, Eden, Kalman, and Rich contains the same structure
0020 %   Rich - a structure element contains:
0021 %           Dataset - name of the dataset
0022 %           TPR     - True Positive Rate (Detection)
0023 %           PPV     - Precision (Detection)
0024 %           FDR     - False Discovery Rate (Detection)
0025 %           TP      - number of True Positives / frame
0026 %           FP      - number of False Positives / frame
0027 %           FN      - number of False Negatives / frame
0028 %           RSMEx   - RMSE array
0029 %           tracks  - tracks array
0030 %           IDs     - All IDs of tracked cells
0031 %           nTR     - number of tracked cells
0032 %           nFR     - number of frames
0033 %           RMSE    - Root Means Square Error
0034 %           PTP     - Percentage of Tracked Positions
0035 %           TL      - Track length
0036 %           VD      - Velocity Distribution
0037 %           CC      - structure for Colliding Cells
0038 %           NC      - structure for Non-colliding Cells
0039 %
0040 %SEE ALSO
0041 % convertFrames2Array, classifyCollision, get_velocityDistribution,
0042 % NKTdetectionEvaluation
0043 %
0044 %EXAMPLE
0045 % load _trackMH;
0046 % S = WBCperformance3('clpsaline1bl#3',tracksR);
0047 %
0048 %Written by Rich Nguyen (rich.uncc@gmail.com)
0049 %Version 3.0, Nov 2009.
0050 %
0051 %--------------------------------------------------------------------------
0052 
0053 %% Init
0054 MAX_TRACK_LENGTH = 2;
0055 iPath = '/wbc/Data/';
0056 warning off;
0057 iPath= strcat(iPath,iFolder);
0058 cd(strcat(iPath));
0059 
0060 load ../_param/_parameters.mat
0061 if strcmp(iFolder,'F2') || strcmp(iFolder,'F3')
0062     P.tCollision_Distance = 25;
0063 end
0064 load _frames;
0065 framesR = convertFrames2Array(frames);
0066 load _tracksGT;
0067 % tracksGT nX4   [ trackID frameID Col Row ]
0068 load _Counter;
0069 
0070 %% Remove invalid position
0071 tracksGT(tracksGT(:,2)==999,:)=[];
0072 tr_ids = unique(tracksR(:,1));
0073 
0074 for id = 1: length(tr_ids)
0075     track = tracksR(tracksR(:,1)==tr_ids(id),:);
0076     if size(track,1) <= MAX_TRACK_LENGTH;
0077         tracksR(tracksR(:,1)==tr_ids(id),:) = [];
0078     end
0079 end
0080 
0081 S  = struct;
0082 RMSE = [];
0083 lengthsTR = [];
0084 
0085 frGT = unique(tracksGT(:,2));
0086 frAU = unique(tracksR(:,2));
0087 
0088 % Make sure that tracksGT and tracksR have the same frame number.
0089 frDEL = setdiff(frGT,frAU);
0090 for fr = 1: length(frDEL)
0091     tracksGT(tracksGT(:,2)==frDEL(fr),:) =[];
0092 end
0093 
0094 frDEL = setdiff(frAU,frGT);
0095 for fr = 1: length(frDEL)
0096     tracksR(tracksR(:,2)==frDEL(fr),:) =[];
0097 end
0098 
0099 %% Add Available Column
0100 availableR  = ones(size(tracksR,1),1);
0101 tracksR = horzcat(tracksR, availableR);
0102 
0103 availableGT  = ones(size(tracksGT,1),1);
0104 tracksGT = horzcat(tracksGT, availableGT);
0105 %% Calculate the performance factors for ALL DATA
0106 n_tracksGT = tracksGT(end,1);
0107 for tr = 1 : n_tracksGT
0108     trackGT = tracksGT(tracksGT(:,1)==tr,:);
0109     frameGT_start = min(trackGT(:,2));
0110     frameGT_end = max(trackGT(:,2));
0111     fr = frameGT_start;
0112     while (fr <=frameGT_end)
0113         posGT = trackGT(trackGT(:,2)==fr,:);
0114         % Get all AUTOMATIC positions in frame fr which are available
0115         posAU = tracksR(tracksR(:,2)==fr & tracksR(:,5)==1,:);
0116         n_pos = size(posAU,1);
0117         % Build the distance vector to find the shortest distance
0118         dis_V = Inf(n_pos,1);
0119         for pos = 1: n_pos
0120             dis_V(pos) = sqrt(  (posGT(3) - posAU(pos,3))^2 +...
0121                 (posGT(4) - posAU(pos,4))^2) ;
0122         end
0123         % Choose the shortest distance
0124         [dis_min idx] = min(dis_V);
0125         if dis_min < P.tEval_Distance
0126             % Collect AU track future position of matching position
0127             trackAU_id = posAU(idx,1);
0128             trackAU = tracksR(tracksR(:,1)==trackAU_id & tracksR(:,2) >= fr,:);
0129             frameAU_end = max(trackAU(:,2));
0130             % Trace along the AUTOMATIC track
0131             iRMSE = 0;
0132             while (iRMSE < P.tEval_Distance) && (fr <= frameAU_end) && (fr <=frameGT_end)
0133                 posGT = trackGT(trackGT(:,2)==fr,:);
0134                 posAU = trackAU(trackAU(:,2)==fr,:);
0135                 
0136                 if ~isempty(posAU)
0137                     iRMSE = sqrt( (posGT(3) - posAU(3))^2 + (posGT(4) - posAU(4))^2 );
0138                     if (iRMSE < P.tEval_Distance)
0139                         RMSE = [RMSE; posGT(1) posAU(1) fr iRMSE];
0140                     end
0141                     % the AU position is no longer available
0142                     tracksR(tracksR(:,1)==posAU(1) & tracksR(:,2)==fr, 5)=0;
0143                 end
0144                 fr = fr + 1;
0145             end
0146         else
0147             fr = fr + 1;
0148         end
0149     end
0150     
0151 end
0152 
0153 RMSEx = [];
0154 if (~isempty(RMSE))
0155     ids= unique(RMSE(:,1));
0156     for id = 1: length(ids)
0157         tr = ids(id);
0158         track = RMSE(RMSE(:,1)== tr,:);
0159         
0160         idx = unique(track(:,2));
0161         % Get the maximum correspondence among the automatic track
0162         % this is to ensure 1:1 correspondence
0163         maxlength = 0;
0164         track_AUTO = [];
0165         for id2 = 1: length(idx)
0166             temp = track(track(:,2) == idx(id2),:);
0167             if length(temp) > maxlength
0168                 maxlength = length(temp);
0169                 track_AUTO = temp;
0170             end
0171         end
0172         
0173         lengthTR = size(track_AUTO,1);
0174         if lengthTR > 2
0175             lengthsTR = [lengthsTR; lengthTR ];
0176         end
0177         RMSEx  = [RMSEx; track_AUTO];
0178     end
0179     
0180     % PTP
0181     PTP = size(RMSEx,1) / size(tracksGT,1) ;
0182     % TL
0183     TL = mean(lengthsTR);
0184     % RMSE
0185     mRMSE = mean(RMSEx(:,end));
0186     
0187     
0188     %% Calculate the performance factors for COLLIDING CELLS
0189     
0190     tracks_C = [];
0191     RMSEx_C = [];
0192     lengthsTR_C = [];
0193     tracksGT_C = [];
0194     tracksGT_C2 = [];
0195     
0196     RMSEx_NC = RMSEx;
0197     tracks_NC = tracksR;
0198     lengthsTR_NC = [];
0199     tracksGT_NC = tracksGT;
0200     tracksGT_NC2 = tracksGT;
0201     
0202     collidingCells = classifyCollision(tracksGT,P);
0203     fr_Cx = [collidingCells.Pairs(:,1:2); [collidingCells.Pairs(:,1) collidingCells.Pairs(:,3)] ];
0204     
0205     for id = 1: length(collidingCells.Ids)
0206         
0207         fr_C = unique(fr_Cx(fr_Cx(:,2)== collidingCells.Ids(id),1));
0208         fr_C = unique([fr_C - 2 ; fr_C - 1 ; fr_C ; fr_C + 1 ; fr_C + 2] );
0209         fr_C = unique([fr_C - 1 ; fr_C ; fr_C + 1] );
0210         
0211         for id_fr = 1: length(fr_C)
0212             fr = fr_C(id_fr);
0213             rmse_C = RMSEx(RMSEx(:,1) == collidingCells.Ids(id) & RMSEx(:,3) == fr,:);
0214             RMSEx_C = [RMSEx_C; rmse_C];
0215             RMSEx_NC(RMSEx_NC(:,1) == collidingCells.Ids(id) & RMSEx_NC(:,3) == fr,:)=[];
0216             %             RMSEx_NC(RMSEx_NC(:,1) == collidingCells.Ids(id),:)=[];
0217             
0218             trackGT_C = tracksGT(tracksGT(:,1) == collidingCells.Ids(id) & tracksGT(:,2) == fr,:);
0219             tracksGT_C = [tracksGT_C; trackGT_C];
0220             tracksGT_NC(tracksGT_NC(:,1) == collidingCells.Ids(id) & tracksGT_NC(:,2) == fr,:)=[];
0221             %             tracksGT_NC(tracksGT_NC(:,1) == collidingCells.Ids(id),:)=[];
0222         end
0223         
0224     end
0225     
0226     idx = unique(RMSEx_C(:,2));
0227     for id  = 1: length(idx)
0228         track_C = tracksR(tracksR(:,1) == idx(id),:);
0229         tracks_C = [tracks_C; track_C];
0230         tracks_NC(tracks_NC(:,1) == idx(id),:) = [];
0231         
0232         lengthTR_C = size(track_C,1);
0233         lengthsTR_C = [lengthsTR_C; lengthTR_C];
0234     end
0235     
0236     fr_C2x = unique(collidingCells.Pairs(:,1));
0237     for id_fr = 1: length(fr_C2x)
0238         fr = fr_C2x(id_fr);
0239         trackGT_C2 = tracksGT(tracksGT(:,2) == fr,:);
0240         tracksGT_C2 = [tracksGT_C2; trackGT_C2];
0241         tracksGT_NC2(tracksGT_NC2(:,2) == fr,:)=[];
0242     end
0243     
0244     % PTP
0245     PTP_C = size(RMSEx_C,1) / size(tracksGT_C,1) ;
0246     % TL
0247     TL_C = mean(lengthsTR_C);
0248     % RMSE
0249     mRMSE_C = mean(RMSEx_C(:,end));
0250     
0251     
0252     %% Calculate the performance factors for NON-COLLIDING CELL
0253     idx = unique(RMSEx_NC(:,2));
0254     for id  = 1: length(idx)
0255         track_NC = tracksR(tracksR(:,1) == idx(id),:);
0256         lengthTR_NC = size(track_NC,1);
0257         lengthsTR_NC = [lengthsTR_NC; lengthTR_NC];
0258     end
0259     
0260     
0261     % PTP
0262     PTP_NC = size(RMSEx_NC,1) / size(tracksGT_NC,1) ;
0263     % TL
0264     TL_NC = mean(lengthsTR_NC);
0265     % RMSE
0266     mRMSE_NC = mean(RMSEx_NC(:,end));
0267             
0268     %% Velocity distribution
0269     V = get_velocityDistribution(tracksR);
0270     V_C = get_velocityDistribution(tracks_C);
0271     V_NC = get_velocityDistribution(tracks_NC);
0272     
0273     %% Run detection evaluation
0274     [d_specs] = NKTdetectionEvaluation(iFolder,framesR,tracksGT,0,P);
0275     [d_specs_C] = NKTdetectionEvaluation(iFolder,framesR,tracksGT_C2,0,P);
0276     [d_specs_NC] = NKTdetectionEvaluation(iFolder,framesR,tracksGT_NC2,0,P);
0277     
0278     %% Construct S
0279     S.Dataset = iFolder;
0280     S.TPR = mean(d_specs(:,2));
0281     S.PPV = mean(d_specs(:,3));
0282     S.FDR = mean(d_specs(:,4));
0283     S.TP = mean(d_specs(:,5));
0284     S.FP = mean(d_specs(:,6));
0285     S.FN = mean(d_specs(:,7));
0286     
0287     S.RMSEx = RMSEx;
0288     S.tracks = tracksR;
0289     S.tracksGT = tracksGT;
0290     S.frames = framesR;
0291     S.IDs = unique(tracksR(:,1))';
0292     S.nTR = length(unique(tracksR(:,1)));
0293     S.nFR = length(unique(RMSEx(:,3)));
0294     
0295     S.RMSE = mRMSE;
0296     S.PTP = PTP;
0297     S.TL = TL;
0298     S.VD = [mean(V(:,1)) std(V(:,1))];
0299     
0300     
0301     S.CC.TPR = mean(d_specs_C(:,2));
0302     S.CC.PPV = mean(d_specs_C(:,3));
0303     S.CC.FDR = mean(d_specs_C(:,4));
0304     S.CC.TP = mean(d_specs_C(:,5));
0305     S.CC.FP = mean(d_specs_C(:,6));
0306     S.CC.FN = mean(d_specs_C(:,7));
0307     
0308     S.CC.RMSEx = RMSEx_C;
0309     S.CC.Cases = collidingCells.Pairs;
0310     if ~isempty(tracks_C)
0311         S.CC.tracks = tracks_C;
0312         S.CC.tracksGT = tracksGT_C;
0313         S.CC.IDs = unique(tracks_C(:,1))';
0314         S.CC.nTR = length(unique(tracks_C(:,1)));
0315         S.CC.nFR = length(unique(RMSEx_C(:,3)));
0316         
0317         S.CC.RMSE = mRMSE_C;
0318         S.CC.PTP = PTP_C;
0319         S.CC.TL = TL_C;
0320         S.CC.VD = [mean(V_C(:,1)) std(V_C(:,1))];
0321     else
0322         S.CC.tracks = [];
0323         S.CC.tracksGT = tracksGT_CC;
0324         S.CC.IDs = -1;
0325         S.CC.nTR = 0;
0326         S.CC.nFR = 0;
0327         
0328         S.CC.RMSE = 0;
0329         S.CC.PTP = 0;
0330         S.CC.TL = 0;
0331         S.CC.VD = [0 0];
0332     end
0333     
0334     
0335     S.NC.RMSEx = RMSEx_NC;
0336     
0337     S.NC.TP = mean(d_specs_NC(:,5));
0338     S.NC.FP = mean(d_specs_NC(:,6));
0339     S.NC.FN = mean(d_specs_NC(:,7));
0340     
0341     S.NC.TPR = mean(d_specs_NC(:,2));
0342     S.NC.PPV = mean(d_specs_NC(:,3));
0343     S.NC.FDR = mean(d_specs_NC(:,4));
0344     
0345     S.NC.tracks = tracks_NC;
0346     S.NC.tracksGT = tracksGT_NC;
0347     S.NC.IDs = unique(tracks_NC(:,1))';
0348     S.NC.nTR = length(unique(tracks_NC(:,1)));
0349     S.NC.nFR = length(unique(RMSEx_NC(:,3)));
0350     
0351     S.NC.RMSE = mRMSE_NC;
0352     S.NC.PTP = PTP_NC;
0353     S.NC.TL = TL_NC;
0354     S.NC.VD = [mean(V_NC(:,1)) std(V_NC(:,1))];
0355     
0356     S.frames = framesR;
0357     S.Counter = Counter;
0358 end
0359 %     save _performance.mat S

Generated on Thu 17-Mar-2011 14:45:51 by m2html © 2005