Skip to content

Commit

Permalink
MatlabAPI/CocoEval.m: adding keypoint evaluation code!!! (needs more …
Browse files Browse the repository at this point in the history 
…testing)
  • Loading branch information
@pdollar
pdollar committed Sep 6, 2016
1 parent c95a22b commit af29c4f
Show file tree
Hide file tree
Showing 4 changed files with 122 additions and 19 deletions.
76 changes: 65 additions & 11 deletions MatlabAPI/CocoEval.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
% recThrs - [0:.01:1] R=101 recall thresholds for evaluation
% areaRng - [...] A=4 object area ranges for evaluation
% maxDets - [1 10 100] M=3 thresholds on max detections per image
% iouType - ['segm'] set iouType to 'segm' or 'bbox'
% iouType - ['segm'] set iouType to 'segm', 'bbox' or 'keypoints'
% useCats - [1] if true use category labels for evaluation
% Note: iouType replaced the now DEPRECATED useSegm parameter.
% Note: if useCats=0 category labels are ignored as in proposal scoring.
Expand Down Expand Up @@ -83,17 +83,25 @@
end

methods
function ev = CocoEval( cocoGt, cocoDt )
function ev = CocoEval( cocoGt, cocoDt, iouType )
% Initialize CocoEval using coco APIs for gt and dt.
if(nargin>0), ev.cocoGt = cocoGt; end
if(nargin>1), ev.cocoDt = cocoDt; end
if(nargin>0), ev.params.imgIds = sort(ev.cocoGt.getImgIds()); end
if(nargin>0), ev.params.catIds = sort(ev.cocoGt.getCatIds()); end
if(nargin<3), iouType='segm'; end
ev.params.iouThrs = .5:.05:.95;
ev.params.recThrs = 0:.01:1;
ev.params.areaRng = [0 1e5; 0 32; 32 96; 96 1e5].^2;
ev.params.maxDets = [1 10 100];
ev.params.iouType = 'segm';
if( any(strcmp(iouType,{'bbox','segm'})) )
ev.params.areaRng = [0 1e5; 0 32; 32 96; 96 1e5].^2;
ev.params.maxDets = [1 10 100];
elseif( strcmp(iouType,'keypoints') )
ev.params.areaRng = [0 1e5; 32 96; 96 1e5].^2;
ev.params.maxDets = 20;
else
error('unknown iouType: %s',iouType);
end
ev.params.iouType = iouType;
ev.params.useCats = 1;
end

Expand Down Expand Up @@ -122,6 +130,9 @@ function evaluate( ev )
f='bbox'; if(isempty(dt)), [dt(:).(f)]=deal(); end
if(~isfield(dt,f)), s=MaskApi.toBbox([dt.segmentation]);
for d=1:nDt(i), dt(d).(f)=s(d,:); end; end
elseif( strcmp(p.iouType,'keypoints') )
gtIg=[gt.ignore]|[gt.num_keypoints]==0;
for g=1:nGt(i), gt(g).ignore=gtIg(g); end
else
error('unknown iouType: %s',p.iouType);
end
Expand Down Expand Up @@ -186,10 +197,17 @@ function accumulate( ev )
function summarize( ev )
% Compute and display summary metrics for evaluation results.
if(isempty(ev.eval)), error('Please run accumulate() first'); end
k=100; M={{1,':','all',k},{1,.50,'all',k}, {1,.75,'all',k},...
{1,':','small',k}, {1,':','medium',k}, {1,':','large',k},...
{0,':','all',1}, {0,':','all',10}, {0,':','all',k},...
{0,':','small',k}, {0,':','medium',k}, {0,':','large',k}};
if( any(strcmp(ev.params.iouType,{'bbox','segm'})) )
k=100; M={{1,':','all',k},{1,.50,'all',k}, {1,.75,'all',k},...
{1,':','small',k}, {1,':','medium',k}, {1,':','large',k},...
{0,':','all',1}, {0,':','all',10}, {0,':','all',k},...
{0,':','small',k}, {0,':','medium',k}, {0,':','large',k}};
elseif( strcmp(ev.params.iouType,'keypoints') )
k=20; M={{1,':','all',k},{1,.50,'all',k}, {1,.75,'all',k},...
{1,':','medium',k}, {1,':','large',k},...
{0,':','all',k},{0,.50,'all',k}, {0,.75,'all',k},...
{0,':','medium',k}, {0,':','large',k}};
end
k=length(M); ev.stats=zeros(1,k);
for s=1:k, ev.stats(s)=summarize1(M{s}{:}); end

Expand Down Expand Up @@ -349,10 +367,11 @@ function makeplot( rs, ps, outDir, nm )
if(D>p.maxDets), D=p.maxDets; dt=dt(1:D); end
% compute iou between each dt and gt region
iscrowd = uint8([gt.iscrowd]);
t=find(strcmp(p.iouType,{'segm','bbox'}));
t=find(strcmp(p.iouType,{'segm','bbox','keypoints'}));
if(t==1), g=[gt.segmentation]; elseif(t==2), g=cat(1,gt.bbox); end
if(t==1), d=[dt.segmentation]; elseif(t==2), d=cat(1,dt.bbox); end
ious=MaskApi.iou(d,g,iscrowd);
if(t<=2), ious=MaskApi.iou(d,g,iscrowd); else
ious=CocoEval.oks(gt,dt); end
% attempt to match each (sorted) dt to each (sorted) gt
gtm=zeros(T,G); gtIds=[gt.id]; gtIg=[gt.ignore];
dtm=zeros(T,D); dtIds=[dt.id]; dtIg=zeros(T,D);
Expand Down Expand Up @@ -380,5 +399,40 @@ function makeplot( rs, ps, outDir, nm )
dtImgIds=ones(1,D)*p.imgIds; gtImgIds=ones(1,G)*p.imgIds;
e = {dtIds,gtIds,dtImgIds,gtImgIds,dtm,gtm,[dt.score],dtIg,gtIg};
end

function o = oks( gt, dt )
% Compute Object Keypoint Similarity (OKS) between objects.
G=length(gt); D=length(dt); o=zeros(D,G); if(~D||~G), return; end
% sigmas hard-coded for person class, will need params eventually
sigmas=[.26 .25 .25 .35 .35 .79 .79 .72 .72 .62 ...
.62 1.07 1.07 .87 .87 .89 .89]/10;
vars=(sigmas*2).^2; k=length(sigmas); m=k*3; bb=cat(1,gt.bbox);
% create bounds for ignore regions (double the gt bbox)
x0=bb(:,1)-bb(:,3); x1=bb(:,1)+bb(:,3)*2;
y0=bb(:,2)-bb(:,4); y1=bb(:,2)+bb(:,4)*2;
% extract keypoint locations and visibility flags
gKp=cat(1,gt.keypoints); assert(size(gKp,2)==m);
dKp=cat(1,dt.keypoints); assert(size(dKp,2)==m);
xg=gKp(:,1:3:m); yg=gKp(:,2:3:m); vg=gKp(:,3:3:m);
xd=dKp(:,1:3:m); yd=dKp(:,2:3:m);
% compute oks between each detection and ground truth object
for d=1:D
for g=1:G
v=vg(g,:); x=xd(d,:); y=yd(d,:); k1=nnz(v);
if( k1>0 )
% measure the per-keypoint distance if keypoints visible
dx=x-xg(g,:); dy=y-yg(g,:);
else
% measure minimum distance to keypoints in (x0,y0) & (x1,y1)
dx=max(0,x0(g,:)-x)+max(0,x-x1(g,:));
dy=max(0,y0(g,:)-y)+max(0,y-y1(g,:));
end
% use the distances to compute the oks
e=(dx.^2+dy.^2)./vars/gt(g).area/2;
if(k1>0), e=e(v>0); else k1=k; end
o(d,g)=sum(exp(-e))/k1;
end
end
end
end
end
16 changes: 8 additions & 8 deletions MatlabAPI/evalDemo.m
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,18 @@
%% Demo demonstrating the algorithm result formats for COCO

%% select results type for demo (either bbox or segm)
type = {'segm','bbox'}; type = type{1}; % specify type here
type = {'segm','bbox','keypoints'}; type = type{1}; % specify type here
fprintf('Running demo for *%s* results.\n\n',type);

%% initialize COCO ground truth api
dataDir='../'; dataType='val2014';
annFile=sprintf('%s/annotations/instances_%s.json',dataDir,dataType);
if(~exist('cocoGt','var')), cocoGt=CocoApi(annFile); end
dataDir='../'; prefix='instances'; dataType='val2014';
if(strcmp(type,'keypoints')), prefix='person_keypoints'; end
annFile=sprintf('%s/annotations/%s_%s.json',dataDir,prefix,dataType);
cocoGt=CocoApi(annFile);

%% initialize COCO detections api
resFile='%s/results/instances_%s_fake%s100_results.json';
resFile=sprintf(resFile,dataDir,dataType,type);
resFile='%s/results/%s_%s_fake%s100_results.json';
resFile=sprintf(resFile,dataDir,prefix,dataType,type);
cocoDt=cocoGt.loadRes(resFile);

%% visialuze gt and dt side by side
Expand All @@ -33,9 +34,8 @@
if(0), f=fopen(resFile,'w'); fwrite(f,gason(res)); fclose(f); end

%% run COCO evaluation code (see CocoEval.m)
cocoEval=CocoEval(cocoGt,cocoDt);
cocoEval=CocoEval(cocoGt,cocoDt,type);
cocoEval.params.imgIds=imgIds;
cocoEval.params.iouType=type;
cocoEval.evaluate();
cocoEval.accumulate();
cocoEval.summarize();
Expand Down
1 change: 1 addition & 0 deletions results/person_keypoints_val2014_fakekeypoints100_results.json
View file

Large diffs are not rendered by default.

48 changes: 48 additions & 0 deletions results/val2014_fake_eval_res.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
------------------------------------------------------------------------------
type=segm
Running per image evaluation... DONE (t=0.45s).
Accumulating evaluation results... DONE (t=0.08s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.320
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.562
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.299
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.387
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.310
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.327
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.268
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.415
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.417
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.469
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.377
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.381

------------------------------------------------------------------------------
type=bbox
Running per image evaluation... DONE (t=0.34s).
Accumulating evaluation results... DONE (t=0.08s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.505
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.697
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.573
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.586
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.519
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.501
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.387
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.594
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.595
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.640
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.566
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.564

------------------------------------------------------------------------------
type=keypoints
Running per image evaluation... DONE (t=0.06s).
Accumulating evaluation results... DONE (t=0.00s).
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets= 20 ] = 0.372
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets= 20 ] = 0.636
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets= 20 ] = 0.348
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets= 20 ] = 0.384
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets= 20 ] = 0.386
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 20 ] = 0.514
Average Recall (AR) @[ IoU=0.50 | area= all | maxDets= 20 ] = 0.734
Average Recall (AR) @[ IoU=0.75 | area= all | maxDets= 20 ] = 0.504
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets= 20 ] = 0.508
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets= 20 ] = 0.522

0 comments on commit af29c4f

Please sign in to comment.