1 #include "calibmodel.h"
4 char CalibModel::progress_string
[2048];
6 CalibModel::CalibModel()
11 win
= "The Artvertiser 0.4";
12 train_working_image
= 0;
13 interactive_train_running
= false;
14 interactive_train_should_stop
= false;
15 debounce_green
= false;
16 debounce_redblue
= false;
17 strcpy(progress_string
, "");
19 cvInitFont(&train_font
, CV_FONT_HERSHEY_SIMPLEX
, .5, .5, 0, 0, CV_AA
);
22 CalibModel::~CalibModel()
24 if (image
) cvReleaseImage(&image
);
25 if ( train_working_image
)
27 cvReleaseImage( &train_working_image
);
28 cvReleaseImage( &train_shot
);
32 void CalibModel::useModelFile(const char* file
)
37 CalibModel
*objectPtr
=0;
39 void CalibModel::onMouse(int event
, int x
, int y
, int flags
)
42 if ( state
== ARTVERT_CORNERS
)
43 ptr
= artvert_corners
;
46 if (event
== CV_EVENT_LBUTTONDOWN
)
48 // try to grab something
50 for (int i
=0; i
<4; i
++)
54 if (sqrt((double)(dx
*dx
+dy
*dy
)) <10) {
67 if (event
== CV_EVENT_LBUTTONUP
)
73 bool CalibModel::buildCached(int nbcam
, CvCapture
*capture
,
74 bool cache
, planar_object_recognizer
&detector
,
75 bool running_on_binoculars
)
80 printf("buildCached about to grab a lock\n");
85 detector
.ransac_dist_threshold
= 5;
86 detector
.max_ransac_iterations
= 800;
87 //detector.ransac_stop_support = 50;
88 //detector.non_linear_refine_threshold = 15.0f;
89 //detector.point_detector_tau = 10;
91 // A lower threshold will allow detection in harder conditions, but
92 // might lead to false positives.
93 detector
.match_score_threshold
=.03f
;
95 detector
.min_view_rate
=.1;
96 detector
.views_number
= 1000;
98 //detector.min_view_rate = .2;
100 static const int MAX_MODEL_KEYPOINTS
= 500; // maximum number of keypoints on the model
101 static const int PATCH_SIZE
= 32; // patch size in pixels
102 static const int YAPE_RADIUS
= 5; // yape radius
103 static const int NUM_TREES
= 12; // num classifier trees
104 static const int NUM_GAUSSIAN_LEVELS
= 3; // num gaussian levels
105 /*static const int MAX_MODEL_KEYPOINTS = 500; // maximum number of keypoints on the model
106 static const int PATCH_SIZE = 32; // patch size in pixels
107 static const int YAPE_RADIUS = 5; // yape radius
108 static const int NUM_TREES = 12; // num classifier trees
109 static const int NUM_GAUSSIAN_LEVELS = 3; // num gaussian levels*/
112 // Should we train or load the classifier ?
114 printf("model.buildCached() trying to load from %s...\n", modelfile
);
115 if(cache
&& detector
.build_with_cache(
116 string(modelfile
), // mode image file name
117 /*500, // maximum number of keypoints on the model
118 //200, // maximum number of keypoints on the model
119 32, // patch size in pixels
120 5, // yape radius. Use 3,5 or 7.
121 //3, // yape radius. Use 3,5 or 7.
122 12, // number of trees for the classifier. Somewhere between 12-50
123 //20, // number of trees for the classifier. Somewhere between 12-50
124 3*/ // number of levels in the gaussian pyramid
126 MAX_MODEL_KEYPOINTS
, // max keypoints
127 PATCH_SIZE
, // patch size
128 YAPE_RADIUS
, // yape radius. Use 3,5 or 7.
129 NUM_TREES
, // number of trees for the classifier. Somewhere between 12-50
130 NUM_GAUSSIAN_LEVELS
// number of levels in the gaussian pyramid
134 // loading worked. Remember the region of interest.
135 corners
[0].x
= detector
.new_images_generator
.u_corner1
;
136 corners
[0].y
= detector
.new_images_generator
.v_corner1
;
137 corners
[1].x
= detector
.new_images_generator
.u_corner2
;
138 corners
[1].y
= detector
.new_images_generator
.v_corner2
;
139 corners
[2].x
= detector
.new_images_generator
.u_corner3
;
140 corners
[2].y
= detector
.new_images_generator
.v_corner3
;
141 corners
[3].x
= detector
.new_images_generator
.u_corner4
;
142 corners
[3].y
= detector
.new_images_generator
.v_corner4
;
144 MultiThreadCapture
* mtc
= MultiThreadCaptureManager::getInstance()->getCaptureForCam(capture
);
146 IplImage* init_image = NULL;
150 got = mtc->getCopyOfLastFrame( &init_image );
153 !usleep( 10*1000 ) &&
155 if ( init_image == NULL )
157 printf("getCopyOfLastFrame timed out: capture failed\n");
163 cvReleaseImage( &image
);
164 image
= cvLoadImage(modelfile
, mtc
->getNumChannelsRaw()==3 );
165 image_width
= image
->width
;
166 image_height
= image
->height
;
167 //cvReleaseImage(&init_image);
169 /*printf("dumping loaded cache:\n");
177 cvReleaseImage( &image
);
179 // ask the user the take a shot of the model
181 if( running_on_binoculars
)
182 result
= interactiveTrainBinoculars();
184 result
= interactiveSetup(capture
);
187 printf("interactiveSetup failed\n");
194 printf("interactiveSetup succeeded\n");
197 image_width
= image
->width
;
198 image_height
= image
->height
;
200 // train the classifier to detect this model
201 //if (!detector.build(image, 500, 32, 3, 12, 3,0, 0))
202 //if (!detector.build(image, 200, 32, 5, 20, 3,0, 0))
203 int working_roi
[8] = { corners
[0].x
, corners
[0].y
,
204 corners
[1].x
, corners
[1].y
,
205 corners
[2].x
, corners
[2].y
,
206 corners
[3].x
, corners
[3].y
208 //LEARNPROGRESSION progress;
209 //LEARNPROGRESSION progress = 0;
211 printf("about to call detector.build...\n");
212 learn_running
= true;
213 strcpy(progress_string
, "preparing..");
214 if (!detector
.build(image
,
215 MAX_MODEL_KEYPOINTS
, // max keypoints
216 PATCH_SIZE
, // patch size
217 YAPE_RADIUS
, // yape radius. Use 3,5 or 7.
218 NUM_TREES
, // number of trees for the classifier. Somewhere between 12-50
219 NUM_GAUSSIAN_LEVELS
, // number of levels in the gaussian pyramid
220 &learnProgressionFunc
,
224 learn_running
= false;
225 printf("build based on interactiveSetup failed\n");
231 printf("detector.build succeeded\n");
232 strcpy( progress_string
, "saving..\n");
234 if (!cvSaveImage(modelfile
, image
))
236 printf("saving input image failed\n");
237 learn_running
= false;
243 // and the region of interest (ROI)
244 string roifn
= string(modelfile
) + ".roi";
245 ofstream
roif(roifn
.c_str());
248 learn_running
= false;
251 printf("saving .roi file failed\n");
254 for (int i
=0;i
<4; i
++)
255 roif
<< corners
[i
].x
<< " " << corners
[i
].y
<< "\n";
258 // and the artvert corners
259 roifn
= string(modelfile
) + ".artvertroi";
260 ofstream
artvert_roif(roifn
.c_str());
261 if (!artvert_roif
.good())
263 learn_running
= false;
266 printf("saving .artvertroi file failed\n");
269 for (int i
=0;i
<4; i
++)
270 artvert_roif
<< artvert_corners
[i
].x
<< " " << artvert_corners
[i
].y
<< "\n";
271 artvert_roif
.close();
273 // and the trained classifier
274 string classifier_directory
= string(modelfile
)+".classifier";
275 detector
.save(classifier_directory
);
277 string stable_points_filename
= string(modelfile
)+"_stable_points.bmp";
278 printf("saving stable points to %s\n", stable_points_filename
.c_str());
279 detector
.save_image_of_model_points(PATCH_SIZE
, stable_points_filename
.c_str() );
281 const char* initial_points_filename
= "initial_model_points.bmp";
282 string initial_points_new_filename
= string(modelfile
)+"_initial_points.bmp";
283 printf("renaming %s to %s\n", initial_points_filename
, initial_points_new_filename
.c_str() );
284 rename(initial_points_filename
, initial_points_new_filename
.c_str() );
286 /*printf("dumping trained cache:\n");
289 learn_running
= false;
293 assert( detector
.isReady() );
296 for (int i
=0; i
<4; i
++)
298 cn
[i
][0] = corners
[i
].x
;
299 cn
[i
][1] = corners
[i
].y
;
300 cout
<< corners
[i
].x
<< " " << corners
[i
].y
<< endl
;
303 // prepare the light calibration reference
304 return map
.init(nbcam
, image
, cn
, 8, 6);
307 static void putText(IplImage
*im
, const char *text
, CvPoint p
, CvFont
*f1
)
309 cvPutText(im
,text
,p
,f1
, cvScalar(0,255, 255));
313 IplImage
*myRetrieveFrame(CvCapture
*capture
)
315 #ifdef USE_MULTITHREADCAPTURE
316 assert(false && "don't call myRetrieveFrame when USE_MULTITHREADCAPTURE");
319 static IplImage
*s
=0;
320 IplImage
*frame
=cvRetrieveFrame(capture
);
321 if (frame
== 0) return 0;
323 if (frame
->nChannels
==1) {
324 printf(" PERFORMANCE WARNING: myRetrieveFrame converting colour\n");
325 if (!s
) s
=cvCreateImage(cvSize(frame
->width
,frame
->height
), IPL_DEPTH_8U
, 3);
326 cvCvtColor(frame
,s
,CV_GRAY2BGR
);
330 printf(" PERFORMANCE WARNING: myRetrieveFrame flipping\n");
331 if (!s
) s
=cvCreateImage(cvSize(frame
->width
,frame
->height
), IPL_DEPTH_8U
, 3);
339 IplImage
*myQueryFrame(CvCapture
*capture
)
341 #ifdef USE_MULTITHREADCAPTURE
342 assert(false && "don't call myQueryFrame when USE_MULTITHREADCAPTURE");
345 cvGrabFrame(capture
);
346 return myRetrieveFrame(capture
);
350 bool CalibModel::interactiveTrainBinoculars()
354 train_should_proceed
= false;
355 // we want the interactive training to run
356 interactive_train_running
= true;
357 interactive_train_should_stop
= false;
358 debounce_green
= true;
359 debounce_redblue
= true;
364 while ( interactive_train_running
&& !interactive_train_should_stop
&& timeout
>0 )
366 printf("waiting for interactiveTrainBinoculars to complete .. %i\n", timeout
);
371 return train_should_proceed
;
374 void CalibModel::interactiveTrainBinocularsUpdate( IplImage
* frame
,
379 if ( !interactive_train_running
)
382 if ( interactive_train_should_stop
)
384 interactive_train_running
= false;
389 if ( train_working_image
== 0 )
391 train_working_image
= cvCreateImage( cvGetSize( frame
), frame
->depth
, frame
->nChannels
);
392 train_shot
= cvCreateImage( cvGetSize( frame
), frame
->depth
, frame
->nChannels
);
396 bool R__
= ( button_red
&&!button_green
&&!button_blue
);
397 bool _G_
= (!button_red
&& button_green
&&!button_blue
);
398 bool __B
= (!button_red
&&!button_green
&& button_blue
);
399 bool RG_
= ( button_red
&& button_green
&&!button_blue
);
400 bool _GB
= (!button_red
&& button_green
&& button_blue
);
401 bool R_B
= ( button_red
&&!button_green
&& button_blue
);
402 bool RGB
= ( button_red
&& button_green
&& button_blue
);
406 // don't allow just green if we're trying to debounce
408 debounce_green
= false;
409 else if ( debounce_green
&& _G_
)
412 // don't allow red+blue if we're trying to debounce
413 if ( !button_red
|| !button_blue
)
414 debounce_redblue
= false;
415 else if ( debounce_redblue
&& R_B
)
430 cvCopy( frame
, train_shot
);
435 corners
[1].x
= frame
->width
-d
;
437 corners
[2].x
= frame
->width
-d
;
438 corners
[2].y
= frame
->height
-d
;
440 corners
[3].y
= frame
->height
-d
;
443 x
= corners
[corner_index
].x
;
444 y
= corners
[corner_index
].y
;
445 // debounce the green button
446 debounce_green
= true;
451 interactive_train_running
= false;
452 debounce_redblue
= true;
456 cvCopy( frame
, train_working_image
);
457 putText(train_working_image
, modelfile
, cvPoint(3,20), &train_font
);
458 putText(train_working_image
,"Please take a frontal view", cvPoint(3,40), &train_font
);
459 putText(train_working_image
,"of a textured planar surface", cvPoint(3,60), &train_font
);
460 putText(train_working_image
,"and press green", cvPoint(3,80), &train_font
);
461 putText(train_working_image
,"Press red+blue to abort", cvPoint(3,100), &train_font
);
465 cvCopy( train_shot
, train_working_image
);
466 putText(train_working_image
, modelfile
, cvPoint(3,20), &train_font
);
467 putText(train_working_image
, "Move green corners to match the", cvPoint(3,40), &train_font
);
468 putText(train_working_image
, "calibration target", cvPoint(3,60), &train_font
);
469 putText(train_working_image
, "Press red+blue to restart", cvPoint(3,80), &train_font
);
470 putText(train_working_image
, "Press green when ready", cvPoint(3,100), &train_font
);
481 // update corner if necessary
483 corners
[corner_index
].x
= x
;
485 corners
[corner_index
].y
= y
;
486 // continue checking buttons
490 corners
[corner_index
].x
= x
;
491 corners
[corner_index
].y
= y
;
493 if ( corner_index
> 3 )
497 for ( int i
=0; i
<4; i
++ )
498 artvert_corners
[i
] = corners
[i
];
499 state
= ARTVERT_CORNERS
;
503 x
= artvert_corners
[corner_index
].x
;
504 y
= artvert_corners
[corner_index
].y
;
508 x
= corners
[corner_index
].x
;
509 y
= corners
[corner_index
].y
;
512 debounce_green
= true;
518 if ( corner_index
< 0 )
522 x
= corners
[corner_index
].x
;
523 y
= corners
[corner_index
].y
;
525 debounce_redblue
= true;
529 cvPolyLine(train_working_image
, &ptr
, &four
, 1, 1,
531 cvCircle( train_working_image
, corners
[corner_index
], 10, cvScalar(0,255,0));
535 case ARTVERT_CORNERS
:
536 cvCopy( train_shot
, train_working_image
);
537 putText(train_working_image
, modelfile
, cvPoint(3,20), &train_font
);
538 putText(train_working_image
, "Move red corners to match the", cvPoint(3,40), &train_font
);
539 putText(train_working_image
, "artvert target area;", cvPoint(3,60), &train_font
);
540 putText(train_working_image
, "Press red+blue to restart", cvPoint(3,80), &train_font
);
541 putText(train_working_image
, "Press green when ready", cvPoint(3,100), &train_font
);
552 // update corner if necessary
554 artvert_corners
[corner_index
].x
= x
;
556 artvert_corners
[corner_index
].y
= y
;
557 // continue checking buttons
561 artvert_corners
[corner_index
].x
= x
;
562 artvert_corners
[corner_index
].y
= y
;
564 if ( corner_index
> 3 )
568 cvReleaseImage( &image
);
569 image
= cvCreateImage( cvGetSize( train_shot
), train_shot
->depth
, train_shot
->nChannels
);
570 cvCopy( train_shot
, image
);
571 train_should_proceed
= true;
572 interactive_train_running
= false;
576 x
= artvert_corners
[corner_index
].x
;
577 y
= artvert_corners
[corner_index
].y
;
579 debounce_green
= true;
585 if ( corner_index
< 0 )
589 x
= artvert_corners
[corner_index
].x
;
590 y
= artvert_corners
[corner_index
].y
;
592 debounce_redblue
= true;
597 cvPolyLine(train_working_image
, &ptr
, &four
, 1, 1,
599 ptr
= artvert_corners
;
600 cvPolyLine(train_working_image
, &ptr
, &four
, 1, 1,
602 cvCircle( train_working_image
, artvert_corners
[corner_index
], 10, cvScalar(0,0,255));
607 train_texture
.setImage( train_working_image
);
611 void CalibModel::interactiveTrainBinocularsDraw()
613 if ( !interactive_train_running
)
616 IplTexture
* tex
= &train_texture
;
618 IplImage
*im
= tex
->getIm();
620 int h
= im
->height
-1;
622 glMatrixMode(GL_PROJECTION
);
624 glMatrixMode(GL_MODELVIEW
);
628 glDisable(GL_DEPTH_TEST
);
635 glTexCoord2f(tex
->u(0), tex
->v(0));
637 glTexCoord2f(tex
->u(w
), tex
->v(0));
639 glTexCoord2f(tex
->u(w
), tex
->v(h
));
641 glTexCoord2f(tex
->u(0), tex
->v(h
));
645 tex
->disableTexture();
650 bool CalibModel::interactiveSetup(CvCapture
*capture
)
656 CvFont font
, fontbold
;
658 cvInitFont(&font
, CV_FONT_HERSHEY_SIMPLEX
, .5, .5, 0, 0, CV_AA
);
660 cvNamedWindow(win
, CV_WINDOW_AUTOSIZE
);
664 cvSetMouseCallback(win
, onMouseStatic
, this);
668 IplImage
*frame_gray
, *frame
= NULL
;
670 MultiThreadCapture
* mtc
= MultiThreadCaptureManager::getInstance()->getCaptureForCam(capture
);
674 got
= mtc
->getLastDetectFrame( &frame_gray
, &frame
, ×tamp
, /*blocking*/true );
678 (timeout
-=100) > 0 );
681 printf("capture failed\n");
685 IplImage
*shot
=0, *text
=0;
689 bool accepted
=false;
690 bool artvert_accepted
= false;
694 char k
= cvWaitKey(10);
696 if (k
==27 || k
=='q') {
697 if (shot
) cvReleaseImage(&shot
);
698 if (text
) cvReleaseImage(&text
);
702 // clear text or grab the image to display
703 if (!pause
|| shot
==0) {
704 bool got
= mtc
->getLastDetectFrame( &frame_gray
, &frame
, NULL
,/*block until available*/true );
708 text
=cvCreateImage(cvGetSize(frame
), IPL_DEPTH_8U
, 3);
712 corners
[1].x
= frame
->width
-d
;
714 corners
[2].x
= frame
->width
-d
;
715 corners
[2].y
= frame
->height
-d
;
717 corners
[3].y
= frame
->height
-d
;
719 if (frame
->nChannels
==1)
720 cvCvtColor(frame
, text
, CV_GRAY2BGR
);
724 if (shot
->nChannels
==1)
725 cvCvtColor(shot
, text
, CV_GRAY2BGR
);
732 // display text / react to keyboard
737 if (shot
) cvCopy(frame
,shot
);
738 else shot
= cvCloneImage(frame
);
743 putText(text
, modelfile
, cvPoint(3,20), &font
);
744 putText(text
,"Please take a frontal view", cvPoint(3,40), &font
);
745 putText(text
,"of a textured planar surface", cvPoint(3,60), &font
);
746 putText(text
,"and press space", cvPoint(3,80), &font
);
750 putText(text
, modelfile
, cvPoint(3,20), &font
);
751 putText(text
, "Drag green corners to match the", cvPoint(3,40), &font
);
752 putText(text
, "calibration target", cvPoint(3,60), &font
);
753 putText(text
, "press 'r' to restart", cvPoint(3,80), &font
);
754 putText(text
, "press space when ready", cvPoint(3,100), &font
);
760 for ( int i
=0;i
<4; i
++ )
762 artvert_corners
[i
].x
= corners
[i
].x
;
763 artvert_corners
[i
].y
= corners
[i
].y
;
765 state
= ARTVERT_CORNERS
;
768 cvPolyLine(text
, &ptr
, &four
, 1, 1,
771 case ARTVERT_CORNERS
:
772 putText(text
, "Drag red corners to match the", cvPoint(3,20), &font
);
773 putText(text
, "artvert target area;", cvPoint(3,40), &font
);
774 putText(text
, "press 'r' to restart", cvPoint(3,60), &font
);
775 putText(text
, "press space when ready", cvPoint(3,80), &font
);
784 cvPolyLine(text
, &ptr
, &four
, 1, 1,
786 ptr
= artvert_corners
;
787 cvPolyLine(text
, &ptr
, &four
, 1, 1,
791 cvShowImage(win
, text
);
794 cvReleaseImage(&text
);
797 cvDestroyWindow( win
);
798 // make sure the destroy window succeeds
804 void CalibModel::onMouseStatic(int event
, int x
, int y
, int flags
, void* param
)
807 ((CalibModel
*)param
)->onMouse(event
,x
,y
,flags
);
809 objectPtr
->onMouse(event
,x
,y
,flags
);
811 cerr
<< "onMouseStatic(): null-pointer.\n";
815 void CalibModel::learnProgressionFunc( int phase
, int current
, int total
)
817 sprintf(progress_string
, "learning, phase %i/4: %4.1f%%", phase
+1, 100.0f
*float(current
)/total
);
820 printf("\rphase %i/4: %4i/%4i ", phase
+1, current
, total
);
823 /*for ( int i=0; i<4; i++ )
826 strcat(progress_string, "[----------------------------------------]\n" );
827 else if ( phase == i )
829 float finishedPct = float(current)/float(total);
830 int count = finishedPct*40;
831 strcat(progress_string, "[");
832 for ( int j=0; j<count; j++ )
834 strcat(progress_string, "-");
836 for ( int j=0; j<40-count; j++ )
838 strcat(progress_string, " ");
840 strcat(progress_string, "]\n");
843 strcat( progress_string, "[ ]\n");