#include "ros/ros.h" #include #include #include #include #include #include using namespace std; class Traite_image { public: const static int SENSITIVITY_VALUE = 20; const static int BLUR_Size = 9; const static int CLOSE_SIZE = 30; const static int ERODE_SIZE = 2; const static int NB_FRAME_DROP = 1; vector prevs; cv::Mat last_T; bool first = true; int resize_f = 1; int theObject[2] = {0,0}; ros::NodeHandle n; image_transport::ImageTransport it; image_transport::Publisher pub_img; image_transport::Publisher pub_thres; ros::Publisher pub_cmd; image_transport::Subscriber sub; Traite_image() : n("~"),it(n) { pub_img = it.advertise("/papillon/image_out", 1); pub_thres = it.advertise("/papillon/thres_out", 1); pub_cmd = n.advertise("/papillon/bbox", 1); sub = it.subscribe("/bebop/image_raw", 1, [this](const sensor_msgs::ImageConstPtr& img) -> void { this->on_image(img);},ros::VoidPtr(),image_transport::TransportHints("compressed")); } // This processes an image and publishes the result. void on_image(const sensor_msgs::ImageConstPtr& msg) { cv_bridge::CvImageConstPtr bridge_input; try { bridge_input = cv_bridge::toCvShare(msg,sensor_msgs::image_encodings::RGB8); } catch (cv::Exception& e) { std::ostringstream errstr; errstr << "cv_bridge exception caught: " << e.what(); return; } //cv::Mat& input = const_cast(bridge_input->image); const cv::Mat& input = bridge_input->image; cv::Mat next; resize(input, next, cv::Size(input.size().width/resize_f, input.size().height/resize_f)); cv::Mat output;// = input.clone(); // (input.rows, input.cols, CV_32FC2); if (first) { for (int i = 0; i < NB_FRAME_DROP; ++i) { prevs.push_back(next.clone()); } first = false; ROS_INFO("first done"); } cv::Mat next_stab; stabiliseImg(prevs.back(), next, next_stab); int crop_ratio = 6; float crop_x = next_stab.size().width/crop_ratio; float crop_y = next_stab.size().height/crop_ratio; float crop_w = next_stab.size().width*(1-2.0/crop_ratio); float crop_h = next_stab.size().height*(1-2.0/crop_ratio); cv::Rect myROI(crop_x, crop_y, crop_w, crop_h); cv::Mat next_stab_cropped = next_stab(myROI); cv::Mat prev_cropped = prevs.back()(myROI); cv::Mat closed_thres; searchForMovement(prev_cropped, next_stab_cropped, output, closed_thres); pub_img.publish(cv_bridge::CvImage(msg->header, "rgb8", output).toImageMsg()); pub_thres.publish(cv_bridge::CvImage(msg->header, "mono8", closed_thres).toImageMsg()); // bridge_input is handled by a smart-pointer. No explicit delete needed. //droneTracking(Rect(Point(0,0), output.size())); prevs.pop_back(); prevs.insert(prevs.begin(), next.clone()); } //int to string helper function string intToString(int number){ //this function has a number input and string output std::stringstream ss; ss << number; return ss.str(); } void stabiliseImg(cv::Mat prev, cv::Mat cur, cv::Mat &output){ cv::Mat cur_grey, prev_grey; cv::cvtColor(cur, cur_grey, cv::COLOR_BGR2GRAY); cv::cvtColor(prev, prev_grey, cv::COLOR_BGR2GRAY); // vector from prev to cur vector prev_corner, cur_corner; vector prev_corner2, cur_corner2; vector status; vector err; cv::goodFeaturesToTrack(prev_grey, prev_corner, 200, 0.01, 30); cv::calcOpticalFlowPyrLK(prev_grey, cur_grey, prev_corner, cur_corner, status, err); // weed out bad cv::Matches for(size_t i=0; i < status.size(); i++) { if(status[i]) { prev_corner2.push_back(prev_corner[i]); cur_corner2.push_back(cur_corner[i]); } } cv::Mat T = estimateRigidTransform(prev_corner2, cur_corner2, true); // false = rigid transform, no scaling/shearing if(T.data == NULL) { last_T.copyTo(T); } T.copyTo(last_T); cv::Mat cur2; cv::warpAffine(cur, cur2, T, cur.size(),cv::INTER_CUBIC|cv::WARP_INVERSE_MAP); cur2.copyTo(output); } void searchForMovement(cv::Mat prev, cv::Mat cur, cv::Mat &output, cv::Mat &out2){ cv::Mat cur_grey, prev_grey; cur.copyTo(output); cv::cvtColor(prev, prev_grey, cv::COLOR_BGR2GRAY); cv::cvtColor(cur, cur_grey, cv::COLOR_BGR2GRAY); cv::GaussianBlur(prev_grey, prev_grey, cv::Size(BLUR_Size,BLUR_Size), 3.0); cv::GaussianBlur(cur_grey, cur_grey, cv::Size(BLUR_Size,BLUR_Size), 3.0); //blur(prev_grey, prev_grey, cv::Size(BLUR_Size, BLUR_Size)); //blur(cur_grey, cur_grey, cv::Size(BLUR_Size, BLUR_Size)); // Subtract the 2 last frames and threshold them cv::Mat thres; cv::absdiff(prev_grey,cur_grey,thres); // threshold(thres, thres, SENSITIVITY_VALUE, 255, THRESH_BINARY); // // Blur to eliminate noise // blur(thres, thres, cv::Size(BLUR_Size, BLUR_Size)); cv::Mat element = getStructuringElement( cv::MORPH_ELLIPSE, cv::Size( 2*ERODE_SIZE + 1, 2*ERODE_SIZE+1 ), cv::Point( ERODE_SIZE, ERODE_SIZE ) ); // Apply the dilation operation cv::erode(thres, thres, element ); thres.copyTo(out2); cv::threshold(thres, thres, SENSITIVITY_VALUE, 255, cv::THRESH_BINARY); cv::Mat closed_thres; cv::Mat structuringElement = getStructuringElement(cv::MORPH_ELLIPSE, cv::Size(CLOSE_SIZE, CLOSE_SIZE)); cv::morphologyEx( thres, closed_thres, cv::MORPH_CLOSE, structuringElement ); // dilated_thres.copyTo(output); //closed_thres.copyTo(output); //notice how we use the '&' operator for objectDetected and output. This is because we wish //to take the values passed into the function and manipulate them, rather than just working with a copy. //eg. we draw to the output to be displayed in the main() function. bool objectDetected = false; cv::Mat temp; closed_thres.copyTo(temp); //these two vectors needed for output of findContours vector< vector > contours; vector hierarchy; //find contours of filtered image using openCV findContours function //findContours(temp,contours,hierarchy,CV_RETR_CCOMP,CV_CHAIN_APPROX_SIMPLE );// retrieves all contours cv::findContours(temp,contours,hierarchy,CV_RETR_EXTERNAL,CV_CHAIN_APPROX_SIMPLE );// retrieves external contours //if contours vector is not empty, we have found some objects if(contours.size()>0)objectDetected=true; else objectDetected = false; if(objectDetected){ //the largest contour is found at the end of the contours vector //we will simply assume that the biggest contour is the object we are looking for. //vector< vector > largestContourVec; //largestContourVec.push_back(contours.at(contours.size()-1)); //make a bounding rectangle around the largest contour then find its centroid //this will be the object's final esticv::Mated position. vector nc_rects; // Non connected rectangles for(size_t i=0; i c_rects; // Connected rectangles cleanBBoxes(nc_rects, cur.size(), c_rects); if (c_rects.size() > 0) { for (const auto& rect : c_rects) cv::rectangle(output, rect, cv::Scalar(0, 255, 0), 2); cv::Rect objBRect = c_rects.front(); //cv::rectangle(output, objBRect, cv::Scalar(0, 255, 0), 2); papillon::BoundingBox bbox = papillon::BoundingBox(); bbox.x = objBRect.x / (float)cur.size().width; bbox.y = objBRect.y / (float)cur.size().height; bbox.width = objBRect.width / (float)cur.size().width; bbox.height = objBRect.height / (float)cur.size().height; pub_cmd.publish(bbox); } } //make some temp x and y variables so we dont have to type out so much //~ int x = objectBoundingRectangle.x; //~ int y = objectBoundingRectangle.y; //~ int width = objectBoundingRectangle.width; //~ int height = objectBoundingRectangle.height; //draw a rectangle around the object //rectangle(output, Point(x,y), Point(x+width, y+height), cv::Scalar(0, 255, 0), 2); //write the position of the object to the screen //putText(output,"Tracking object at (" + intToString(x)+","+intToString(y)+")",Point(x,y),1,1,cv::Scalar(255,0,0),2); } void cleanBBoxes(vector nc_rects, cv::Size img, vector &c_rects) { int max = 0; for (auto r : nc_rects) { cv::Point center = rectCenter(r); if (r.width > img.width / 2 || r.height > img.height / 8) continue; if (center.y > img.height * 2 / 3 || center.y < img.height / 3) continue; int r_area = r.area(); if (max < r_area) { max = r_area; c_rects.insert(c_rects.begin(), r); } else { c_rects.push_back(r); } } } cv::Point rectCenter(cv::Rect r) { return cv::Point(r.x + r.width/2, r.y + r.height / 2); } inline bool isFlowCorrect(cv::Point2f u) { return !cvIsNaN(u.x) && !cvIsNaN(u.y) && fabs(u.x) < 1e9 && fabs(u.y) < 1e9; } }; int main(int argc, char **argv) { ros::init(argc, argv, "test_opencv"); Traite_image dataset=Traite_image(); ros::spin(); return 0; }