How do I select the correct rectangles in the image?

I wrote the code for this in OpenCV.

The basic process is like yours, find outlines and then weed out small and non-convex outlines.

After that, you can iterate over your contours, for each of which you do the following:

1. Sort the border pixels in ascending order using Y then X coords
2. Iterate across the points. For each Y, add all points between each X and the next X to a vector. You now have a vector of all points contained within the vector. You can also use this to calculate the centroid and to calculate the mean RGB colour as below:

      

        
        
        
      

    

Below is some sample code, although note that it is incomplete, it should give you an idea.

void meanColourOfContour( const Mat& frame, vector<Point> contour, Vec3b& colour, vector<Point>& pointsInContour ) {
    sort(contour.begin(), contour.end(), pointSorter);


    //
    // Mean RGB values
    //
    int rsum = 0;
    int gsum = 0;
    int bsum = 0;

    int index = 0;
    Point lastP = contour[index++];
    pointsInContour.push_back(lastP);

    Vec3b rgbValue = frame.at<Vec3b>(lastP);
    rsum += rgbValue[0];
    gsum += rgbValue[1];
    bsum += rgbValue[2];

    int currentRow = lastP.y;
    int lastX = lastP.x;

    // For all remaining points in contour
    while( index < contour.size() ) {
        Point nextP = contour[index];

        // Save it
        pointsInContour.push_back(nextP);

        // If we're on the same row, add in values of intervening points
        if( nextP.y == currentRow ) {
            for( int x = lastX; x < nextP.x; x++ ) {
                Point p(x, currentRow);
                pointsInContour.push_back(p);
                rgbValue = frame.at<Vec3b>(p);
                rsum += rgbValue[0];
                gsum += rgbValue[1];
                bsum += rgbValue[2];
            }
        }
        // Add nextP
        rgbValue = frame.at<Vec3b>(nextP);
        rsum += rgbValue[0];
        gsum += rgbValue[1];
        bsum += rgbValue[2];

        lastX = nextP.x;
        currentRow = nextP.y;
        index++;
    }

    // Calculate mean
    size_t pointCount = pointsInContour.size();
    colour =Vec3b( rsum/pointCount, gsum/pointCount, bsum/pointCount);
}


void extractFacelets( const Mat& frame, vector<tFacelet>& facelets) {
    // Convert to Grey
    Mat greyFrame;
    cvtColor(frame, greyFrame, CV_BGR2GRAY);
    blur( greyFrame, greyFrame, Size(3,3));

    // Canny and find contours
    Mat cannyOut;
    Canny(greyFrame, cannyOut, 100, 200);

    vector<vector<Point>> contours;
    vector<Vec4i> hierarchy;
    findContours(cannyOut, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_NONE);

    // Filter out non convex contours
    for( int i=contours.size()-1; i>=0; i-- ) {
        if( contourArea(contours[i]) < 400 ) {
            contours.erase(contours.begin()+i);
        }
    }

    // For each contour, calculate mean RGB and plot in output
    int cindex = 0;
    for( auto iter = contours.begin(); iter != contours.end(); iter ++ ) {

        // Sort points in contour on ascending Y then X coord
        vector<Point> contour = (vector<Point>)*iter;
        vector<Vec3b> meanColours;

        Vec3b meanColour;
        vector<Point> pointsInContour;
        meanColourOfContour(frame, contour, meanColour, pointsInContour);

        meanColours.push_back(meanColour);

        long x=0; long y=0;
        for( auto iter=pointsInContour.begin(); iter != pointsInContour.end(); iter++ ) {
            Point p = (Point) *iter;
            x += p.x;
            y += p.y;
        }

        tFacelet f;
        f.centroid.x = (int) (x / pointsInContour.size());
        f.centroid.y = (int) (y / pointsInContour.size());
        f.colour = meanColour;
        f.visible = true;
        facelets.push_back(f);
    }

}