How to detect unclosed hexagon?

edit

OpenCV has a useful tool called line iterator that can set up an imaginary line between 2 points. You can step along the pixels of this line and examine the pixel intensity and compare its change from the previous pixel. If we set up line iterators radially from the center outward every few degrees and search for changes from white to black, we can find points along the edges. Then we can use OpenCV's FitLine function to find groups of edge points that fit one of the lines of the hexagon. Below is the code I used to get a first line of a hexagon. The only thing not in OpenCV is the function PointLineDistance that just gets the perpendicular distance of a point from a line. To finish this, find the other 5 lines of the hexagon, then find the intersection points.

                            // create a color markup image to view results
                        Size imgSize = srcImg.size();
                        Mat markupImg;
                        cvtColor(srcImg, markupImg, CV_GRAY2BGR);

                        // edge points we find using line iterator
                        vector<Point> edgePoints;

                        // assume center is near middle of image
                        Point2f ctr(imgSize.width / 2, imgSize.height / 2);
                        // step from middle to outer edges of image
                        float startRadius = 0;
                        float endRadius = max(imgSize.width, imgSize.height);
                        // color change threshold
                        int edgeChangeThreshold = 200;
                        // num radial lines to check
                        int numRadials = 120;
                        float dAngle = 360.0 / numRadials;
                        for (float angle = 0; angle < 360; angle += dAngle) {
                            // set up a line iterator from ctr to edge in direction of angle
                            float ar = angle * DTOR;
                            Point2f p1(ctr.x + startRadius * cos(ar), ctr.y - startRadius * sin(ar));
                            Point2f p2(ctr.x + endRadius * cos(ar), ctr.y - endRadius * sin(ar));
                            LineIterator li(srcImg, p1, p2);

                            // get the current value of line point
                            // and step along line looking for intensity change greater than threshold
                            uchar prevVal = *(*li++);
                            uchar currVal;
                            for (int i = 1; i < li.count; i++, ++li) {
                                currVal = *(*li);
                                if (abs(currVal - prevVal) > edgeChangeThreshold) {
                                    // found intensity change, store this point and stop iteration along this radial
                                    edgePoints.push_back(li.pos());
                                    break;
                                }
                            }
                        }

                        for (auto& pt : edgePoints) {
                            circle(markupImg, pt, 3, Scalar(255, 0, 255));
                        }

                        // this will hold the edge points that fit the first line of the hexagon
                        vector<Point> firstLinePts;
                        // take 3 adjacent pts at random and try to fit a line
                        vector<Point> randomPts(3);
                        bool ptsFit = false;
                        while (!ptsFit) {
                            int randIndex = rand() % edgePoints.size() - 4;
                            for (int i = 0; i < randomPts.size(); ++i) {
                                randomPts[i] = edgePoints[randIndex + i];
                            }

                            // try to fit a line with these 3 points
                            Vec4f line;
                            fitLine(randomPts, line, CV_DIST_L2, 0, 0.01, 0.01);
                            Point2f linePt1(line[2], line[3]);
                            Point2f linePt2 = linePt1 + Point2f(line[0], line[1]) * 50;

                            // search for points in edge point list that are close enough to be on the fitted line
                            for (auto& pt : edgePoints) {
                                float dist = PointLineDistance(pt, linePt1, linePt2);
                                if (dist < 4) {
                                    firstLinePts.push_back(pt);
                                }
                            }
                            // if we found enough that fit, assume we have a line of the hexagon
                            if (firstLinePts.size() > (numRadials / 6 / 2)) {
                                ptsFit = true;
                            }
                        }

                        for (auto& pt : firstLinePts ...