OpenCV detects most rectangular contours only if they are not aligned with the camera

edit

Even though I am using OpenCVForUnity, I don't think the problem is about the wrapper, but OpenCV in general.
If the markers are aligned with the camera i.e. parallel to the image border, then most of the rectangles are not detected, but when I rotate the camera they are detected (example: https://i.stack.imgur.com/XVmzA.gif).

I had to shoten the code to make it more clear. But there are only two parts, the first to find contours and the second to filter them

// Phase 1: detect contours
var thresh = Imgproc.threshold(gray, edged, 125, 255, Imgproc.THRESH_BINARY_INV);
Imgproc.findContours(edged, contourCandidates, hierarchy, Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE);

var filteredContours = new List<MatOfPoint>();

var cntArray = contourCandidates.ToArray(); var cnt2fArray = new MatOfPoint2f[cntArray.Length];

// Phase 2: filter contours
for (int i = 0; i < contourCandidates.Count; ++i) 
{   
    if (cntArray[i].rows() < detectorParameters.get_minMarkerPerimeterRate() 
        || contourCandidates[i].rows() < detectorParameters.get_maxMarkerPerimeterRate())
    { 
        continue;
    }

    cnt2fArray[i] = new MatOfPoint2f(cntArray[i].toArray());

    // check if square and convex   
    MatOfPoint2f approxCurve = new MatOfPoint2f();  
    Imgproc.approxPolyDP(cnt2fArray[i], approxCurve, contourCandidates[i].rows() * detectorParameters.get_polygonalApproxAccuracyRate(), true);     
    if (approxCurve.rows() != 4 || !Imgproc.isContourConvex(new MatOfPoint(approxCurve.toArray()))) continue;

    var approxArray = approxCurve.toArray();

    //// filter small contours  
    //// sum of square length of all sides  
    double outlineLenght = 0;   
    for (int j = 0; j < 4; j++)     
    {       
        // c^2 = a^2 + b^2      
        outlineLenght += Math.Pow(approxArray[j].x - approxArray[(j + 1) % 4].x, 2) +
                            Math.Pow(approxArray[j].y - approxArray[(j + 1) % 4].y, 2);     
    }

    double minOutlineSqLength = 4 * Math.Pow(20, 2);    
    if (outlineLenght < minOutlineSqLength) continue;

    // check if it is too near to the image border  
    var minDistanceToBorder = detectorParameters.get_minDistanceToBorder();     
    bool tooNearBorder = false;     
    for (int j = 0; j < 4; j++)     {       
    if (approxArray[j].x < minDistanceToBorder 
        || approxArray[j].y < minDistanceToBorder 
        || approxArray[j].x > src.cols() - 1 - minDistanceToBorder 
        || approxArray[j].y > src.rows() - 1 - minDistanceToBorder)             
        tooNearBorder = true;   
    }   

    if (tooNearBorder) continue;

    // passed all test -> add candidate     
    var currentCandidatePoints = new List<Point>();     
    for (int j = 0; j < 4; j++)     
    {       
        currentCandidatePoints.Add(approxArray[j]);     
    }

    filteredContours.Add(new MatOfPoint(currentCandidatePoints.ToArray())); 
}