Cut faraway objects based on depth map

edit

I would like to do grabcut which uses a depth map that cuts away far objects, that is used in mixed reality application. So I would like to show just the front of what I see and the background as virtual reality scene.

The problem right now I tried to adapt so code and what I get is front which is cut but in black color, the mask actually.

I don't know where is the problem settle.

The input is a depth map from zed camera. here is a picture of the behaviour: enter image description here

 private void convertToGrayScaleValues(Mat mask)
    {
        int width = mask.rows();
        int height = mask.cols();
        byte[] buffer = new byte[width * height];
        mask.get(0, 0, buffer);
        for (int x = 0; x < width; x++)
        {
            for (int y = 0; y < height; y++)
            {
                int value = buffer[y * width + x];

                if (value == Imgproc.GC_BGD)
                {
                    buffer[y * width + x] = 0; // for sure background
                }
                else if (value == Imgproc.GC_PR_BGD)
                {
                    buffer[y * width + x] = 85; // probably background
                }
                else if (value == Imgproc.GC_PR_FGD)
                {
                    buffer[y * width + x] = (byte)170; // probably foreground
                }
                else
                {
                    buffer[y * width + x] = (byte)255; // for sure foreground
                }
            }
        }
        mask.put(0, 0, buffer);
    }

For Each depth frame from Camera:

  Mat erodeElement = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(4, 4));
        Mat dilateElement = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(7, 7));


        depth.copyTo(maskFar);


        Core.normalize(maskFar, maskFar, 0, 255, Core.NORM_MINMAX, CvType.CV_8U);

        Imgproc.cvtColor(maskFar, maskFar, Imgproc.COLOR_BGR2GRAY);

        Imgproc.threshold(maskFar, maskFar, 180, 255, Imgproc.THRESH_BINARY);
        Imgproc.dilate(maskFar, maskFar, erodeElement);
        Imgproc.erode(maskFar, maskFar, dilateElement);

        Mat bgModel = new Mat();
        Mat fgModel = new Mat();

        Imgproc.grabCut(image, maskFar, new OpenCVForUnity.CoreModule.Rect(), bgModel, fgModel, 1, Imgproc.GC_INIT_WITH_MASK);

        convertToGrayScaleValues(maskFar); // back to grayscale values
        Imgproc.threshold(maskFar, maskFar, 180, 255, Imgproc.THRESH_TOZERO);

        Mat foreground = new Mat(image.size(), CvType.CV_8UC4, new Scalar(0, 0, 0));
        image.copyTo(foreground, maskFar);

        Utils.fastMatToTexture2D(foreground, texture);