dnn: classify with mean subtraction, results different for caffe and opencv

edit

This question is strongly connected to: http://answers.opencv.org/question/17...

I'm working with OpenCV 3.4.1 and caffe 1.0.0

I have different results for classification in caffe and OpenCV only when I'm using mean subtraction, without it everything works fine. I have the same observation for some caffenet and squeezenet_v1.1 models. My code:

import cv2
import caffe
import numpy as np

#  function comparing results of image classification in opencv and caffe:
#  with mean substraction results don't match
def caffe_vs_ocv_with_mean_subs(caffemodel, deploy_prototxt):
    #  read mean
    blob = caffe.proto.caffe_pb2.BlobProto()
    with open("mean.binaryproto", 'rb') as f:
        blob.ParseFromString(f.read())
        data = np.array(blob.data).reshape([blob.channels, blob.height, blob.width])
        cv_mean = [np.mean(data[0]), np.mean(data[1]), np.mean(data[2])]

    # define caffe net
    net = caffe.Net(deploy_prototxt, caffemodel, caffe.TEST)
    transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})
    transformer.set_mean('data', data)
    transformer.set_transpose('data', (2, 0, 1))

    # get caffe prediction
    img = cv2.imread("example.jpg")
    img = cv2.resize(img, (227, 227), cv2.INTER_LINEAR)
    net.blobs['data'].data[...] = transformer.preprocess('data', img)
    out = net.forward()
    print "mean: " + str(cv_mean)
    print "caffe probs: " + str(out['prob'][0])

    # define openCv net
    cv_net = cv2.dnn.readNetFromCaffe(deploy_prototxt, caffemodel)
    # get opencv predictions:
    img_for_cv = cv2.imread("example.jpg")
    #  swapRB=False, just like in http://answers.opencv.org/question/178680/how-to-use-meanbinaryproto-with-blobfromimages/,
    #  but I tried any combinations of swapRB, crop flags
    cv_blob = cv2.dnn.blobFromImage(img_for_cv, 1, (227,227), cv_mean, swapRB=False, crop=False)
    cv_net.setInput(cv_blob)
    prob = cv_net.forward()
    print "cv probs" + str(prob)


#  no mean substrucation - this works fine
def caffe_vs_ocv_simple(caffemodel, deploy_prototxt):
    # no mean substraction

    # define caffe net
    net = caffe.Net(deploy_prototxt, caffemodel, caffe.TEST)
    transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})
    transformer.set_transpose('data', (2, 0, 1))

    # get caffe prediction
    img = cv2.imread("example.jpg")
    img = cv2.resize(img, (227, 227), cv2.INTER_LINEAR) # the same way as resizing cv2.dnn.blobFromImage when crop=False
    net.blobs['data'].data[...] = transformer.preprocess('data', img)
    out = net.forward()
    print "caffe probs: " + str(out['prob'][0])

  # define openCv net
    cv_net = cv2.dnn.readNetFromCaffe(deploy_prototxt, caffemodel)

    # get opencv predictions:
    img_for_cv = cv2.imread("example.jpg")
    cv_blob = cv2.dnn.blobFromImage(img_for_cv, 1, (227,227), (), swapRB=False, crop=False)
    cv_net.setInput(cv_blob)
    prob = cv_net.forward()
    print "cv probs" + str(prob)


#  tested on different models:

#  normal reference caffemodel (but with some other mean):
caffe_vs_ocv_with_mean_subs("caffenet_1000.caffemodel", "caffenet_deploy_1000.prototxt")
#  my custom versions of caffenet and squeezenet with different number of output classes
caffe_vs_ocv_with_mean_subs("caffenet_5.caffemodel", "caffenet_deploy_5.prototxt")
caffe_vs_ocv_with_mean_subs("squeezenet_1.1.caffemodel", "squeezenet_deploy.prototxt")
#  all tests with mean substraction failed, giving completely wrong results

#  examples without mean substraction:
#  here differences between probabilities are tolerable, it works fine:
caffe_vs_ocv_simple("caffenet_model.caffemodel", "caffenet_deploy.prototxt")
caffe_vs_ocv_simple("caffenet_5.caffemodel", "caffenet_deploy_5.prototxt")