Imagine I have a squared game board the same as Match-3 game (namely Candy Crush, Puzzle and Dragon, etc), my objective is to detect the orb type on their screen and save the result as an array. I have a set of orbs image with transparent background as the source image.
I worked on capturing the board screen and splitting the image into chunks of small square image each containing the orb but with the board color in the background.
I tried Histogram matching (both 4 methods) but all return false result. Template Matching yield only some correct matches but mostly incorrect. Possibly due to the background color of the board in the captured image. And in some cases, if my source image is an orb in grayscale (most image is colored), I got this error while comparing my captured image (colored) with the grayscale source image.
OpenCV(3.4.3) Error: Assertion failed ((depth == CV_8U || depth == CV_32F) && type == _templ.type() && _img.dims() <= 2) in void cv::matchTemplate(cv::InputArray, cv::InputArray, cv::OutputArray, int, cv::InputArray), file /build/3_4_pack-android/opencv/modules/imgproc/src/templmatch.cpp, line 1102 E/org.opencv.imgproc: imgproc::matchTemplate_11() caught cv::Exception: OpenCV(3.4.3) /build/3_4_pack-android/opencv/modules/imgproc/src/templmatch.cpp:1102: error: (-215:Assertion failed) (depth == CV_8U || depth == CV_32F) && type == _templ.type() && _img.dims() <= 2 in function 'void cv::matchTemplate(cv::InputArray, cv::InputArray, cv::OutputArray, int, cv::InputArray)'
with this code:
baseImage = new Mat[69]; baseImage[0] = Utils.loadResource(this, R.drawable.water, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED); //normal orb baseImage[1] = Utils.loadResource(this, R.drawable.w_s, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED); //shiny orb baseImage[2] = Utils.loadResource(this, R.drawable.w_w, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED); //weather orb baseImage[3] = Utils.loadResource(this, R.drawable.w_s_w, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED); //shiny weather orb baseImage[4] = Utils.loadResource(this, R.drawable.w_mono, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED); //monotone orb baseImage[5] = Utils.loadResource(this, R.drawable.w_s_mono, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED); //shiny monotone orb baseImage[6] = Utils.loadResource(this, R.drawable.w_target, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED); //targeted orb baseImage[7] = Utils.loadResource(this, R.drawable.w_i0, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED); //initial froze orb baseImage[8] = Utils.loadResource(this, R.drawable.w_i1, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED); //completely frozen orb baseImage[9] = Utils.loadResource(this, R.drawable.w_i2, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED); //second stage frozen orb baseImage[10] = Utils.loadResource(this, R.drawable.w_i3, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED); //last stage frozen orb
baseImage[11] = Utils.loadResource(this, R.drawable.fire, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[12] = Utils.loadResource(this, R.drawable.f_s, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[13] = Utils.loadResource(this, R.drawable.f_w, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[14] = Utils.loadResource(this, R.drawable.f_s_w, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[15] = Utils.loadResource(this, R.drawable.f_mono, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[16] = Utils.loadResource(this, R.drawable.f_s_mono, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[17] = Utils.loadResource(this, R.drawable.f_target, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[18] = Utils.loadResource(this, R.drawable.f_i0, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[19] = Utils.loadResource(this, R.drawable.f_i1, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[20] = Utils.loadResource(this, R.drawable.f_i2, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[21] = Utils.loadResource(this, R.drawable.f_i3, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[22] = Utils.loadResource(this, R.drawable.wood, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[23] = Utils.loadResource(this, R.drawable.p_s, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[24] = Utils.loadResource(this, R.drawable.p_w, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[25] = Utils.loadResource(this, R.drawable.p_s_w, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[26] = Utils.loadResource(this, R.drawable.p_mono, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[27] = Utils.loadResource(this, R.drawable.p_s_mono, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[28] = Utils.loadResource(this, R.drawable.p_target, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[29] = Utils.loadResource(this, R.drawable.p_i0, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[30] = Utils.loadResource(this, R.drawable.p_i1, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[31] = Utils.loadResource(this, R.drawable.p_i2, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[32] = Utils.loadResource(this, R.drawable.p_i3, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[33] = Utils.loadResource(this, R.drawable.light, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[34] = Utils.loadResource(this, R.drawable.l_s, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[35] = Utils.loadResource(this, R.drawable.l_w, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[36] = Utils.loadResource(this, R.drawable.l_s_w, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[37] = Utils.loadResource(this, R.drawable.l_mono, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[38] = Utils.loadResource(this, R.drawable.l_s_mono, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[39] = Utils.loadResource(this, R.drawable.l_target, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[40] = Utils.loadResource(this, R.drawable.l_i0, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[41] = Utils.loadResource(this, R.drawable.l_i1, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[42] = Utils.loadResource(this, R.drawable.l_i2, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[43] = Utils.loadResource(this, R.drawable.l_i3, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[44] = Utils.loadResource(this, R.drawable.dark, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[45] = Utils.loadResource(this, R.drawable.d_s, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[46] = Utils.loadResource(this, R.drawable.d_w, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[47] = Utils.loadResource(this, R.drawable.d_s_w, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[48] = Utils.loadResource(this, R.drawable.d_mono, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[49] = Utils.loadResource(this, R.drawable.d_s_mono, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[50] = Utils.loadResource(this, R.drawable.d_target, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[51] = Utils.loadResource(this, R.drawable.d_i0, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[52] = Utils.loadResource(this, R.drawable.d_i1, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[53] = Utils.loadResource(this, R.drawable.d_i2, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[54] = Utils.loadResource(this, R.drawable.d_i3, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[55] = Utils.loadResource(this, R.drawable.heart, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[56] = Utils.loadResource(this, R.drawable.h_s, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[57] = Utils.loadResource(this, R.drawable.h_w, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[58] = Utils.loadResource(this, R.drawable.h_s_w, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[59] = Utils.loadResource(this, R.drawable.h_mono, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[60] = Utils.loadResource(this, R.drawable.h_s_mono, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[61] = Utils.loadResource(this, R.drawable.h_target, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[62] = Utils.loadResource(this, R.drawable.h_i0, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[63] = Utils.loadResource(this, R.drawable.h_i1, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[64] = Utils.loadResource(this, R.drawable.h_i2, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[65] = Utils.loadResource(this, R.drawable.h_i3, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
baseImage[66] = Utils.loadResource(this, R.drawable.question, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED); //question orb
baseImage[67] = Utils.loadResource(this, R.drawable.stone, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED); //stone orb
baseImage[68] = Utils.loadResource(this, R.drawable.x, Imgcodecs.CV_LOAD_IMAGE_UNCHANGED); //random weather orb
//lazy implementation of template matching
private int compareImage(Mat tmp) {
//Imgproc.cvtColor(tmp, tmp, Imgproc.COLOR_RGBA2BGR);
int matchedIndex = -1;
double maxScore = 0.0;
double threshold = 0.9;
//match against 3 abnormal orb
for(int i=66; i<69; i++) {
double maxVal = compareImage(baseImage[i], tmp);
if(maxVal >= threshold && maxVal > maxScore) {
maxScore = maxVal;
matchedIndex = i;
if(maxVal == 1.0) {
return matchedIndex;
}
}
}
if(matchedIndex < 0) {
int[] normalOrb = new int[]{0, 11, 22, 33, 44, 55};
//match against 6 different normal element orb to find root
for(int i=0; i<normalOrb.length; i++) {
//Log.d("idx", Integer.toString(normalOrb[i]));
//Log.d("t.row", Boolean.toString(baseImage[normalOrb[i]].rows() > tmp.rows()));
//Log.d("t.row", Boolean.toString(baseImage[normalOrb[i]].cols() > tmp.cols()));
double maxVal = compareImage(baseImage[normalOrb[i]], tmp);
if (maxVal > maxScore) {
maxScore = maxVal;
matchedIndex = normalOrb[i];
if(maxVal == 1.0) {
return matchedIndex;
}
}
}
//match against same element orb of different properties
int last_idx = matchedIndex;
for(int j=last_idx+1; j<last_idx+10; j++) {
//Log.d("idx", Integer.toString(j));
double maxVal = compareImage(baseImage[j], tmp);
if(maxVal > maxScore) {
maxScore = maxVal;
matchedIndex = j;
if(maxVal == 1.0) {
break;
}
}
}
}
return matchedIndex;
}
private double compareImage(Mat source, Mat tmp) {
if(source.rows() > tmp.rows() && source.cols() > tmp.cols()) {
int r_rows = source.rows() - tmp.rows() + 1;
int r_cols = source.cols() - tmp.cols() + 1;
Mat result = new Mat();
result.create(r_rows, r_cols, CvType.CV_32F);
Imgproc.matchTemplate(source, tmp, result, Imgproc.TM_CCOEFF_NORMED);
//Core.normalize(result, result, 0, 1, Core.NORM_MINMAX, -1, new Mat());
Core.MinMaxLocResult mmr = Core.minMaxLoc(result);
double maxVal = mmr.maxVal;
return maxVal;
} else {
Log.d("Compare failed", "Changing to Monotone mode.");
Imgproc.cvtColor(source, source, Imgproc.COLOR_RGBA2GRAY);
Mat mTmp = new Mat();
tmp.copyTo(mTmp);
Imgproc.cvtColor(mTmp, mTmp, Imgproc.COLOR_RGBA2GRAY);
if(source.rows() < tmp.rows() && source.cols() < tmp.cols()) {
Mat copy = new Mat();
source.copyTo(copy);
tmp.copyTo(source);
copy.copyTo(tmp);
}
Log.d("DepthMatch?", Boolean.toString(mTmp.depth()== CV_32F));
int r_rows = (source.rows() > tmp.rows() && source.cols() > tmp.cols())? (source.rows() - mTmp.rows() + 1) : (mTmp.rows() - source.rows() + 1);
Log.d("r_rows",""+r_rows);
int r_cols = (source.rows() > tmp.rows() && source.cols() > tmp.cols())? (source.cols() - mTmp.cols() + 1) : (mTmp.cols() - source.cols() + 1);
Log.d("r_cols",""+r_cols);
Mat result = new Mat();
result.create(r_rows, r_cols, CvType.CV_32F);
Imgproc.matchTemplate(source, mTmp, result, Imgproc.TM_CCOEFF_NORMED);
//Core.normalize(result, result, 0, 1, Core.NORM_MINMAX, -1, new Mat());
Core.MinMaxLocResult mmr = Core.minMaxLoc(result);
double maxVal = mmr.maxVal;
return maxVal;
}
Histogram comparison give me erroneous result, but rather fast while Template Matching offers more promising result but at cost of slower speed. For my case, each captured image needs to be compared with 19 source images as worst case, Template Matching consumes at least few seconds for just a 6x6 board. Is there any suggestion on other approach for the aforementioned scenario, or any possibly fixes to my code to generate better match at faster pace?
