This is a follow up question to my question here: http://stackoverflow.com/questions/21715908/local-enhancing-of-license-plate-in-video-sequence
I implemented a basic super-resolution technique suggested by the answer, but I am having trouble obtaining better resolution results. I trimmed the video sequence to 40 frames as seen below:
and using the code below, which takes the first 5 frames and performs Super-resolution, then the super-resolved frame is repeatedly updated with the processing of the rest of the frames in the sequence:
public void Process(Image<Gray, byte> frame)
{
SetRegionOfInterest(frame);
var resizedFrame = ResizeFrame(frame);
InputFrames.Add(resizedFrame);
if(InputFrames.Count > 5)
{
InputFrames.RemoveAt(0);
PerformSuperResolution();
}
}
public void PerformSuperResolution()
{
// WindowSize = 5
var referenceFrame = InputFrames[WindowSize-1].Convert<Gray, byte>();
var featuresToTrack = referenceFrame.GoodFeaturesToTrack(100, 0.1, 5, 10);
referenceFrame.FindCornerSubPix(featuresToTrack, new Size(WindowSize,WindowSize), new Size(-1,-1), new MCvTermCriteria(20, 0.03d));
var resultFrame = InputFrames[WindowSize-1].Convert<Gray, double>();
for(var frameCounter = 0; frameCounter < WindowSize-1; frameCounter++)
{
// Get shift between frames
var shiftResult = GetShiftResult(InputFrames[frameCounter],referenceFrame, featuresToTrack);
// Warp to correct shift
var warpMatrix = new Matrix<double>(new[,] { {1, 0, -shiftResult.ShiftX}, {0, 1, -shiftResult.ShiftY}, {0, 0, 1}});
var warpedFrame = InputFrames[frameCounter].WarpPerspective(warpMatrix,
INTER.CV_INTER_NN,
WARP.CV_WARP_DEFAULT,
new Gray(0));
resultFrame.RunningAvg(warpedFrame.Convert<Gray,double>(), 1, resultFrame.Convert<Gray, byte>());
}
SuperResolutionFrame = resultFrame.Convert<Gray, byte>();
}
public ShiftResult GetShiftResult(Image<Gray, byte> inputFrame, Image<Gray, byte> referenceFrame, PointF[][] ActualFeature)
{
var result = new ShiftResult();
PointF[] NextFeature;
Byte[] Status;
float[] TrackError;
// optical flow
OpticalFlow.PyrLK(referenceFrame, inputFrame, ActualFeature[0],
new Size(WindowSize, WindowSize), 5, new MCvTermCriteria(20, 0.1d),
out NextFeature, out Status, out TrackError);
//get displacements
float[] XdisplacementVectors = new float[NextFeature.Length];
float[] YdisplacementVectors = new float[NextFeature.Length];
for(int i = 0; i < NextFeature.Length; i++)
{
XdisplacementVectors[i] = NextFeature[i].X - ActualFeature[0][i].X;
YdisplacementVectors[i] = NextFeature[i].Y - ActualFeature[0][i].Y;
}
// gets average of displacements (disregards outliers)
result.ShiftX = getAVG(XdisplacementVectors);
result.ShiftY = getAVG(YdisplacementVectors);
return result;
}
where the program calls Process(frame) on an array of input video frames (totaling 40) as such:
for(int i = 0; i < image_array.Count; i++)
{
Res.Process(image_array[i]);
}
and I obtained this resulting image:
As you can see there are only small improvements in the overall resolution, and not much improvement to the readability of the license plate. I've tried experimenting with the various parameters, but the parameters used above seem to be the best I can do.
I am open to any suggestions for improvement (need not be limited to the use of C#/Emgu CV as above), and even different approaches in implementation.
