Detect and fix text skew by rotating image

Question

Is there a way (using something like OpenCV) to detect text skew and correct it by rotating the image? Pretty much like this?

Rotating an image seems easy enough if you know the angle, but for the images I'm processing, I wont...it will need to be detected somehow.

Answer 1

Based on your above comment, here is the code based on the tutorial here, working fine for the above image,

Source

Rotated

 Mat src=imread("text.png",0);
 Mat thr,dst;
 threshold(src,thr,200,255,THRESH_BINARY_INV);
 imshow("thr",thr);

  std::vector<cv::Point> points;
  cv::Mat_<uchar>::iterator it = thr.begin<uchar>();
  cv::Mat_<uchar>::iterator end = thr.end<uchar>();
  for (; it != end; ++it)
    if (*it)
      points.push_back(it.pos());

  cv::RotatedRect box = cv::minAreaRect(cv::Mat(points));
  cv::Mat rot_mat = cv::getRotationMatrix2D(box.center, box.angle, 1);

  //cv::Mat rotated(src.size(),src.type(),Scalar(255,255,255));
  Mat rotated;
  cv::warpAffine(src, rotated, rot_mat, src.size(), cv::INTER_CUBIC);
 imshow("rotated",rotated);

Edit:

Also see the answer here , might be helpful.

Answer 2

Here's an implementation of the Projection Profile Method algorithm for skew angle estimation. Various angle points are projected into an accumulator array where the skew angle can be defined as the angle of projection within a search interval that maximizes alignment. The idea is to rotate the image at various angles and generate a histogram of pixels for each iteration. To determine the skew angle, we compare the maximum difference between peaks and using this skew angle, rotate the image to correct the skew.

---

Input

Result

Skew angle: -5

import cv2
import numpy as np
from scipy.ndimage import interpolation as inter

def correct_skew(image, delta=1, limit=5):
    def determine_score(arr, angle):
        data = inter.rotate(arr, angle, reshape=False, order=0)
        histogram = np.sum(data, axis=1, dtype=float)
        score = np.sum((histogram[1:] - histogram[:-1]) ** 2, dtype=float)
        return histogram, score

    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1] 

    scores = []
    angles = np.arange(-limit, limit + delta, delta)
    for angle in angles:
        histogram, score = determine_score(thresh, angle)
        scores.append(score)

    best_angle = angles[scores.index(max(scores))]

    (h, w) = image.shape[:2]
    center = (w // 2, h // 2)
    M = cv2.getRotationMatrix2D(center, best_angle, 1.0)
    corrected = cv2.warpAffine(image, M, (w, h), flags=cv2.INTER_CUBIC, \
            borderMode=cv2.BORDER_REPLICATE)

    return best_angle, corrected

if __name__ == '__main__':
    image = cv2.imread('1.png')
    angle, corrected = correct_skew(image)
    print('Skew angle:', angle)
    cv2.imshow('corrected', corrected)
    cv2.waitKey()

Note: You may have to adjust the delta or limit values depending on the image. The delta value controls iteration step, it will iterate up until the limit which controls the maximum angle. This method is straightforward by iteratively checking each angle + delta and currently only works to correct skew in the range of +/- 5 degrees. If you need to correct at a larger angle, adjust the limit value.

Answer 3

I would provide javacv for your reference.

package com.test13;

import org.opencv.core.*;
import org.opencv.imgproc.Imgproc;
import org.opencv.imgcodecs.Imgcodecs;

public class EdgeDetection {

    static{ System.loadLibrary(Core.NATIVE_LIBRARY_NAME); }

    public static void main( String[] args ) throws Exception{      
        Mat src = Imgcodecs.imread("src//data//inclined_text.jpg");
        Mat src_gray = new Mat();
        Imgproc.cvtColor(src, src_gray, Imgproc.COLOR_BGR2GRAY);
        Imgcodecs.imwrite("src//data//inclined_text_src_gray.jpg", src_gray);

        Mat output = new Mat();
        Core.bitwise_not(src_gray, output);
        Imgcodecs.imwrite("src//data//inclined_text_output.jpg", output);

        Mat points = Mat.zeros(output.size(),output.type());  
        Core.findNonZero(output, points);   

        MatOfPoint mpoints = new MatOfPoint(points);    
        MatOfPoint2f points2f = new MatOfPoint2f(mpoints.toArray());
        RotatedRect box = Imgproc.minAreaRect(points2f);

        Mat src_squares = src.clone();
        Mat rot_mat = Imgproc.getRotationMatrix2D(box.center, box.angle, 1);
        Mat rotated = new Mat(); 
        Imgproc.warpAffine(src_squares, rotated, rot_mat, src_squares.size(), Imgproc.INTER_CUBIC);
        Imgcodecs.imwrite("src//data//inclined_text_squares_rotated.jpg",rotated);    
    }
}

Answer 4

private fun main(){
    val bmp:Bitmap? = null //Any bitmap (if you are working with bitmap)
    var mRgba = Mat() // else you can direct use MAT on onCameraFrame 
    val mGray = Mat()
    val bmp32: Bitmap = bmp.copy(Bitmap.Config.ARGB_8888, true)
    Utils.bitmapToMat(bmp32, mRgba)
    Imgproc.cvtColor(mRgba, mGray, Imgproc.COLOR_BGR2GRAY)
    mRgba = makeOrientationCorrection(mRgba,mGray)// here actual magic starts
    Imgproc.cvtColor(mRgba, mGray, Imgproc.COLOR_BGR2GRAY)
    val bmpOutX = Bitmap.createBitmap(
        mRgba.cols(),
        mRgba.rows(),
        Bitmap.Config.ARGB_8888
    )
    Utils.matToBitmap(mRgba, bmpOutX)
    binding.imagePreview.setImageBitmap(bmpOutX!!)
}

private fun makeOrientationCorrection(mRGBA:Mat, mGRAY:Mat):Mat{
    val dst = Mat()
    val cdst = Mat()
    val cdstP: Mat
    Imgproc.Canny(mGRAY, dst, 50.0, 200.0, 3, false)
    Imgproc.cvtColor(dst, cdst, Imgproc.COLOR_GRAY2BGR)
    cdstP = cdst.clone()

    val linesP = Mat()
    Imgproc.HoughLinesP(dst, linesP, 1.0, Math.PI/180, 50, 50.0, 10.0)

    var biggestLineX1 = 0.0
    var biggestLineY1 = 0.0
    var biggestLineX2 = 0.0
    var biggestLineY2 = 0.0
    var biggestLine = 0.0
    for (x in 0 until linesP.rows()) {
        val l = linesP[x, 0]

        Imgproc.line(
            cdstP, org.opencv.core.Point(l[0], l[1]),
            org.opencv.core.Point(l[2], l[3]),
            Scalar(0.0, 0.0, 255.0), 3, Imgproc.LINE_AA, 0)
    }

    for (x in 0 until linesP.rows()) {
        val l = linesP[x, 0]
        val x1 = l[0]
        val y1 = l[1]
        val x2 = l[2]
        val y2 = l[3]
        val lineHeight = sqrt(((x2 - x1).pow(2.0)) + ((y2 - y1).pow(2.0)))
        if(biggestLine<lineHeight){
            val angleOfRotationX1 = angleOf(PointF(x1.toFloat(),y1.toFloat()),PointF(x2.toFloat(),y2.toFloat()))
            Log.e("angleOfRotationX1","$angleOfRotationX1")
            if(angleOfRotationX1<45.0 || angleOfRotationX1>270.0){
                biggestLine = lineHeight
                if(angleOfRotationX1<45.0){
                    biggestLineX1 = x1
                    biggestLineY1 = y1
                    biggestLineX2 = x2
                    biggestLineY2 = y2
                }
                if(angleOfRotationX1>270.0){
                    biggestLineX1 = x2
                    biggestLineY1 = y2
                    biggestLineX2 = x1
                    biggestLineY2 = y1
                }
            }
        }
        if(x==linesP.rows()-1){
            Imgproc.line(
                cdstP, org.opencv.core.Point(biggestLineX1, biggestLineY1),
                org.opencv.core.Point(biggestLineX2, biggestLineY2),
                Scalar(255.0, 0.0, 0.0), 3, Imgproc.LINE_AA, 0)
        }
    }
    var angle = angleOf(PointF(biggestLineX1.toFloat(),biggestLineY1.toFloat()),PointF(biggestLineX2.toFloat(),biggestLineY2.toFloat()))
    Log.e("angleOfRotationX2","$angle")
    angle -= (angle * 2)
    return deskew(mRGBA,angle)
}

fun angleOf(p1: PointF, p2: PointF): Double {
    val deltaY = (p1.y - p2.y).toDouble()
    val deltaX = (p2.x - p1.x).toDouble()
    val result = Math.toDegrees(Math.atan2(deltaY, deltaX))
    return if (result < 0) 360.0 + result else result
}

private fun deskew(src:Mat, angle:Double):Mat{
    val center = org.opencv.core.Point((src.width() / 2).toDouble(), (src.height() / 2).toDouble())
    val scaleBy = if(angle<0){
        1.0+((0.5*angle)/45)//max scale down by 0.50(50%) based on angle
    }else{
        1.0-((0.3*angle)/45)//max scale down by 0.50(50%) based on angle
    }

    Log.e("scaleBy",""+scaleBy)
    val rotImage = Imgproc.getRotationMatrix2D(center, angle, scaleBy)
    val size = Size(src.width().toDouble(), src.height().toDouble())
    Imgproc.warpAffine(src, src, rotImage, size, Imgproc.INTER_LINEAR + Imgproc.CV_WARP_FILL_OUTLIERS)
    return src
}

Make sure you run this "makeOrientationCorrection()" method on another thread. otherwise, UI won't update for 2-5 sec.