重写字符提取算法

src/main/java/com/izouma/awesomeadmin/util/VideoProcessTool.java

@@ -229,87 +229,6 @@ public class VideoProcessTool {
         return result;
     }
 
-    public String extractRank1(Mat src) {
-        String result = null;
-        int m = src.rows();
-        int n = src.cols();
-        Mat roi = src.rowRange(0, (int) (m * 0.4)).colRange((int) (n * 0.75), n);
-        Mat filtered = rankFilter(roi);
-        Mat gray = new Mat();
-        adaptiveThreshold(filtered, gray, 255, CV_ADAPTIVE_THRESH_MEAN_C, CV_THRESH_BINARY_INV, 15, 12);
-        MatVector contours = new MatVector();
-        findContours(gray, contours, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE);
-        List<Rect> rects = new ArrayList<>();
-        for (int i = 0; i < contours.size(); i++) {
-            Mat contour = contours.get(i);
-            Rect rect = boundingRect(contour);
-            rects.add(rect);
-            contour.release();
-        }
-        rects.sort((o1, o2) -> o2.area() - o1.area());
-        for (int i = 0; i < rects.size(); i++) {
-            Rect rect = rects.get(i);
-            if (i < 5) {
-                imwrite("/Users/drew/Desktop/img/" + System.currentTimeMillis() + ".jpg", filtered.rowRange(rect.y(), rect.y() + rect.height()).colRange(rect.x(), rect.x() + rect.width()));
-                String str = doOCR(filtered.rowRange(rect.y(), rect.y() + rect.height()).colRange(rect.x(), rect.x() + rect.width()));
-                System.out.println(str);
-                rectangle(roi, rect, new Scalar(0, 0, 255, 0), 2, LINE_AA, 0);
-                if (str != null) {
-                    str = str.replaceAll("[ \n]", "");
-                    if (Pattern.matches("^第\\d{1,3}$", str)) {
-                        rectangle(roi, rect, new Scalar(0, 255, 0, 0), 2, LINE_AA, 0);
-                        result = str;
-                        break;
-                    }
-                }
-            }
-        }
-        filtered.release();
-        gray.release();
-        roi.release();
-        return result;
-    }
-
-    public String extractGameTime1(Mat src) {
-        String result = null;
-        int m = src.rows();
-        int n = src.cols();
-        Mat roi = src.rowRange((int) (m * 0.5), (int) (m * 0.8)).colRange((int) (n * 0.5), n);
-        Mat filtered = timeFilter(roi);
-        Mat gray = new Mat();
-        adaptiveThreshold(filtered, gray, 255, CV_ADAPTIVE_THRESH_MEAN_C, CV_THRESH_BINARY_INV, 15, 12);
-        MatVector contours = new MatVector();
-        findContours(gray, contours, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE);
-        List<Rect> rects = new ArrayList<>();
-        for (int i = 0; i < contours.size(); i++) {
-            Mat contour = contours.get(i);
-            Rect rect = boundingRect(contour);
-            rects.add(rect);
-            contour.release();
-        }
-        rects.sort((o1, o2) -> o2.area() - o1.area());
-        for (int i = 0; i < rects.size(); i++) {
-            Rect rect = rects.get(i);
-            if (i < 5) {
-                imwrite("/Users/drew/Desktop/img/" + System.currentTimeMillis() + ".jpg", filtered.rowRange(rect.y(), rect.y() + rect.height()).colRange(rect.x(), rect.x() + rect.width()));
-                String str = doOCR(filtered.rowRange(rect.y(), rect.y() + rect.height()).colRange(rect.x(), rect.x() + rect.width()));
-                System.out.println(str);
-                rectangle(roi, rect, new Scalar(0, 0, 255, 0), 2, LINE_AA, 0);
-                if (str != null) {
-                    str = str.replaceAll("[ \n]", "");
-                    if (Pattern.matches("((^([0-9]+[.][0-9]*))|(^\\d{1,2}))分钟", str)) {
-                        rectangle(roi, rect, new Scalar(0, 255, 0, 0), 2, LINE_AA, 0);
-                        result = str;
-                        break;
-                    }
-                }
-            }
-        }
-        filtered.release();
-        gray.release();
-        roi.release();
-        return result;
-    }
 
     private void dilate(Mat src, Mat dst, int dilation_size) {
         Mat dilationElement = getStructuringElement(MORPH_ELLIPSE,
@@ -355,51 +274,6 @@ public class VideoProcessTool {
         resultGrayIndexer.release();
     }
 
-    public Mat rankFilter(Mat src) {
-        Mat result = new Mat(src.rows(), src.cols(), CV_8UC1, new Scalar(0));
-        UByteRawIndexer srcIndexer = src.createIndexer();
-        UByteRawIndexer resultIndexer = result.createIndexer();
-        for (int i = 0; i < src.rows(); i++) {
-            for (int j = 0; j < src.cols(); j++) {
-                int db = Math.abs(srcIndexer.get(i, j, 0) - 12);
-                int dg = Math.abs(srcIndexer.get(i, j, 1) - 243);
-                int dr = Math.abs(srcIndexer.get(i, j, 2) - 252);
-
-                int gray = 255 - (db + dg + dr);
-                if (gray < 0) gray = 0;
-                if (gray > 255) gray = 255;
-                gray *= 1.2;
-                if (gray > 255) gray = 255;
-                resultIndexer.put(i, j, gray);
-            }
-        }
-        srcIndexer.release();
-        resultIndexer.release();
-        return result;
-    }
-
-    public Mat timeFilter(Mat src) {
-        Mat result = new Mat(src.rows(), src.cols(), CV_8UC1, new Scalar(0));
-        UByteRawIndexer srcIndexer = src.createIndexer();
-        UByteRawIndexer resultIndexer = result.createIndexer();
-        for (int i = 0; i < src.rows(); i++) {
-            for (int j = 0; j < src.cols(); j++) {
-                int db = Math.abs(srcIndexer.get(i, j, 0) - 36);
-                int dg = Math.abs(srcIndexer.get(i, j, 1) - 183);
-                int dr = Math.abs(srcIndexer.get(i, j, 2) - 230);
-
-                int gray = 255 - (db + dg + dr);
-                if (gray < 0) gray = 0;
-                if (gray > 255) gray = 255;
-                gray *= 1.2;
-                if (gray > 255) gray = 255;
-                resultIndexer.put(i, j, gray);
-            }
-        }
-        srcIndexer.release();
-        resultIndexer.release();
-        return result;
-    }
 
     private String doOCR(Mat img) {
         imwrite("/tmp/text.jpg", img);

src/main/java/com/izouma/awesomeadmin/util/VideoProcessToolNew.java

@@ -0,0 +1,274 @@
+package com.izouma.awesomeadmin.util;
+
+import com.izouma.awesomeadmin.service.OSSFileService;
+import org.apache.commons.lang3.StringUtils;
+import org.bytedeco.javacpp.indexer.UByteRawIndexer;
+import org.bytedeco.javacpp.tesseract;
+import org.bytedeco.javacv.FFmpegFrameGrabber;
+import org.bytedeco.javacv.Frame;
+import org.bytedeco.javacv.FrameGrabber;
+import org.bytedeco.javacv.OpenCVFrameConverter;
+import org.hibernate.validator.internal.util.privilegedactions.GetResource;
+
+import javax.imageio.ImageIO;
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.regex.Pattern;
+
+import static org.bytedeco.javacpp.lept.pixRead;
+import static org.bytedeco.javacpp.opencv_core.*;
+import static org.bytedeco.javacpp.opencv_imgcodecs.imencode;
+import static org.bytedeco.javacpp.opencv_imgcodecs.imwrite;
+import static org.bytedeco.javacpp.opencv_imgproc.*;
+import static org.bytedeco.javacpp.opencv_ml.SVM;
+import static org.bytedeco.javacpp.tesseract.PSM_SINGLE_LINE;
+
+public class VideoProcessToolNew {
+    private OSSFileService ossFileService = new OSSFileService();
+
+    public Map<String, String> processVideo(String path, int frameSkip) throws FrameGrabber.Exception {
+        SVM svm = SVM.load(GetResource.class.getClassLoader().getResource("trainneddata/pubg.xml").getPath());
+        Map<String, String> map = new HashMap<>();
+        OpenCVFrameConverter.ToMat converterToMat = new OpenCVFrameConverter.ToMat();
+        long frameCount = 0;
+        FrameGrabber videoGrabber = new FFmpegFrameGrabber(path);
+        videoGrabber.start();
+        Frame vFrame;
+        do {
+            vFrame = videoGrabber.grabFrame();
+            if (vFrame != null) {
+                frameCount++;
+                if (frameCount % 200 == 0) {
+                    System.gc();
+                }
+                if (frameSkip > 0 && frameCount % frameSkip != 0) {
+                    continue;
+                }
+                Mat frame = converterToMat.convert(vFrame);
+                if (matchGameOver(svm, frame)) {
+                    String rank = extractRank(frame);
+                    if (StringUtils.isEmpty(rank)) continue;
+                    String time = extractGameTime(frame);
+                    if (StringUtils.isEmpty(time)) continue;
+
+                    map.put("rank", rank);
+                    map.put("time", time);
+                    map.put("image", uploadImage(frame));
+                    System.out.println(map);
+                    break;
+                }
+                frame.release();
+                System.out.println(frameCount + " frame processed");
+            }
+        } while (vFrame != null);
+        videoGrabber.stop();
+        videoGrabber.release();
+        svm.deallocate();
+        System.gc();
+        return map;
+    }
+
+    public boolean matchGameOver(SVM svm, Mat inputImage) {
+        Mat rotated = new Mat();
+        if (inputImage.cols() < inputImage.rows()) {
+            rotate(inputImage, rotated, ROTATE_90_COUNTERCLOCKWISE);
+        } else {
+            inputImage.copyTo(rotated);
+        }
+        Mat resized = new Mat();
+        Mat noBlackBar = new Mat();
+        removeBlackBar(rotated, noBlackBar);
+        resize(noBlackBar, resized, new Size(854, 480));
+        Mat reshaped = resized.reshape(1, 1);
+        Mat toPredict = new Mat();
+        reshaped.convertTo(toPredict, CV_32F);
+        float res = svm.predict(toPredict);
+
+        rotated.release();
+        resized.release();
+        noBlackBar.release();
+        reshaped.release();
+        toPredict.release();
+        return res == 1;
+    }
+
+
+    public String extractRank(Mat src) {
+        String result = null;
+        int m = src.rows();
+        int n = src.cols();
+        Mat roi = src.rowRange(0, (int) (m * 0.4)).colRange((int) (n * 0.75), n);
+        Mat filtered = rankFilter(roi);
+        Mat gray = new Mat();
+        adaptiveThreshold(filtered, gray, 255, CV_ADAPTIVE_THRESH_MEAN_C, CV_THRESH_BINARY_INV, 15, 12);
+        MatVector contours = new MatVector();
+        findContours(gray, contours, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE);
+        List<Rect> rects = new ArrayList<>();
+        for (int i = 0; i < contours.size(); i++) {
+            Mat contour = contours.get(i);
+            Rect rect = boundingRect(contour);
+            rects.add(rect);
+            contour.release();
+        }
+        rects.sort((o1, o2) -> o2.area() - o1.area());
+        for (int i = 0; i < rects.size(); i++) {
+            Rect rect = rects.get(i);
+            if (i < 5) {
+                imwrite("/Users/drew/Desktop/img/" + System.currentTimeMillis() + ".jpg", filtered.rowRange(rect.y(), rect.y() + rect.height()).colRange(rect.x(), rect.x() + rect.width()));
+                String str = doOCR(filtered.rowRange(rect.y(), rect.y() + rect.height()).colRange(rect.x(), rect.x() + rect.width()));
+                System.out.println(str);
+                rectangle(roi, rect, new Scalar(0, 0, 255, 0), 2, LINE_AA, 0);
+                if (str != null) {
+                    str = str.replaceAll("[ \n]", "");
+                    if (Pattern.matches("^第\\d{1,3}$", str)) {
+                        rectangle(roi, rect, new Scalar(0, 255, 0, 0), 2, LINE_AA, 0);
+                        result = str;
+                        break;
+                    }
+                }
+            }
+        }
+        filtered.release();
+        gray.release();
+        roi.release();
+        return result;
+    }
+
+    public String extractGameTime(Mat src) {
+        String result = null;
+        int m = src.rows();
+        int n = src.cols();
+        Mat roi = src.rowRange((int) (m * 0.5), (int) (m * 0.8)).colRange((int) (n * 0.5), n);
+        Mat filtered = timeFilter(roi);
+        Mat gray = new Mat();
+        adaptiveThreshold(filtered, gray, 255, CV_ADAPTIVE_THRESH_MEAN_C, CV_THRESH_BINARY_INV, 15, 12);
+        MatVector contours = new MatVector();
+        findContours(gray, contours, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE);
+        List<Rect> rects = new ArrayList<>();
+        for (int i = 0; i < contours.size(); i++) {
+            Mat contour = contours.get(i);
+            Rect rect = boundingRect(contour);
+            rects.add(rect);
+            contour.release();
+        }
+        rects.sort((o1, o2) -> o2.area() - o1.area());
+        for (int i = 0; i < rects.size(); i++) {
+            Rect rect = rects.get(i);
+            if (i < 5) {
+                imwrite("/Users/drew/Desktop/img/" + System.currentTimeMillis() + ".jpg", filtered.rowRange(rect.y(), rect.y() + rect.height()).colRange(rect.x(), rect.x() + rect.width()));
+                String str = doOCR(filtered.rowRange(rect.y(), rect.y() + rect.height()).colRange(rect.x(), rect.x() + rect.width()));
+                System.out.println(str);
+                rectangle(roi, rect, new Scalar(0, 0, 255, 0), 2, LINE_AA, 0);
+                if (str != null) {
+                    str = str.replaceAll("[ \n]", "");
+                    if (Pattern.matches("((^([0-9]+[.][0-9]*))|(^\\d{1,2}))分钟", str)) {
+                        rectangle(roi, rect, new Scalar(0, 255, 0, 0), 2, LINE_AA, 0);
+                        result = str;
+                        break;
+                    }
+                }
+            }
+        }
+        filtered.release();
+        gray.release();
+        roi.release();
+        return result;
+    }
+
+    public Mat bgrFilter(Mat src, int b, int g, int r) {
+        Mat result = new Mat(src.rows(), src.cols(), CV_8UC1, new Scalar(0));
+        UByteRawIndexer srcIndexer = src.createIndexer();
+        UByteRawIndexer resultIndexer = result.createIndexer();
+        for (int i = 0; i < src.rows(); i++) {
+            for (int j = 0; j < src.cols(); j++) {
+                int db = Math.abs(srcIndexer.get(i, j, 0) - b);
+                int dg = Math.abs(srcIndexer.get(i, j, 1) - g);
+                int dr = Math.abs(srcIndexer.get(i, j, 2) - r);
+
+                int gray = 255 - (db + dg + dr);
+                if (gray < 0) gray = 0;
+                if (gray > 255) gray = 255;
+                gray *= 1.2;
+                if (gray > 255) gray = 255;
+                resultIndexer.put(i, j, gray);
+            }
+        }
+        srcIndexer.release();
+        resultIndexer.release();
+        return result;
+    }
+
+    public Mat rankFilter(Mat src) {
+        return bgrFilter(src, 12, 243, 252);
+    }
+
+    public Mat timeFilter(Mat src) {
+        return bgrFilter(src, 36, 183, 230);
+    }
+
+    private String doOCR(Mat img) {
+        imwrite("/tmp/text.jpg", img);
+        final tesseract.TessBaseAPI baseApi = new tesseract.TessBaseAPI();
+        baseApi.Init(GetResource.class.getClassLoader().getResource("trainneddata").getPath(), "pubg");
+        baseApi.SetPageSegMode(PSM_SINGLE_LINE);
+        baseApi.SetImage(pixRead("/tmp/text.jpg"));
+        String recognizedText = baseApi.GetUTF8Text().getString();
+        return recognizedText;
+    }
+
+    private void removeBlackBar(Mat src, Mat dst) {
+        Mat gray = new Mat();
+        cvtColor(src, gray, COLOR_BGR2GRAY);
+        Mat thres = new Mat();
+        threshold(gray, thres, 2, 255, THRESH_BINARY);
+        MatVector contours = new MatVector();
+        findContours(thres, contours, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE);
+        Rect rect = boundingRect(contours.get(0));
+        for (int i = 1; i < contours.size(); i++) {
+            Rect r = boundingRect(contours.get(i));
+            if (r.area() > rect.area()) {
+                rect.deallocate();
+                rect = r;
+            }
+        }
+        src.colRange(rect.x(), rect.x() + rect.width()).rowRange(rect.y(), rect.y() + rect.height()).copyTo(dst);
+        gray.release();
+        thres.release();
+        contours.deallocate();
+        rect.deallocate();
+    }
+
+    public BufferedImage createBufferedImage(Mat mat) {
+        ByteBuffer byteBuffer = ByteBuffer.allocate(mat.cols() * mat.rows());
+        imencode(".jpg", mat, byteBuffer);
+        byte[] ba = byteBuffer.array();
+        BufferedImage bi = null;
+        try {
+            bi = ImageIO.read(new ByteArrayInputStream(ba));
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        return bi;
+    }
+
+    public String uploadImage(Mat frame) {
+        try {
+            BufferedImage image = createBufferedImage(frame);
+            ByteArrayOutputStream baos = new ByteArrayOutputStream();//io流
+            ImageIO.write(image, "png", baos);//写入流中
+            byte[] bytes = baos.toByteArray();//转换成字节
+            InputStream in = new ByteArrayInputStream(bytes);
+            String path = String.format("images/%s-%s.jpg", new SimpleDateFormat("yyyy-MM-dd-hh-mm-ss").format(new Date()), UUID.randomUUID().toString());
+            return ossFileService.upload(in, path);
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
+}

src/test/java/VideoProcessTest.java

@@ -1,4 +1,5 @@
 import com.izouma.awesomeadmin.util.VideoProcessTool;
+import com.izouma.awesomeadmin.util.VideoProcessToolNew;
 import org.bytedeco.javacpp.opencv_ml;
 import org.bytedeco.javacv.FrameGrabber;
 import org.hibernate.validator.internal.util.privilegedactions.GetResource;
@@ -15,13 +16,28 @@ import static org.bytedeco.javacpp.opencv_imgcodecs.imread;
 @RunWith(JUnit4.class)
 public class VideoProcessTest {
 
-    private VideoProcessTool videoProcessTool = new VideoProcessTool();
+    private VideoProcessTool    videoProcessTool    = new VideoProcessTool();
+    private VideoProcessToolNew videoProcessToolNew = new VideoProcessToolNew();
 
     @Test
     public void testVideo() {
         try {
-            Map<String, String> result = videoProcessTool.processVideo("/Users/drew/Downloads/record1553760548328.mp4", 0);
+            long ts = System.currentTimeMillis();
+            Map<String, String> result = videoProcessTool.processVideo("/Users/drew/Downloads/2019-03-29-03-41-57-5o8n74nz.flv", 0);
             System.out.println(result);
+            System.out.println((System.currentTimeMillis() - ts) / 1000f / 60);
+        } catch (FrameGrabber.Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    @Test
+    public void testVideoNew() {
+        try {
+            long ts = System.currentTimeMillis();
+            Map<String, String> result = videoProcessToolNew.processVideo("/Users/drew/Downloads/2019-03-29-03-41-57-5o8n74nz.flv", 0);
+            System.out.println(result);
+            System.out.println((System.currentTimeMillis() - ts) / 1000f / 60);
         } catch (FrameGrabber.Exception e) {
             e.printStackTrace();
         }
@@ -36,28 +52,45 @@ public class VideoProcessTest {
     }
 
     @Test
-    public void testTime() {
+    public void testRank() {
         Mat img = imread("/Users/drew/Pictures/Screenshots/2019-03-29-03-20-01-svu0rzwo-0001.jpg");
-        String time = videoProcessTool.extractGameTime1(img);
-        System.out.println("time:" + time);
+        String rank = videoProcessToolNew.extractRank(img);
+        System.out.println("rank:" + rank);
+
+        long ts = System.currentTimeMillis();
+        videoProcessTool.extractRank(img);
+        System.out.println(System.currentTimeMillis() - ts);
+
+        ts = System.currentTimeMillis();
+        videoProcessToolNew.extractRank(img);
+        System.out.println(System.currentTimeMillis() - ts);
     }
 
     @Test
-    public void testRank() {
+    public void testTime() {
         Mat img = imread("/Users/drew/Pictures/Screenshots/2019-03-29-03-20-01-svu0rzwo-0001.jpg");
-        String rank = videoProcessTool.extractRank1(img);
-        System.out.println("rank:" + rank);
+        String time = videoProcessToolNew.extractGameTime(img);
+        System.out.println("time:" + time);
+
+        long ts = System.currentTimeMillis();
+        videoProcessTool.extractGameTime(img);
+        System.out.println(System.currentTimeMillis() - ts);
+
+        ts = System.currentTimeMillis();
+        videoProcessToolNew.extractRank(img);
+        System.out.println(System.currentTimeMillis() - ts);
     }
 
+
     @Test
     public void testRankFilter() {
         Mat img = imread("/Users/drew/Pictures/Screenshots/2019-03-29-03-20-01-svu0rzwo-0001.jpg");
-        videoProcessTool.rankFilter(img);
+        videoProcessToolNew.rankFilter(img);
     }
 
     @Test
     public void testTimeFilter() {
         Mat img = imread("/Users/drew/Pictures/Screenshots/2019-03-29-03-20-01-svu0rzwo-0001.jpg");
-        videoProcessTool.timeFilter(img);
+        videoProcessToolNew.timeFilter(img);
     }
 }