mobileCyberGamesApp/android/app/src/main/java/com/izouma/mobilecybergames/VideoProcessUtil.java

package com.izouma.mobilecybergames;

import android.content.Context;
import android.graphics.Bitmap;
import android.os.Environment;
import android.text.TextUtils;
import android.util.Log;
import android.util.Pair;

import com.googlecode.tesseract.android.ResultIterator;
import com.googlecode.tesseract.android.TessBaseAPI;

import org.opencv.android.Utils;
import org.opencv.core.Core;
import org.opencv.core.CvException;
import org.opencv.core.Mat;
import org.opencv.core.MatOfPoint;
import org.opencv.core.Point;
import org.opencv.core.Rect;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.imgproc.Imgproc;
import org.opencv.ml.SVM;
import org.opencv.videoio.VideoCapture;
import org.opencv.videoio.Videoio;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import static org.opencv.core.Core.NATIVE_LIBRARY_NAME;
import static org.opencv.core.Core.ROTATE_90_COUNTERCLOCKWISE;
import static org.opencv.core.Core.rotate;
import static org.opencv.core.CvType.CV_32F;
import static org.opencv.core.CvType.CV_32FC1;
import static org.opencv.core.CvType.CV_8U;
import static org.opencv.core.CvType.CV_8UC1;
import static org.opencv.core.CvType.CV_8UC3;
import static org.opencv.imgproc.Imgproc.CHAIN_APPROX_SIMPLE;
import static org.opencv.imgproc.Imgproc.COLOR_BGR2GRAY;
import static org.opencv.imgproc.Imgproc.COLOR_BGR2HSV;
import static org.opencv.imgproc.Imgproc.MORPH_ELLIPSE;
import static org.opencv.imgproc.Imgproc.MORPH_RECT;
import static org.opencv.imgproc.Imgproc.RETR_CCOMP;
import static org.opencv.imgproc.Imgproc.bilateralFilter;
import static org.opencv.imgproc.Imgproc.blur;
import static org.opencv.imgproc.Imgproc.boundingRect;
import static org.opencv.imgproc.Imgproc.cvtColor;
import static org.opencv.imgproc.Imgproc.filter2D;
import static org.opencv.imgproc.Imgproc.findContours;
import static org.opencv.imgproc.Imgproc.getStructuringElement;
import static org.opencv.imgproc.Imgproc.medianBlur;
import static org.opencv.imgproc.Imgproc.rectangle;
import static org.opencv.imgproc.Imgproc.resize;

public class VideoProcessUtil {
    static {
        System.loadLibrary(NATIVE_LIBRARY_NAME);
    }

    public static void init(Context context) {
        File dir = context.getExternalFilesDir("tessdata");
        if (!dir.exists()) {
            dir.mkdirs();
        }
        copyAssets(context, "pubg.traineddata", dir);
        copyAssets(context, "pubg.xml", dir);
    }

    public static void copyAssets(Context context, String name, File dir) {
        try {
            InputStream is = context.getAssets().open(name);
            FileOutputStream fos = new FileOutputStream(new File(dir, name));
            byte[] buffer = new byte[1024];
            int byteCount = 0;
            while ((byteCount = is.read(buffer)) != -1) {
                fos.write(buffer, 0, byteCount);
            }
            fos.flush();
            is.close();
            fos.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public static Map<String, Object> processVideo(Context context, String path) {
        String xmlPath = new File(context.getExternalFilesDir("tessdata"), "pubg.xml").getPath();

        SVM svm = SVM.load(xmlPath);
        Map<String, Object> map = new HashMap<>();
        Mat frame = new Mat();
        Mat v = new Mat();
        long frameCount = 0;
        VideoCapture capture = new VideoCapture();
        if (capture.open(path)) {
            System.out.println(capture.get(Videoio.CAP_PROP_FRAME_WIDTH));
            System.out.println(capture.get(Videoio.CAP_PROP_FRAME_HEIGHT));
        } else {
            return null;
        }
        while (capture.read(frame)) {
            frameCount++;
            if (frameCount % 200 == 0) {
                System.gc();
            }
            if (frameCount % 3 != 0) {
                continue;
            }
            if (matchGameOver(svm, frame)) {
                String rank = extractRank(frame);
                String time = extractGameTime(frame);
                if (TextUtils.isEmpty(rank) || TextUtils.isEmpty(time)) {
                    continue;
                }
                rank = rank.trim().replace(" ", "").replace("\n", "");
                time = time.trim().replace(" ", "").replace("\n", "");
                if (!Pattern.matches("^第\\d{1,3}$", rank)
                        || !Pattern.matches("((^([0-9]+[.][0-9]*))|(^\\d{1,2}))分钟", time)) {
                    continue;
                }
                map.put("rank", rank);
                map.put("time", time);
                System.out.println(map);
                break;
            }
            System.out.println(frameCount + " frame processed");
        }
        capture.release();
        frame.release();
        v.release();
        System.gc();
        return map;
    }

    private static boolean matchGameOver(SVM svm, Mat inputImage) {
        if (inputImage.cols() < inputImage.rows()) {
            rotate(inputImage, inputImage, ROTATE_90_COUNTERCLOCKWISE);
        }
        Mat v = new Mat();
        removeBlackBar(inputImage, inputImage);
        resize(inputImage, v, new Size(854, 480));
        Mat p = v.reshape(1, 1);
        p.convertTo(p, CV_32FC1);
        float res = svm.predict(p);
        return res == 1;
    }

    private static String extractRank(Mat inputImage) {
        String result = null;
        int m = inputImage.rows();
        int n = inputImage.cols();
        inputImage = inputImage.rowRange(0, (int) (m * 0.5)).colRange((int) (n * 0.5), n);

        Mat resultGray = new Mat(inputImage.rows(), inputImage.cols(), CV_8U, new Scalar(0));
        Mat resultColor = new Mat(inputImage.rows(), inputImage.cols(), CV_8UC3, new Scalar(255, 255, 255));
        HSVFilter(inputImage, resultGray, resultColor, 25, 65, 40, 255, 130, 255);

        int erosion_size = m / 400;
        erode(resultGray, resultGray, erosion_size);

        int dilation_size = m / 400 * 4;
        dilate(resultGray, resultGray, dilation_size);

        List<MatOfPoint> contours = new ArrayList<>();
        Mat hierarchy = new Mat();
        findContours(resultGray, contours, hierarchy, RETR_CCOMP, CHAIN_APPROX_SIMPLE);
        Rect textRect = new Rect();
        for (int i = 0; i < contours.size(); i++) {
            Rect rect = boundingRect(contours.get(i));
            if (rect.area() > textRect.area()) {
                textRect = rect;
            }
        }
        if (textRect.width > 0 && textRect.height > 0) {
            Mat textImg = inputImage.rowRange(textRect.y, textRect.y + textRect.height).colRange(textRect.x, textRect.x + textRect.width);
            Mat mat1 = new Mat(textImg.rows(), textImg.cols(), CV_8UC1);
            for (int i = 0; i < textImg.rows(); i++) {
                for (int j = 0; j < textImg.cols(); j++) {
                    double s = (textImg.get(i, j)[1] + textImg.get(i, j)[2]) / 2;
                    mat1.put(i, j, s > 220 ? s : 0);
                }
            }
            medianBlur(mat1, mat1, 3);
            for (MatOfPoint contour : contours) {
                Rect rect = boundingRect(contour);
                rectangle(inputImage, rect, new Scalar(0, 0, 255), 1);
            }
            rectangle(inputImage, textRect, new Scalar(0, 255, 0), 2);
            result = doOCR(mat1);
            textImg.release();
            mat1.release();
        }
        inputImage.release();
        resultGray.release();
        resultColor.release();

        System.gc();
        return result;
    }

    private static String extractGameTime(Mat inputImage) {
        String result = null;
        int m = inputImage.rows();
        int n = inputImage.cols();
        inputImage = inputImage.rowRange((int) (m * 0.5), (int) (m * 0.8)).colRange((int) (n * 0.5), n);
        Mat resultGray = new Mat(inputImage.rows(), inputImage.cols(), CV_8U, new Scalar(0));
        Mat resultColor = new Mat(inputImage.rows(), inputImage.cols(), CV_8UC3, new Scalar(255, 255, 255));
        HSVFilter(inputImage, resultGray, resultColor, 15, 28, 80, 255, 80, 255);

        int dilation_size = resultGray.rows() / 120 * 3;
        dilate(resultGray, resultGray, dilation_size);

        int erosion_size = resultGray.rows() / 120 * 3;
        erode(resultGray, resultGray, erosion_size);

        dilate(resultGray, resultGray, dilation_size);

        List<MatOfPoint> contours = new ArrayList<>();
        Mat hierarchy = new Mat();
        findContours(resultGray, contours, hierarchy, RETR_CCOMP, CHAIN_APPROX_SIMPLE);
        Rect textRect = new Rect();
        for (int i = 0; i < contours.size(); i++) {
            Rect rect = boundingRect(contours.get(i));
            if (rect.area() > textRect.area() && rect.height < inputImage.rows() / 3) {
                textRect = rect;
            }
        }
        if (textRect.width > 0 && textRect.height > 0) {
            Mat textImg = inputImage.rowRange(textRect.y, textRect.y + textRect.height).colRange(textRect.x, textRect.x + textRect.width);
            textImg.convertTo(textImg, CV_8UC3);
            Mat imageBf = new Mat();
            bilateralFilter(textImg, imageBf, 3, 3, 2);//双边滤波
            Mat imageEnhance = new Mat();
            Mat kernel = new Mat(3, 3, CV_32F);
            kernel.put(0, 0, 0);
            kernel.put(0, 1, -1);
            kernel.put(0, 2, 0);
            kernel.put(1, 0, 0);
            kernel.put(1, 1, 4);
            kernel.put(1, 2, 0);
            kernel.put(2, 0, 0);
            kernel.put(2, 1, -1);
            kernel.put(2, 2, 0);
            filter2D(imageBf, imageEnhance, CV_8UC3, kernel);
            cvtColor(textImg, textImg, COLOR_BGR2GRAY);
            textImg.convertTo(textImg, CV_8UC1);
            for (MatOfPoint contour : contours) {
                Rect rect = boundingRect(contour);
                rectangle(inputImage, rect, new Scalar(0, 0, 255), 1);
            }
            rectangle(inputImage, textRect, new Scalar(0, 255, 0), 1);

            result = doOCR(textImg);
            imageBf.release();
            imageEnhance.release();
            textImg.release();
        }
        inputImage.release();
        resultGray.release();
        resultColor.release();

        System.gc();
        return result;
    }

    private static void dilate(Mat src, Mat dst, int dilation_size) {
        Mat dilationElement = getStructuringElement(MORPH_ELLIPSE,
                new Size(2 * dilation_size + 1, 2 * dilation_size + 1),
                new Point(dilation_size, dilation_size));
        Imgproc.dilate(src, dst, dilationElement);
    }

    private static void erode(Mat src, Mat dst, int erosion_size) {
        Mat erodeElement = getStructuringElement(MORPH_RECT,
                new Size(2 * erosion_size + 1, 2 * erosion_size + 1),
                new Point(erosion_size, erosion_size));
        Imgproc.erode(src, dst, erodeElement);
    }

    private static void HSVFilter(Mat inputImage, Mat resultGray, Mat resultColor, int h_min, int h_max, int s_min, int s_max, int v_min, int v_max) {
        Mat hsvImage = new Mat();
        blur(inputImage, hsvImage, new Size(3, 3));
        cvtColor(hsvImage, hsvImage, COLOR_BGR2HSV);
        double H = 0.0, S = 0.0, V = 0.0;
        for (int i = 0; i < hsvImage.rows(); i++) {
            for (int j = 0; j < hsvImage.cols(); j++) {
                H = hsvImage.get(i, j)[0];
                S = hsvImage.get(i, j)[1];
                V = hsvImage.get(i, j)[2];

                if ((H >= h_min && H <= h_max)
                        && (S >= s_min && S <= s_max)
                        && (V >= v_min && V <= v_max)) {
                    resultGray.put(i, j, 255d);
                    resultColor.put(i, j, inputImage.get(i, j));
                }
            }
        }
        hsvImage.release();
    }

    private static String doOCR(Mat img) {
        Bitmap bmp = null;
        try {
            Imgproc.cvtColor(img, img, Imgproc.COLOR_GRAY2RGBA, 4);
            bmp = Bitmap.createBitmap(img.cols(), img.rows(), Bitmap.Config.ARGB_8888);
            Utils.matToBitmap(img, bmp);
        } catch (CvException e) {
            Log.d("Exception", e.getMessage());
        }
        final TessBaseAPI baseApi = new TessBaseAPI();
        baseApi.init("", "pubg");
        baseApi.setPageSegMode(TessBaseAPI.PageSegMode.PSM_SINGLE_LINE);
        baseApi.setVariable(TessBaseAPI.VAR_SAVE_BLOB_CHOICES, TessBaseAPI.VAR_TRUE);
        // Ensure that text is recognized.
        baseApi.setImage(bmp);
        String recognizedText = baseApi.getUTF8Text();

        // Iterate through the results.
        ResultIterator iterator = baseApi.getResultIterator();
        List<Pair<String, Double>> choicesAndConfidences;
        iterator.begin();
        do {
            choicesAndConfidences = iterator.getSymbolChoicesAndConfidence();

            for (Pair<String, Double> choiceAndConfidence : choicesAndConfidences) {
                String choice = choiceAndConfidence.first;
                Double conf = choiceAndConfidence.second;
                System.out.println("choice:" + choice + ", conf:" + conf);
            }
        } while (iterator.next(TessBaseAPI.PageIteratorLevel.RIL_SYMBOL));
        iterator.delete();
        baseApi.end();
        bmp.recycle();
        return recognizedText;
    }

    private static void removeBlackBar(Mat src, Mat dst) {
        int[] top = new int[20];
        int[] bottom = new int[20];
        int[] left = new int[20];
        int[] right = new int[20];
        int rowStep = (int) (src.rows() * 0.4 / 20);
        int colStep = (int) (src.cols() * 0.4 / 20);
        int r = (int) (src.rows() * 0.3);
        for (int i = 0; i < 20; i++) {
            for (int c = 2; c + 2 < src.cols(); c++) {
                double B = (src.get(r, c - 2)[0] + src.get(r, c - 1)[0] + src.get(r, c)[0] + src.get(r, c + 1)[0] + src.get(r, c + 2)[0]) / 3f;
                double G = (src.get(r, c - 2)[1] + src.get(r, c - 1)[1] + src.get(r, c)[1] + src.get(r, c + 1)[1] + src.get(r, c + 2)[1]) / 3f;
                double R = (src.get(r, c - 2)[2] + src.get(r, c - 1)[2] + src.get(r, c)[2] + src.get(r, c + 1)[2] + src.get(r, c + 2)[2]) / 3f;
                if (B > 5 && G > 5 && R > 5) {
                    left[i] = c;
                    break;
                }
            }
            for (int c = src.cols() - 3; c - 2 >= 0; c--) {
                double B = (src.get(r, c - 2)[0] + src.get(r, c - 1)[0] + src.get(r, c)[0] + src.get(r, c + 1)[0] + src.get(r, c + 2)[0]) / 3f;
                double G = (src.get(r, c - 2)[1] + src.get(r, c - 1)[1] + src.get(r, c)[1] + src.get(r, c + 1)[1] + src.get(r, c + 2)[1]) / 3f;
                double R = (src.get(r, c - 2)[2] + src.get(r, c - 1)[2] + src.get(r, c)[2] + src.get(r, c + 1)[2] + src.get(r, c + 2)[2]) / 3f;
                if (B > 5 && G > 5 && R > 5) {
                    right[i] = c;
                    break;
                }
            }
            r += rowStep;
        }
        int c = (int) (src.cols() * 0.3);
        for (int i = 0; i < 20; i++) {
            for (r = 2; r + 2 < src.rows(); r++) {
                double B = (src.get(r, c - 2)[0] + src.get(r, c - 1)[0] + src.get(r, c)[0] + src.get(r, c + 1)[0] + src.get(r, c + 2)[0]) / 3f;
                double G = (src.get(r, c - 2)[1] + src.get(r, c - 1)[1] + src.get(r, c)[1] + src.get(r, c + 1)[1] + src.get(r, c + 2)[1]) / 3f;
                double R = (src.get(r, c - 2)[2] + src.get(r, c - 1)[2] + src.get(r, c)[2] + src.get(r, c + 1)[2] + src.get(r, c + 2)[2]) / 3f;
                if (B > 5 && G > 5 && R > 5) {
                    top[i] = r;
                    break;
                }
            }
            for (r = src.rows() - 3; r - 2 >= 0; r--) {
                double B = (src.get(r, c - 2)[0] + src.get(r, c - 1)[0] + src.get(r, c)[0] + src.get(r, c + 1)[0] + src.get(r, c + 2)[0]) / 3f;
                double G = (src.get(r, c - 2)[1] + src.get(r, c - 1)[1] + src.get(r, c)[1] + src.get(r, c + 1)[1] + src.get(r, c + 2)[1]) / 3f;
                double R = (src.get(r, c - 2)[2] + src.get(r, c - 1)[2] + src.get(r, c)[2] + src.get(r, c + 1)[2] + src.get(r, c + 2)[2]) / 3f;
                if (B > 5 && G > 5 && R > 5) {
                    bottom[i] = r;
                    break;
                }
            }
            c += colStep;
        }
        int offsetTop = (int) Math.max(avg(top) - 2, 0);
        int offsetBottom = (int) Math.min(avg(bottom) + 2, src.rows() - 1);
        int offsetLeft = (int) Math.max(avg(left) - 2, 0);
        int offsetRight = (int) Math.min(avg(right) + 2, src.cols() - 1);
        new Mat(src, new Rect(new Point(offsetLeft, offsetTop), new Point(offsetRight, offsetBottom))).copyTo(dst);
    }

    private static double avg(int[] arr) {
        int sum = 0;
        for (int i : arr) {
            sum += i;
        }
        return sum / (double) arr.length;
    }
}