package com.izouma.mobilecybergames; import android.content.Context; import android.graphics.Bitmap; import android.os.Environment; import android.text.TextUtils; import android.util.Log; import android.util.Pair; import com.googlecode.tesseract.android.ResultIterator; import com.googlecode.tesseract.android.TessBaseAPI; import org.opencv.android.Utils; import org.opencv.core.Core; import org.opencv.core.CvException; import org.opencv.core.Mat; import org.opencv.core.MatOfPoint; import org.opencv.core.Point; import org.opencv.core.Rect; import org.opencv.core.Scalar; import org.opencv.core.Size; import org.opencv.imgproc.Imgproc; import org.opencv.ml.SVM; import org.opencv.videoio.VideoCapture; import org.opencv.videoio.Videoio; import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.regex.Pattern; import java.util.stream.Collectors; import static org.opencv.core.Core.NATIVE_LIBRARY_NAME; import static org.opencv.core.Core.ROTATE_90_COUNTERCLOCKWISE; import static org.opencv.core.Core.rotate; import static org.opencv.core.CvType.CV_32F; import static org.opencv.core.CvType.CV_32FC1; import static org.opencv.core.CvType.CV_8U; import static org.opencv.core.CvType.CV_8UC1; import static org.opencv.core.CvType.CV_8UC3; import static org.opencv.imgproc.Imgproc.CHAIN_APPROX_SIMPLE; import static org.opencv.imgproc.Imgproc.COLOR_BGR2GRAY; import static org.opencv.imgproc.Imgproc.COLOR_BGR2HSV; import static org.opencv.imgproc.Imgproc.MORPH_ELLIPSE; import static org.opencv.imgproc.Imgproc.MORPH_RECT; import static org.opencv.imgproc.Imgproc.RETR_CCOMP; import static org.opencv.imgproc.Imgproc.bilateralFilter; import static org.opencv.imgproc.Imgproc.blur; import static org.opencv.imgproc.Imgproc.boundingRect; import static org.opencv.imgproc.Imgproc.cvtColor; import static org.opencv.imgproc.Imgproc.filter2D; import static org.opencv.imgproc.Imgproc.findContours; import static org.opencv.imgproc.Imgproc.getStructuringElement; import static org.opencv.imgproc.Imgproc.medianBlur; import static org.opencv.imgproc.Imgproc.rectangle; import static org.opencv.imgproc.Imgproc.resize; public class VideoProcessUtil { static { System.loadLibrary(NATIVE_LIBRARY_NAME); } public static void init(Context context) { File dir = context.getExternalFilesDir("tessdata"); if (!dir.exists()) { dir.mkdirs(); } copyAssets(context, "pubg.traineddata", dir); copyAssets(context, "pubg.xml", dir); } public static void copyAssets(Context context, String name, File dir) { try { InputStream is = context.getAssets().open(name); FileOutputStream fos = new FileOutputStream(new File(dir, name)); byte[] buffer = new byte[1024]; int byteCount = 0; while ((byteCount = is.read(buffer)) != -1) { fos.write(buffer, 0, byteCount); } fos.flush(); is.close(); fos.close(); } catch (IOException e) { e.printStackTrace(); } } public static Map processVideo(Context context, String path) { String xmlPath = new File(context.getExternalFilesDir("tessdata"), "pubg.xml").getPath(); SVM svm = SVM.load(xmlPath); Map map = new HashMap<>(); Mat frame = new Mat(); Mat v = new Mat(); long frameCount = 0; VideoCapture capture = new VideoCapture(); if (capture.open(path)) { System.out.println(capture.get(Videoio.CAP_PROP_FRAME_WIDTH)); System.out.println(capture.get(Videoio.CAP_PROP_FRAME_HEIGHT)); } else { return null; } while (capture.read(frame)) { frameCount++; if (frameCount % 200 == 0) { System.gc(); } if (frameCount % 3 != 0) { continue; } if (matchGameOver(svm, frame)) { String rank = extractRank(frame); String time = extractGameTime(frame); if (TextUtils.isEmpty(rank) || TextUtils.isEmpty(time)) { continue; } rank = rank.trim().replace(" ", "").replace("\n", ""); time = time.trim().replace(" ", "").replace("\n", ""); if (!Pattern.matches("^第\\d{1,3}$", rank) || !Pattern.matches("((^([0-9]+[.][0-9]*))|(^\\d{1,2}))分钟", time)) { continue; } map.put("rank", rank); map.put("time", time); System.out.println(map); break; } System.out.println(frameCount + " frame processed"); } capture.release(); frame.release(); v.release(); System.gc(); return map; } private static boolean matchGameOver(SVM svm, Mat inputImage) { if (inputImage.cols() < inputImage.rows()) { rotate(inputImage, inputImage, ROTATE_90_COUNTERCLOCKWISE); } Mat v = new Mat(); removeBlackBar(inputImage, inputImage); resize(inputImage, v, new Size(854, 480)); Mat p = v.reshape(1, 1); p.convertTo(p, CV_32FC1); float res = svm.predict(p); return res == 1; } private static String extractRank(Mat inputImage) { String result = null; int m = inputImage.rows(); int n = inputImage.cols(); inputImage = inputImage.rowRange(0, (int) (m * 0.5)).colRange((int) (n * 0.5), n); Mat resultGray = new Mat(inputImage.rows(), inputImage.cols(), CV_8U, new Scalar(0)); Mat resultColor = new Mat(inputImage.rows(), inputImage.cols(), CV_8UC3, new Scalar(255, 255, 255)); HSVFilter(inputImage, resultGray, resultColor, 25, 65, 40, 255, 130, 255); int erosion_size = m / 400; erode(resultGray, resultGray, erosion_size); int dilation_size = m / 400 * 4; dilate(resultGray, resultGray, dilation_size); List contours = new ArrayList<>(); Mat hierarchy = new Mat(); findContours(resultGray, contours, hierarchy, RETR_CCOMP, CHAIN_APPROX_SIMPLE); Rect textRect = new Rect(); for (int i = 0; i < contours.size(); i++) { Rect rect = boundingRect(contours.get(i)); if (rect.area() > textRect.area()) { textRect = rect; } } if (textRect.width > 0 && textRect.height > 0) { Mat textImg = inputImage.rowRange(textRect.y, textRect.y + textRect.height).colRange(textRect.x, textRect.x + textRect.width); Mat mat1 = new Mat(textImg.rows(), textImg.cols(), CV_8UC1); for (int i = 0; i < textImg.rows(); i++) { for (int j = 0; j < textImg.cols(); j++) { double s = (textImg.get(i, j)[1] + textImg.get(i, j)[2]) / 2; mat1.put(i, j, s > 220 ? s : 0); } } medianBlur(mat1, mat1, 3); for (MatOfPoint contour : contours) { Rect rect = boundingRect(contour); rectangle(inputImage, rect, new Scalar(0, 0, 255), 1); } rectangle(inputImage, textRect, new Scalar(0, 255, 0), 2); result = doOCR(mat1); textImg.release(); mat1.release(); } inputImage.release(); resultGray.release(); resultColor.release(); System.gc(); return result; } private static String extractGameTime(Mat inputImage) { String result = null; int m = inputImage.rows(); int n = inputImage.cols(); inputImage = inputImage.rowRange((int) (m * 0.5), (int) (m * 0.8)).colRange((int) (n * 0.5), n); Mat resultGray = new Mat(inputImage.rows(), inputImage.cols(), CV_8U, new Scalar(0)); Mat resultColor = new Mat(inputImage.rows(), inputImage.cols(), CV_8UC3, new Scalar(255, 255, 255)); HSVFilter(inputImage, resultGray, resultColor, 15, 28, 80, 255, 80, 255); int dilation_size = resultGray.rows() / 120 * 3; dilate(resultGray, resultGray, dilation_size); int erosion_size = resultGray.rows() / 120 * 3; erode(resultGray, resultGray, erosion_size); dilate(resultGray, resultGray, dilation_size); List contours = new ArrayList<>(); Mat hierarchy = new Mat(); findContours(resultGray, contours, hierarchy, RETR_CCOMP, CHAIN_APPROX_SIMPLE); Rect textRect = new Rect(); for (int i = 0; i < contours.size(); i++) { Rect rect = boundingRect(contours.get(i)); if (rect.area() > textRect.area() && rect.height < inputImage.rows() / 3) { textRect = rect; } } if (textRect.width > 0 && textRect.height > 0) { Mat textImg = inputImage.rowRange(textRect.y, textRect.y + textRect.height).colRange(textRect.x, textRect.x + textRect.width); textImg.convertTo(textImg, CV_8UC3); Mat imageBf = new Mat(); bilateralFilter(textImg, imageBf, 3, 3, 2);//双边滤波 Mat imageEnhance = new Mat(); Mat kernel = new Mat(3, 3, CV_32F); kernel.put(0, 0, 0); kernel.put(0, 1, -1); kernel.put(0, 2, 0); kernel.put(1, 0, 0); kernel.put(1, 1, 4); kernel.put(1, 2, 0); kernel.put(2, 0, 0); kernel.put(2, 1, -1); kernel.put(2, 2, 0); filter2D(imageBf, imageEnhance, CV_8UC3, kernel); cvtColor(textImg, textImg, COLOR_BGR2GRAY); textImg.convertTo(textImg, CV_8UC1); for (MatOfPoint contour : contours) { Rect rect = boundingRect(contour); rectangle(inputImage, rect, new Scalar(0, 0, 255), 1); } rectangle(inputImage, textRect, new Scalar(0, 255, 0), 1); result = doOCR(textImg); imageBf.release(); imageEnhance.release(); textImg.release(); } inputImage.release(); resultGray.release(); resultColor.release(); System.gc(); return result; } private static void dilate(Mat src, Mat dst, int dilation_size) { Mat dilationElement = getStructuringElement(MORPH_ELLIPSE, new Size(2 * dilation_size + 1, 2 * dilation_size + 1), new Point(dilation_size, dilation_size)); Imgproc.dilate(src, dst, dilationElement); } private static void erode(Mat src, Mat dst, int erosion_size) { Mat erodeElement = getStructuringElement(MORPH_RECT, new Size(2 * erosion_size + 1, 2 * erosion_size + 1), new Point(erosion_size, erosion_size)); Imgproc.erode(src, dst, erodeElement); } private static void HSVFilter(Mat inputImage, Mat resultGray, Mat resultColor, int h_min, int h_max, int s_min, int s_max, int v_min, int v_max) { Mat hsvImage = new Mat(); blur(inputImage, hsvImage, new Size(3, 3)); cvtColor(hsvImage, hsvImage, COLOR_BGR2HSV); double H = 0.0, S = 0.0, V = 0.0; for (int i = 0; i < hsvImage.rows(); i++) { for (int j = 0; j < hsvImage.cols(); j++) { H = hsvImage.get(i, j)[0]; S = hsvImage.get(i, j)[1]; V = hsvImage.get(i, j)[2]; if ((H >= h_min && H <= h_max) && (S >= s_min && S <= s_max) && (V >= v_min && V <= v_max)) { resultGray.put(i, j, 255d); resultColor.put(i, j, inputImage.get(i, j)); } } } hsvImage.release(); } private static String doOCR(Mat img) { Bitmap bmp = null; try { Imgproc.cvtColor(img, img, Imgproc.COLOR_GRAY2RGBA, 4); bmp = Bitmap.createBitmap(img.cols(), img.rows(), Bitmap.Config.ARGB_8888); Utils.matToBitmap(img, bmp); } catch (CvException e) { Log.d("Exception", e.getMessage()); } final TessBaseAPI baseApi = new TessBaseAPI(); baseApi.init("", "pubg"); baseApi.setPageSegMode(TessBaseAPI.PageSegMode.PSM_SINGLE_LINE); baseApi.setVariable(TessBaseAPI.VAR_SAVE_BLOB_CHOICES, TessBaseAPI.VAR_TRUE); // Ensure that text is recognized. baseApi.setImage(bmp); String recognizedText = baseApi.getUTF8Text(); // Iterate through the results. ResultIterator iterator = baseApi.getResultIterator(); List> choicesAndConfidences; iterator.begin(); do { choicesAndConfidences = iterator.getSymbolChoicesAndConfidence(); for (Pair choiceAndConfidence : choicesAndConfidences) { String choice = choiceAndConfidence.first; Double conf = choiceAndConfidence.second; System.out.println("choice:" + choice + ", conf:" + conf); } } while (iterator.next(TessBaseAPI.PageIteratorLevel.RIL_SYMBOL)); iterator.delete(); baseApi.end(); bmp.recycle(); return recognizedText; } private static void removeBlackBar(Mat src, Mat dst) { int[] top = new int[20]; int[] bottom = new int[20]; int[] left = new int[20]; int[] right = new int[20]; int rowStep = (int) (src.rows() * 0.4 / 20); int colStep = (int) (src.cols() * 0.4 / 20); int r = (int) (src.rows() * 0.3); for (int i = 0; i < 20; i++) { for (int c = 2; c + 2 < src.cols(); c++) { double B = (src.get(r, c - 2)[0] + src.get(r, c - 1)[0] + src.get(r, c)[0] + src.get(r, c + 1)[0] + src.get(r, c + 2)[0]) / 3f; double G = (src.get(r, c - 2)[1] + src.get(r, c - 1)[1] + src.get(r, c)[1] + src.get(r, c + 1)[1] + src.get(r, c + 2)[1]) / 3f; double R = (src.get(r, c - 2)[2] + src.get(r, c - 1)[2] + src.get(r, c)[2] + src.get(r, c + 1)[2] + src.get(r, c + 2)[2]) / 3f; if (B > 5 && G > 5 && R > 5) { left[i] = c; break; } } for (int c = src.cols() - 3; c - 2 >= 0; c--) { double B = (src.get(r, c - 2)[0] + src.get(r, c - 1)[0] + src.get(r, c)[0] + src.get(r, c + 1)[0] + src.get(r, c + 2)[0]) / 3f; double G = (src.get(r, c - 2)[1] + src.get(r, c - 1)[1] + src.get(r, c)[1] + src.get(r, c + 1)[1] + src.get(r, c + 2)[1]) / 3f; double R = (src.get(r, c - 2)[2] + src.get(r, c - 1)[2] + src.get(r, c)[2] + src.get(r, c + 1)[2] + src.get(r, c + 2)[2]) / 3f; if (B > 5 && G > 5 && R > 5) { right[i] = c; break; } } r += rowStep; } int c = (int) (src.cols() * 0.3); for (int i = 0; i < 20; i++) { for (r = 2; r + 2 < src.rows(); r++) { double B = (src.get(r, c - 2)[0] + src.get(r, c - 1)[0] + src.get(r, c)[0] + src.get(r, c + 1)[0] + src.get(r, c + 2)[0]) / 3f; double G = (src.get(r, c - 2)[1] + src.get(r, c - 1)[1] + src.get(r, c)[1] + src.get(r, c + 1)[1] + src.get(r, c + 2)[1]) / 3f; double R = (src.get(r, c - 2)[2] + src.get(r, c - 1)[2] + src.get(r, c)[2] + src.get(r, c + 1)[2] + src.get(r, c + 2)[2]) / 3f; if (B > 5 && G > 5 && R > 5) { top[i] = r; break; } } for (r = src.rows() - 3; r - 2 >= 0; r--) { double B = (src.get(r, c - 2)[0] + src.get(r, c - 1)[0] + src.get(r, c)[0] + src.get(r, c + 1)[0] + src.get(r, c + 2)[0]) / 3f; double G = (src.get(r, c - 2)[1] + src.get(r, c - 1)[1] + src.get(r, c)[1] + src.get(r, c + 1)[1] + src.get(r, c + 2)[1]) / 3f; double R = (src.get(r, c - 2)[2] + src.get(r, c - 1)[2] + src.get(r, c)[2] + src.get(r, c + 1)[2] + src.get(r, c + 2)[2]) / 3f; if (B > 5 && G > 5 && R > 5) { bottom[i] = r; break; } } c += colStep; } int offsetTop = (int) Math.max(avg(top) - 2, 0); int offsetBottom = (int) Math.min(avg(bottom) + 2, src.rows() - 1); int offsetLeft = (int) Math.max(avg(left) - 2, 0); int offsetRight = (int) Math.min(avg(right) + 2, src.cols() - 1); new Mat(src, new Rect(new Point(offsetLeft, offsetTop), new Point(offsetRight, offsetBottom))).copyTo(dst); } private static double avg(int[] arr) { int sum = 0; for (int i : arr) { sum += i; } return sum / (double) arr.length; } }