对于获取的实时画面,不利于在软件侧(客户端侧)调用机器学习或者深度学习的库,因为整个软件采用 Java/Kotlin 编写的。
object WImagesProcess { init { System.load("${FileUtil.loadPath}WImagesProcess.dll") } /** * 堆代码 duidaima.com * 算法的版本号 */ external fun getVersion():String /** * 获取 OpenCV 对应相机的 index id * @param pidvid 相机的 pid、vid */ external fun getCameraIndexIdFromPidVid(pidvid:String):Int /** * 开启俯拍相机 * @param index 相机的 index id * @param cameraParaMap 相机相关的参数 * @param listener jni 层给 Java 层的回调 */ external fun startTopVideoCapture(index:Int, cameraParaMap:Map<String,String>, listener: VideoCaptureListener) /** * 开启侧拍相机 * @param index 相机的 index id * @param cameraParaMap 相机相关的参数 * @param listener jni 层给 Java 层的回调 */ external fun startRightVideoCapture(index:Int, cameraParaMap:Map<String,String>, listener: VideoCaptureListener) /** * 调用对应的相机拍摄照片,使用时需要将 IntArray 转换成 BufferedImage * @param cameraId 1:俯拍相机; 2:侧拍相机 */ external fun takePhoto(cameraId:Int): IntArray /** * 设置相机的曝光 * @param cameraId 1:俯拍相机; 2:侧拍相机 */ external fun exposure(cameraId: Int, value: Double):Double /** * 设置相机的亮度 * @param cameraId 1:俯拍相机; 2:侧拍相机 */ external fun brightness(cameraId: Int, value: Double):Double /** * 设置相机的焦距 * @param cameraId 1:俯拍相机; 2:侧拍相机 */ external fun focus(cameraId: Int, value: Double):Double /** * 关闭相机,释放相机的资源 * @param cameraId 1:俯拍相机; 2:侧拍相机 */ external fun closeVideoCapture(cameraId:Int) }其中,VideoCaptureListener 是监听 USB 摄像头(相机)行为的 Listener。
interface VideoCaptureListener { /** * Native 层调用相机成功 */ fun onSuccess() /** * jni 将 Native 层调用相机获取每一帧的 Mat 转换成 IntArray,回调给 Java 层 * @param array 回调给 Java 层的 IntArray,Java 层可以将其转化成 BufferedImage */ fun onRead(array: IntArray) /** * Native 层调用相机失败 */ fun onFailed() }VideoCaptureListener#onRead() 方法是在摄像头(相机)打开后,会实时将每一帧的数据通过回调的形式返回给应用层。
#include <jni.h> #ifndef _Include_xxx_WImagesProcess #define _Include_xxx_WImagesProcess #ifdef __cplusplus extern "C" { #endif JNIEXPORT jstring JNICALL Java_xxx_WImagesProcess_getVersion (JNIEnv* env, jobject); JNIEXPORT void JNICALL Java_xxx_WImagesProcess_startTopVideoCapture (JNIEnv* env, jobject,int index,jobject cameraParaMap ,jobject listener); JNIEXPORT void JNICALL Java_xxx_WImagesProcess_startRightVideoCapture (JNIEnv* env, jobject, int index, jobject cameraParaMap, jobject listener); JNIEXPORT jintArray JNICALL Java_xxx_WImagesProcess_takePhoto (JNIEnv* env, jobject, int cameraId); JNIEXPORT double JNICALL Java_xxx_WImagesProcess_exposure (JNIEnv* env, jobject, int cameraId,double value); JNIEXPORT double JNICALL Java_xxx_WImagesProcess_brightness (JNIEnv* env, jobject, int cameraId, double value); JNIEXPORT double JNICALL Java_xxx_WImagesProcess_focus (JNIEnv* env, jobject, int cameraId, double value); JNIEXPORT void JNICALL Java_xxx_WImagesProcess_closeVideoCapture (JNIEnv* env, jobject, int cameraId); JNIEXPORT int JNICALL Java_xxx_WImagesProcess_getCameraIndexIdFromPidVid (JNIEnv* env, jobject, jstring pidvid); #ifdef __cplusplus } #endif #endif #pragma oncexxx 代表的是 Java 项目中 WImagesProcess 类所在的 package 名称。毕竟是公司项目,我不便贴出完整的 package 名称。不熟悉这种写法的,可以参考 JNI 的规范。
JNIEXPORT void JNICALL Java_xxx_WImagesProcess_startTopVideoCapture (JNIEnv* env, jobject, int index, jobject cameraParaMap, jobject listener){ jobject topListener = env-> NewLocalRef(listener); std::map<string, string> mapOut; JavaHashMapToStlMap(env,cameraParaMap,mapOut); jclass listenerClass = env->GetObjectClass(topListener); jmethodID successId = env->GetMethodID(listenerClass, "onSuccess", "()V"); jmethodID readId = env->GetMethodID(listenerClass, "onRead", "([I)V"); jmethodID failedId = env->GetMethodID(listenerClass, "onFailed", "()V"); jobject listenerObject = env->NewLocalRef(listenerClass); try { topVideoCapture = wImageProcess.getVideoCapture(index, mapOut); env->CallVoidMethod(listenerObject, successId); jintArray jarray; topVideoCapture >> topFrame; int* data = new int[topFrame.total()]; int size = topFrame.rows * topFrame.cols; jarray = env->NewIntArray(size); char r, g, b; while (topFlag) { topVideoCapture >> topFrame; for (int i = 0;i < topFrame.total();i++) { r = topFrame.data[3 * i + 2]; g = topFrame.data[3 * i + 1]; b = topFrame.data[3 * i + 0]; data[i] = (((jint)r << 16) & 0x00FF0000) + (((jint)g << 8) & 0x0000FF00) + ((jint)b & 0x000000FF); } env->SetIntArrayRegion(jarray, 0, size, (jint*)data); env->CallVoidMethod(listenerObject, readId, jarray); waitKey(100); } topVideoCapture.release(); env->ReleaseIntArrayElements(jarray, env->GetIntArrayElements(jarray, JNI_FALSE), 0); delete []data; } catch (...) { env->CallVoidMethod(listenerObject, failedId); } env->DeleteLocalRef(listenerObject); env->DeleteLocalRef(topListener); }这个方法用了很多 JNI 相关的内容,接下来会简单说明。
void JavaHashMapToStlMap(JNIEnv* env, jobject hashMap, std::map<string, string>& mapOut) { // Get the Map's entry Set. jclass mapClass = env->FindClass("java/util/Map"); if (mapClass == NULL) { return; } jmethodID entrySet = env->GetMethodID(mapClass, "entrySet", "()Ljava/util/Set;"); if (entrySet == NULL) { return; } jobject set = env->CallObjectMethod(hashMap, entrySet); if (set == NULL) { return; } // Obtain an iterator over the Set jclass setClass = env->FindClass("java/util/Set"); if (setClass == NULL) { return; } jmethodID iterator = env->GetMethodID(setClass, "iterator", "()Ljava/util/Iterator;"); if (iterator == NULL) { return; } jobject iter = env->CallObjectMethod(set, iterator); if (iter == NULL) { return; } // Get the Iterator method IDs jclass iteratorClass = env->FindClass("java/util/Iterator"); if (iteratorClass == NULL) { return; } jmethodID hasNext = env->GetMethodID(iteratorClass, "hasNext", "()Z"); if (hasNext == NULL) { return; } jmethodID next = env->GetMethodID(iteratorClass, "next", "()Ljava/lang/Object;"); if (next == NULL) { return; } // Get the Entry class method IDs jclass entryClass = env->FindClass("java/util/Map$Entry"); if (entryClass == NULL) { return; } jmethodID getKey = env->GetMethodID(entryClass, "getKey", "()Ljava/lang/Object;"); if (getKey == NULL) { return; } jmethodID getValue = env->GetMethodID(entryClass, "getValue", "()Ljava/lang/Object;"); if (getValue == NULL) { return; } // Iterate over the entry Set while (env->CallBooleanMethod(iter, hasNext)) { jobject entry = env->CallObjectMethod(iter, next); jstring key = (jstring)env->CallObjectMethod(entry, getKey); jstring value = (jstring)env->CallObjectMethod(entry, getValue); const char* keyStr = env->GetStringUTFChars(key, NULL); if (!keyStr) { return; } const char* valueStr = env->GetStringUTFChars(value, NULL); if (!valueStr) { env->ReleaseStringUTFChars(key, keyStr); return; } mapOut.insert(std::make_pair(string(keyStr), string(valueStr))); env->DeleteLocalRef(entry); env->ReleaseStringUTFChars(key, keyStr); env->DeleteLocalRef(key); env->ReleaseStringUTFChars(value, valueStr); env->DeleteLocalRef(value); } }接下来几行,表示将应用层传递的 VideoCaptureListener 在 JNI 层需要获取其类型。然后,查找 VideoCaptureListener 中的几个方法,便于后面调用。这样 JNI 层就可以跟应用层的 Java/Kotlin 进行交互了。
jclass listenerClass = env->GetObjectClass(topListener); jmethodID successId = env->GetMethodID(listenerClass, "onSuccess", "()V"); jmethodID readId = env->GetMethodID(listenerClass, "onRead", "([I)V"); jmethodID failedId = env->GetMethodID(listenerClass, "onFailed", "()V");接下来,开始打开摄像头(相机),并回调给应用层,这样 VideoCaptureListener#onSuccess() 方法就能收到回调。
topVideoCapture = wImageProcess.getVideoCapture(index, mapOut); env->CallVoidMethod(listenerObject, successId);打开摄像头(相机)后,就可以实时把获取的每一帧返回给应用层。同样,VideoCaptureListener#onRead() 方法就能收到回调。
while (topFlag) { topVideoCapture >> topFrame; for (int i = 0;i < topFrame.total();i++) { r = topFrame.data[3 * i + 2]; g = topFrame.data[3 * i + 1]; b = topFrame.data[3 * i + 0]; data[i] = (((jint)r << 16) & 0x00FF0000) + (((jint)g << 8) & 0x0000FF00) + ((jint)b & 0x000000FF); } env->SetIntArrayRegion(jarray, 0, size, (jint*)data); env->CallVoidMethod(listenerObject, readId, jarray); waitKey(100); }后面的代码是关闭相机,释放资源。
topVideoCapture = wImageProcess.getVideoCapture(index, mapOut);它的用途是通过 index id 打开对应的相机,并设置相机需要的参数,最后返回 VideoCapture 对象。
VideoCapture WImageProcess::getVideoCapture(int index, std::map<string, string> cameraParaMap) { VideoCapture capture(index); for (auto & t : cameraParaMap) { int key = stoi(t.first); double value = stod(t.second); capture.set(key, value); } return capture; }对于存在同时调用多个相机的情况,OpenCV 需要基于 index id 来获取对应的相机。那如何获取 index id 呢?以后有机会再写一篇文章吧。
JNIEXPORT jintArray JNICALL Java_xxx_WImagesProcess_takePhoto (JNIEnv* env, jobject, int cameraId) { Mat mat; if (cameraId == 1) { mat = topFrame; } else if (cameraId == 2) { mat = rightFrame; } int* data = new int[mat.total()]; char r, g, b; for (int i = 0;i < mat.total();i++) { r = mat.data[3 * i + 2]; g = mat.data[3 * i + 1]; b = mat.data[3 * i + 0]; data[i] = (((jint)r << 16) & 0x00FF0000) + (((jint)g << 8) & 0x0000FF00) + ((jint)b & 0x000000FF); } jint* _data = (jint*)data; int size = mat.rows * mat.cols; jintArray jarray = env->NewIntArray(size); env->SetIntArrayRegion(jarray, 0, size, _data); delete []data; return jarray; }最后,将 CV 程序和 JNI 相关的代码最终编译成一个 dll 文件,供软件(上位机)调用,实现最终的需求。
val map = HashMap<String,String>() map[CAP_PROP_FRAME_WIDTH] = 4208.toString() map[CAP_PROP_FRAME_HEIGHT] = 3120.toString() map[CAP_PROP_AUTO_EXPOSURE] = 0.25.toString() map[CAP_PROP_EXPOSURE] = getTopExposure() map[CAP_PROP_GAIN] = getTopFocus() map[CAP_PROP_BRIGHTNESS] = getTopBrightness() WImagesProcess.startTopVideoCapture(index + CAP_DSHOW, map, object : VideoCaptureListener { override fun onSuccess() { ...... } override fun onRead(array: IntArray) { ...... } override fun onFailed() { ...... } })应用层的拍照也很简单:
val bufferedImage = WImagesProcess.takePhoto(cameraId).toBufferedImage()
fun IntArray.toBufferedImage():BufferedImage { val destImage = BufferedImage(FRAME_WIDTH,FRAME_HEIGHT, BufferedImage.TYPE_INT_RGB) destImage.setRGB(0,0,FRAME_WIDTH,FRAME_HEIGHT, this,0,FRAME_WIDTH) return destImage }这样,对于应用层的调用是非常简单的。