1. 前言
尝试在xavier nx平台做图像处理应用 应用摘要如下
1.捕捉摄像头设备 2.颜色转换格式 3.图像处理 4.编码与HW编码器1~3使用OpenCV实现 我想实现的是通过图像处理的结果(例如。覆盖图像与边框)到HW编码器。 我引用了jetson_multimedia_api_reference，但是混淆了
HW: JetsonNX开发工具包，SW: Jetpack4.4
2. 参考python示例 & patch import sys import cv2 def read_cam(): cap = cv2.VideoCapture(“filesrc location=/home/nvidia/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! nvvidconv ! video/x-raw, format=BGRx ! videoconvert ! video/x-raw,format=BGR ! appsink “) w = cap.get(cv2.CAP_PROP_FRAME_WIDTH) h = cap.get(cv2.CAP_PROP_FRAME_HEIGHT) fps = cap.get(cv2.CAP_PROP_FPS) print(Src opened, %dx%d @ %d fps % (w, h, fps)) gst_out = “appsrc ! video/x-raw, format=BGR ! queue ! videoconvert ! video/x-raw,format=BGRx ! nvvidconv ! nvv4l2h264enc ! h264parse ! matroskamux ! filesink location=test.mkv ” out = cv2.VideoWriter(gst_out, cv2.CAP_GSTREAMER, 0, float(fps), (int(w), int(h))) if not out.isOpened(): print(“Failed to open output”) exit() if cap.isOpened(): while True: ret_val, img = cap.read(); if not ret_val: break; out.write(img); cv2.waitKey(1) else: print “pipeline open failed” print(“successfully exit”) cap.release() out.release() if __name__ == __main__: read_cam()<
但现在需要拿编码器输出的数据进行后期处理 考虑使用多媒体api
参考以下patch
diff –git a/multimedia_api/ll_samples/samples/13_multi_camera/main.cpp b/multimedia_api/ll_samples/samples/13_multi_camera/main.cpp index 49a9ab8..0613f0b 100644 — a/multimedia_api/ll_samples/samples/13_multi_camera/main.cpp +++ b/multimedia_api/ll_samples/samples/13_multi_camera/main.cpp @@ -39,6 +39,8 @@ #include <stdio.h> #include <stdlib.h> +#include <opencv2/opencv.hpp> + using namespace Argus; using namespace EGLStream; @@ -259,7 +261,7 @@ bool ConsumerThread::threadInitialize() input_params.width = STREAM_SIZE.width(); input_params.height = STREAM_SIZE.height(); input_params.layout = NvBufferLayout_Pitch; – input_params.colorFormat = NvBufferColorFormat_NV12; + input_params.colorFormat = NvBufferColorFormat_ABGR32; input_params.nvbuf_tag = NvBufferTag_VIDEO_CONVERT; NvBufferCreateEx (&m_compositedFrame, &input_params); @@ -333,8 +335,8 @@ bool ConsumerThread::threadExecute() if (!m_dmabufs[i]) { m_dmabufs[i] = iNativeBuffer->createNvBuffer(iEglOutputStreams[i]->getResolution(), – NvBufferColorFormat_YUV420, – NvBufferLayout_BlockLinear); + NvBufferColorFormat_ABGR32, + NvBufferLayout_Pitch); if (!m_dmabufs[i]) CONSUMER_PRINT(“\tFailed to create NvBuffer\n”); } @@ -349,7 +351,21 @@ bool ConsumerThread::threadExecute() { // Composite multiple input to one frame NvBufferComposite(m_dmabufs, m_compositedFrame, &m_compositeParam); – g_renderer->render(m_compositedFrame); + void *pdata = NULL; + + NvBufferMemMap(m_compositedFrame, 0, NvBufferMem_Read, &pdata); + NvBufferMemSyncForCpu(m_compositedFrame, 0, &pdata); + + cv::Mat imgbuf = cv::Mat(STREAM_SIZE.height(), + STREAM_SIZE.width(), + CV_8UC4, pdata); + cv::Mat display_img; + cvtColor(imgbuf, display_img, CV_RGBA2BGR); + + NvBufferMemUnMap(m_compositedFrame, 0, &pdata); + + cv::imshow(“img”, display_img); + cv::waitKey(1); } else g_renderer->render(m_dmabufs[0]);<
Makefile:
+CPPFLAGS+=`pkg-config –cflags opencv` +LDFLAGS+=`pkg-config –libs opencv` 3. OpenCV安装方法 1不要通过Jetpack安装OpenCV 3.3.1。默认安装。请取消勾选OpenCV 3.3.1 2 Jetpack将禁止安装样例包，如果你不选中OpenCV，请从 https://developer.nvidia.com/embedded/dlc/multimedia-api-r2821 2 3获取脚本https://github.com/AastaNV/JEP/blob/master/script/install_opencv3.4.0_TX2.sh 4执行脚本 mkdir OpenCV $ ./install_opencv3.4.0_TX2.sh OpenCV 5应用补丁并重建09_camera_jpeg_capture 6运行 $ export DISPLAY=:0 09_camera_jpeg_capture$ ./camera_jpeg_capture –disable-jpg –cap-time 10 4. V4L2架构参考sample参考12_camera_v4l2_cuda。 该示例用于v4l2采集
5. gstreamer示例在opencv中，可以像VideoWriter一样使用gstreamer管道
/ Get resolution and framerate from capture unsigned int width = cap.get (cv::CAP_PROP_FRAME_WIDTH); unsigned int height = cap.get (cv::CAP_PROP_FRAME_HEIGHT); unsigned int fps = cap.get (cv::CAP_PROP_FPS); // Create the writer with gstreamer pipeline encoding into H264, muxing into mkv container and saving to file cv::VideoWriter gst_nvh264_writer(“appsrc ! queue ! videoconvert ! video/x-raw,format=BGRx ! nvvidconv ! nvv4l2h264enc ! video/x-h264,format=byte-stream ! h264parse ! matroskamux ! filesink location=test-nvh264-writer.mkv “, 0, fps, cv::Size (width, height)); if (!gst_nvh264_writer.isOpened ()) { std::cout << “Failed to open gst_nvh264 writer.” << std::endl; return (-6); }在循环中，使用以下命令推送处理过的帧(每个捕获帧一个):
gst_nvh264_writer.write(frame); 6. NvVideoEncoder考虑在MMAPI中使用NvVideoEncoder 需要创建NvBuffer和复制cv::Mat数据到它。
方法如下:
1. 创建导入cv::Mat(ARGB32)的NvBuffer 2. 为YUV420M创建匹配编码器 输入格式的NvBuffer。 3. 创建NvVideoEncoder 4. 编码器输出平面缓冲区的出队列 5. 将ARGB转换为YUV420M尝试将cv::Mat转换成NvBuffer 如果par.num_planes = 1，这是正确的 但这种方法似乎是无效的
cv::Mat src = cv::Mat::zeros(height, width, CV_8UC4); ret = NvBufferGetParams(fd, par); //fd is V4L2_PIX_FMT_ABGR32 for( unsigned int plane = 0; plane < par.num_planes; plane++){ ret = NvBufferMemMap(fd, plane, NvBufferMem_Write, &vaddr); // vaddr is void* if(ret == 0){ for( unsigned int i = 0; i < par.height[plane]; i++){ memcpy((uint8_t *)vaddr + i *par.pitch[plane], src.data[ i * src.step], src.width * sizeof((uint8_t)) * 4); } } NvBufferMemSyncForDevice (fd, plane, &vaddr); } 7. CUDA过滤器因为OpenCV的主要格式是BGR，在Jetson平台上并不支持BGR。 在处理之后，需要将其转换为RGBA，复制到NvBuffer，转换到YUV420并进行编码。 它可能不会带来好的性能。
有CUDA过滤器可以应用于RGBA缓冲区。 如果能在用例中使用CUDA过滤器，性能会更好
函数调用如下:
//CUDA postprocess { EGLImageKHR egl_image; egl_image = NvEGLImageFromFd(egl_display, dmabuf_fd); CUresult status; CUeglFrame eglFrame; CUgraphicsResource pResource = NULL; cudaFree(0); status = cuGraphicsEGLRegisterImage(&pResource, egl_image, CU_GRAPHICS_MAP_RESOURCE_FLAGS_NONE); if (status != CUDA_SUCCESS) { printf(“cuGraphicsEGLRegisterImage failed: %d \n”,status); } status = cuGraphicsResourceGetMappedEglFrame(&eglFrame, pResource, 0, 0); status = cuCtxSynchronize(); if (create_filter) { filter = cv::cuda::createSobelFilter(CV_8UC4, CV_8UC4, 1, 0, 3, 1, cv::BORDER_DEFAULT); //filter = cv::cuda::createGaussianFilter(CV_8UC4, CV_8UC4, cv::Size(31,31), 0, 0, cv::BORDER_DEFAULT); create_filter = false; } cv::cuda::GpuMat d_mat(h, w, CV_8UC4, eglFrame.frame.pPitch[0]); filter->apply (d_mat, d_mat); status = cuCtxSynchronize(); status = cuGraphicsUnregisterResource(pResource); NvDestroyEGLImage(egl_display, egl_image); }<
dmabuf_fd是RGBA格式。 经过处理后，可以将其转换回NV12并发送给硬件编码器。
