简单了解iOSCVPixelBuffer（下）

　　1、前言
　　在「简单了解 iOS CVPixelBuffer （中）」中，我们了解了颜色空间 RGB  和 YUV  的区别以及相关的背景知识，最后对 CVPixelBuffer  中的 kCVPixelFormatType  相关类型进行了解读。我们已经对 CVPixelBuffer  有了初步的了解，在这篇文章中，我们将继续聊聊 CVPixelBuffer  在使用过程中的一些格式转换；
　　RGB和YUV格式转换
　　在很多场景下，我们需要将不同的颜色空间进行转换，以此来解决对应的工程性问题。 以下是转换公式： 1.1 YUV -> RGBR = Y + 1.13983 * V G = Y - 0.39465 * U - 0.58060 * V B = Y + 2.03211 * U1.2 RGB -> YUVY = 0.299 * R + 0.587 * G + 0.114 * B U = -0.14713 * R - 0.28886 * G + 0.436 * B V = 0.615 * R - 0.51499 * G - 0.10001 * B2、iOS中常见格式转换
　　在iOS中 RGB  和 YUV  互相转换的方法会使用到 libyuv开源库  ，打开此链接需要梯子，目前国内也有，需要的自取 libyuv开源库·国内仓库
　　iOS在 CVPixelBuffer  转换的上会很复杂，对buffer操作之前需要执行加锁方法 CVPixelBufferLockBaseAddress  进行保护，在处理完后，执行解锁buffer方法 CVPixelBufferUnlockBaseAddress  。
　　以下的方法是我在项目中以及平常的开发中所整理，仅供参考； 2.1 NV12 to I420
　　核心 NV12ToI420  方法是使用了 libyuv开源库  ； /// NV12 to I420 + (CVPixelBufferRef)I420PixelBufferWithNV12:(CVImageBufferRef)cvpixelBufferRef {     CVPixelBufferLockBaseAddress(cvpixelBufferRef, 0);     //图像宽度（像素）     size_t pixelWidth = CVPixelBufferGetWidth(cvpixelBufferRef);     //图像高度（像素）     size_t pixelHeight = CVPixelBufferGetHeight(cvpixelBufferRef);     //获取CVPixelBufferRef中的y数据     const uint8* y_frame = (uint8*)CVPixelBufferGetBaseAddressOfPlane(cvpixelBufferRef,0);     //获取CMVImageBufferRef中的uv数据     const uint8* uv_frame = (uint8*)CVPixelBufferGetBaseAddressOfPlane(cvpixelBufferRef,1);     //y stride     size_t plane1_stride = CVPixelBufferGetBytesPerRowOfPlane (cvpixelBufferRef, 0);     //uv stride     size_t plane2_stride = CVPixelBufferGetBytesPerRowOfPlane (cvpixelBufferRef, 1);     //yuv_size(内存空间)     size_t frame_size = pixelWidth*pixelHeight*3/2;     //开辟frame_size大小的内存空间用于存放转换好的i420数据     uint8* buffer = (unsigned char *)malloc(frame_size);     //buffer为这段内存的首地址,plane1_size代表这一帧中y数据的长度     uint8* dst_u = buffer + pixelWidth*pixelHeight;     //dst_u为u数据的首地,plane1_size/4为u数据的长度     uint8* dst_v = dst_u + pixelWidth*pixelHeight/4;     //libyuv转换     int ret = NV12ToI420(y_frame,                         (int)plane1_stride,                         uv_frame,                         (int)plane2_stride,                         buffer,                         (int)pixelWidth,                         dst_u,                         (int)pixelWidth/2,                         dst_v,                         (int)pixelWidth/2,                         (int)pixelWidth,                         (int)pixelHeight                         );     if (ret) {         return NULL;     }     NSDictionary *pixelAttributes = @{(id)kCVPixelBufferIOSurfacePropertiesKey : @{}};     CVPixelBufferRef pixelBuffer = NULL;     CVReturn result = CVPixelBufferCreate(kCFAllocatorDefault,                                           pixelWidth,                                           pixelHeight,                                           kCVPixelFormatType_420YpCbCr8Planar,                                           (__bridge CFDictionaryRef)(pixelAttributes),                                           &pixelBuffer);     CVPixelBufferLockBaseAddress(pixelBuffer, 0);     size_t d = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);     size_t ud = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);     size_t vd = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 2);     unsigned char* dsty = (unsigned char *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);     unsigned char* dstu = (unsigned char *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1);     unsigned char* dstv = (unsigned char *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 2);     unsigned char* srcy = buffer;     for (unsigned int rIdx = 0; rIdx < pixelHeight; ++rIdx, srcy += pixelWidth, dsty += d) {         memcpy(dsty, srcy, pixelWidth);     }     unsigned char* srcu = buffer + pixelHeight*pixelWidth;     for (unsigned int rIdx = 0; rIdx < pixelHeight/2; ++rIdx, srcu += pixelWidth/2, dstu += ud) {         memcpy(dstu, srcu, pixelWidth/2);     }     unsigned char* srcv = buffer + pixelHeight*pixelWidth*5/4;     for (unsigned int rIdx = 0; rIdx < pixelHeight/2; ++rIdx, srcv += pixelWidth/2, dstv += vd) {         memcpy(dstv, srcv, pixelWidth/2);     }     CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);     if (result != kCVReturnSuccess) {         NSLog(@＂Unable to create cvpixelbuffer %d＂, result);     }     free(buffer); //    CVPixelBufferRelease(cvpixelBufferRef);     return pixelBuffer; }
　　C++音视频学习资料免费获取方法：关注音视频开发T哥  ，点击「链接」即可免费获取2023年最新 C++音视频开发进阶独家免费学习大礼包！ 2.2 NV12 to BGRA
　　核心 NV12ToARGB  方法同样使用了 libyuv开源库  /// NV12 to BGRA + (CVPixelBufferRef)RGBAPixelBufferWithNV12:(CVImageBufferRef)pixelBufferNV12{     CVPixelBufferLockBaseAddress(pixelBufferNV12, 0);     //图像宽度（像素）     size_t pixelWidth = CVPixelBufferGetWidth(pixelBufferNV12);     //图像高度（像素）     size_t pixelHeight = CVPixelBufferGetHeight(pixelBufferNV12);     //y_stride     size_t src_stride_y = CVPixelBufferGetBytesPerRowOfPlane(pixelBufferNV12, 0);     //uv_stride     size_t src_stride_uv = CVPixelBufferGetBytesPerRowOfPlane(pixelBufferNV12,1);     //获取CVImageBufferRef中的y数据     uint8_t *src_y = (unsigned char *)CVPixelBufferGetBaseAddressOfPlane(pixelBufferNV12, 0);     //获取CMVImageBufferRef中的uv数据     uint8_t *src_uv =(unsigned char *) CVPixelBufferGetBaseAddressOfPlane(pixelBufferNV12, 1);     // 创建一个空的32BGRA格式的CVPixelBufferRef     NSDictionary *pixelAttributes = @{(id)kCVPixelBufferIOSurfacePropertiesKey : @{}};     CVPixelBufferRef pixelBufferRGBA = NULL;     CVReturn result = CVPixelBufferCreate(kCFAllocatorDefault,                                          pixelWidth,pixelHeight,kCVPixelFormatType_32BGRA,                                           (__bridge CFDictionaryRef)pixelAttributes,&pixelBufferRGBA);//kCVPixelFormatType_32BGRA                                                if (result != kCVReturnSuccess) {         NSLog(@＂Unable to create cvpixelbuffer %d＂, result);         return NULL;     }     result = CVPixelBufferLockBaseAddress(pixelBufferRGBA, 0);     if (result != kCVReturnSuccess) {         CFRelease(pixelBufferRGBA);         NSLog(@＂Failed to lock base address: %d＂, result);         return NULL;     }     // 得到新创建的CVPixelBufferRef中 rgb数据的首地址     uint8_t *rgb_data = (uint8*)CVPixelBufferGetBaseAddress(pixelBufferRGBA);     // 使用libyuv为rgb_data写入数据，将NV12转换为BGRA     size_t bgraStride = CVPixelBufferGetBytesPerRowOfPlane(pixelBufferRGBA,0);     int ret = NV12ToARGB(src_y, (int)src_stride_y, src_uv, (int)src_stride_uv, rgb_data,(int)bgraStride, (int)pixelWidth, (int)pixelHeight);     if (ret) {         NSLog(@＂Error converting NV12 VideoFrame to BGRA: %d＂, result);         CFRelease(pixelBufferRGBA);         return NULL;     }     CVPixelBufferUnlockBaseAddress(pixelBufferRGBA, 0);     CVPixelBufferUnlockBaseAddress(pixelBufferNV12, 0);     return pixelBufferRGBA; }2.3 CVPixelBufferRef to UIImage
　　以下方法可以将视频帧转成单张图片（比较适用于间隔时间长的截图，高频的使用这个方法很可能会引起内存的问题） /// buffer to image + (UIImage *)convert:(CVPixelBufferRef)pixelBuffer {     CIImage *ciImage = [CIImage imageWithCVPixelBuffer:pixelBuffer];     CIContext *temporaryContext = [CIContext contextWithOptions:nil];     CGImageRef videoImage = [temporaryContext createCGImage:ciImage              fromRect:CGRectMake(0, 0, CVPixelBufferGetWidth(pixelBuffer), CVPixelBufferGetHeight(pixelBuffer))];     UIImage *uiImage = [UIImage imageWithCGImage:videoImage];     CGImageRelease(videoImage);     return uiImage; }2.4 CGImageRef to CVPixelBufferRef
　　以下方法会通过单张图片转成一个PixelBuffer (适用于将某一帧图片转成Buffer添加字幕或者美颜贴纸等等) /// image to buffer + (CVPixelBufferRef)pixelBufferFromCGImage:(CGImageRef)image {     NSDictionary *options = @{                               (NSString*)kCVPixelBufferCGImageCompatibilityKey : @YES,                               (NSString*)kCVPixelBufferCGBitmapContextCompatibilityKey : @YES,                               (NSString*)kCVPixelBufferIOSurfacePropertiesKey: [NSDictionary dictionary]                               };     CVPixelBufferRef pxbuffer = NULL;     CGFloat frameWidth = CGImageGetWidth(image);     CGFloat frameHeight = CGImageGetHeight(image);     CVReturn status = CVPixelBufferCreate(kCFAllocatorDefault,                                           frameWidth,                                           frameHeight,                                           kCVPixelFormatType_32BGRA,                                           (__bridge CFDictionaryRef) options,                                           &pxbuffer);     NSParameterAssert(status == kCVReturnSuccess && pxbuffer != NULL);     CVPixelBufferLockBaseAddress(pxbuffer, 0);     void *pxdata = CVPixelBufferGetBaseAddress(pxbuffer);     NSParameterAssert(pxdata != NULL);     CGColorSpaceRef rgbColorSpace = CGColorSpaceCreateDeviceRGB();     CGContextRef context = CGBitmapContextCreate(pxdata,                                                  frameWidth,                                                  frameHeight,                                                  8,                                                  CVPixelBufferGetBytesPerRow(pxbuffer),                                                  rgbColorSpace,                                                  (CGBitmapInfo)kCGImageAlphaNoneSkipFirst);     NSParameterAssert(context);     CGContextConcatCTM(context, CGAffineTransformIdentity);     CGContextDrawImage(context, CGRectMake(0,                                            0,                                            frameWidth,                                            frameHeight),                        image);     CGColorSpaceRelease(rgbColorSpace);     CGContextRelease(context);     CVPixelBufferUnlockBaseAddress(pxbuffer, 0);     return pxbuffer; }2.5 Buffer Data to UIImage
　　以下方法会通过内存数据转成图片 (根据内存的地址去取出存储的buffer并生成图片，其实这里的内存的地址指向的就是Buffer) // NV12 to image + (UIImage *)YUVtoUIImage:(int)w h:(int)h buffer:(unsigned char *)buffer {     //YUV(NV12)-->CIImage--->UIImage Conversion     NSDictionary *pixelAttributes = @{(NSString*)kCVPixelBufferIOSurfacePropertiesKey:@{}};     CVPixelBufferRef pixelBuffer = NULL;     CVReturn result = CVPixelBufferCreate(kCFAllocatorDefault,                                           w,                                           h,                                           kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange,                                           (__bridge CFDictionaryRef)(pixelAttributes),                                           &pixelBuffer);     CVPixelBufferLockBaseAddress(pixelBuffer,0);     void *yDestPlane = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);     // Here y_ch0 is Y-Plane of YUV(NV12) data.     unsigned char *y_ch0 = buffer;     unsigned char *y_ch1 = buffer + w * h;     memcpy(yDestPlane, y_ch0, w * h);     void *uvDestPlane = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1);     // Here y_ch1 is UV-Plane of YUV(NV12) data.     memcpy(uvDestPlane, y_ch1, w * h * 0.5);     CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);     if (result != kCVReturnSuccess) {         NSLog(@＂Unable to create cvpixelbuffer %d＂, result);     }     // CIImage Conversion     if (@available(iOS 13.0, *)) {         CIImage *coreImage = [CIImage imageWithCVPixelBuffer:pixelBuffer];         CIContext *temporaryContext = [CIContext contextWithOptions:nil];         CGImageRef videoImage = [temporaryContext createCGImage:coreImage                                                            fromRect:CGRectMake(0, 0, w, h)];                  UIImage *finalImage = [[UIImage alloc] initWithCGImage:videoImage];         CVPixelBufferRelease(pixelBuffer);         CGImageRelease(videoImage);         return finalImage;     }     return nil; };2.6 Buffer To NSData+ (NSData *)dataFrompixelBuffer:(CVPixelBufferRef)pixelBuffer {      CVPixelBufferLockBaseAddress(pixelBuffer, 0);     size_t pixelWidth = CVPixelBufferGetWidth(pixelBuffer);     size_t pixelHeight = CVPixelBufferGetHeight(pixelBuffer);     size_t y_size = pixelWidth * pixelHeight;     size_t uv_size = y_size / 2;     uint8_t *yuv_frame = (uint8_t *)malloc(uv_size + y_size);     uint8_t *y_frame = (uint8_t *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);     memcpy(yuv_frame, y_frame, y_size);     uint8_t *uv_frame = (uint8_t *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1);     memcpy(yuv_frame + y_size, uv_frame, uv_size);     CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);     NSData *data = [NSData dataWithBytesNoCopy:yuv_frame length:y_size + uv_size];     return data; }
　　iOS中格式转换涉及到 C、OC、C++  的一些方法，所以很多方法看起来会非常的冗余，需要一定的基础和持续的学习。还有很多是通过OpenGL来绘制图像的方法，更是难得看懂，所以初学者做好笔记，保持耐心，把常用的方法梳理起来，最后封装到工具类中。等到机缘巧合的时候再来深入了解。 3、参考文献
　　一文读懂 YUV 的采样与格式
　　原文链接：简单了解 iOS CVPixelBuffer （下） - 掘金