返回 登录
2

YUV与RGB之间的转化函数实现

最近在做公司项目时,关于视频流格式之间的转换,YUV与RGB格式之间的转换,在网上查阅了相关的资料,但得到的都不理想,项目需求是从摄像机获取到的视频帧格式是YUV,然后将其转成RGB在Unity中播放,因为我们实现的是人脸识别,跟踪算法所以不能直接使用Unity它自身的WebCam,只能自己去处理,考虑到效率问题,将YUV转成RGB放到GPU中执行。近期在翻看Unity引擎自身的代码时发现其内部是支持的,下面将引擎中的部分代码展示给读者。
视频流其实就是一张一张图片组成的,Unity引擎将其定义为YUVFrame,也是一个结构体,定义如下:

struct YuvFrame
{
    unsigned char* y;
    unsigned char* u;
    unsigned char* v;
    int width;
    int height;
    int y_stride;
    int uv_stride;
    int offset_x;
    int offset_y;
    int uv_step;
};

视频转换函数共实现了三个,函数代码如下所示:

void BaseVideoTexture::YuvToRgb (const YuvFrame *yuv)
{
    #if PROFILE_YUV_CONVERSION
    __int64 time0 = GetCpuTicks();
    #endif
    UInt8 *rgbBuffer = (UInt8*)GetImageBuffer ();
    int const rowBytes = GetRowBytesFromWidthAndFormat(GetPaddedWidth(), GetBufferTextureFormat());

    // Somehow related to audio track being placed into an audio source in the
    // scene with play on load checked causes the first frame decoded to return
    // garbage (with yuv->u set to NULL).
    if ( yuv->u == NULL ) {
        return;
    }

    // NOTE: this code goes backwards in lines, two lines at a time. Thus for
    // odd image sizes it can under-run rgbBuffer. BaseVideoTexture code makes
    // sure there's one line worth of allocated memory before the passed
    // rgbBuffer.

    // get destination buffer (and 1 row offset)
    UInt8* dst0 = rgbBuffer + (yuv->height - 1)*rowBytes;
    UInt8 *dst1 = dst0 - rowBytes;

    // find picture offset
    int yOffset  = yuv->y_stride * yuv->offset_y + yuv->offset_x;
    int uvOffset  = yuv->uv_stride * (yuv->offset_y / 2) + (yuv->offset_x / 2);
    const int uvStep = yuv->uv_step;

    for ( int y = 0; y < yuv->height; y += 2 )
    {
        UInt8 *lineStart = dst1;

        // set pointers into yuv buffers (2 lines for y)
        const UInt8 *pY0 = yuv->y + yOffset + y * (yuv->y_stride);
        const UInt8 *pY1 = yuv->y + yOffset + (y | 1) * (yuv->y_stride);
        const UInt8 *pU = yuv->u + uvOffset + ((y * (yuv->uv_stride)) >> 1);
        const UInt8 *pV = yuv->v + uvOffset + ((y * (yuv->uv_stride)) >> 1);

        for (int x = 0; x < yuv->width; x += 2)
        {
            // convert a 2x2 block over
            const int yy00 = sAdjY[pY0[0]];
            const int yy10 = sAdjY[pY0[1]];
            const int yy01 = sAdjY[pY1[0]];
            const int yy11 = sAdjY[pY1[1]];

            // Compute RGB offsets
            const int vv = *pV;
            const int uu = *pU;
            const int R = sAdjCrr[vv];
            const int G = sAdjCrg[vv] + sAdjCbg[uu];
            const int B = sAdjCbb[uu];

            // pixel 0x0
            dst0++;
            *dst0++ = sClamp[yy00 + R];
            *dst0++ = sClamp[yy00 - G];
            *dst0++ = sClamp[yy00 + B];

            // pixel 1x0
            dst0++;
            *dst0++ = sClamp[yy10 + R];
            *dst0++ = sClamp[yy10 - G];
            *dst0++ = sClamp[yy10 + B];

            // pixel 0x1
            dst1++;
            *dst1++ = sClamp[yy01 + R];
            *dst1++ = sClamp[yy01 - G];
            *dst1++ = sClamp[yy01 + B];

            // pixel 1x1
            dst1++;
            *dst1++ = sClamp[yy11 + R];
            *dst1++ = sClamp[yy11 - G];
            *dst1++ = sClamp[yy11 + B];


            pY0 += 2;
            pY1 += 2;
            pV += uvStep;
            pU += uvStep;
        }

        // shift the destination pointers a row (loop increments 2 at a time)
        dst0 = lineStart - rowBytes;
        dst1 = dst0 - rowBytes;
    }

    #if PROFILE_YUV_CONVERSION
    __int64 time1 = GetCpuTicks();
    {
        __int64 deltaTime = (time1 - time0) / 1000;
        static __int64 accumTime = 0;
        static int counter = 0;
        accumTime += deltaTime;
        ++counter;
        if ( counter == 20 )
        {
            printf_console( "YUV Kclocks per frame: %i\n", (int)(accumTime / counter) );
            counter = 0;
            accumTime = 0;
        }
    }
    #endif
}

第二个函数转换如下:

// LUT-based implementation
void BaseVideoTexture::YUYVToRGBA (UInt16 *const src)
{
    YUYVToRGBA(src, GetPaddedWidth ());
}

最后一个函数如下所示:

void BaseVideoTexture::YUYVToRGBA (UInt16 *const src, int srcStride)
{
    #if PROFILE_YUV_CONVERSION
    __int64 time0 = GetCpuTicks();
    #endif

    UInt16 *srcYUYV = src;
    UInt8 *destRGBA = reinterpret_cast<UInt8*> (GetImageBuffer ());
    int const destStride = GetRowBytesFromWidthAndFormat(GetPaddedWidth(), GetBufferTextureFormat());
    int const widthInPixels  = GetDataWidth ();
    int const heightInPixels = GetDataHeight ();
    int y0;
    int u;
    int y1;
    int v;
    int red;
    int green;
    int blue;

    destRGBA += (heightInPixels - 1) * destStride;

    // Lines within the destination rectangle.
    for (int y = 0; y < heightInPixels; ++y)
    {
        UInt8 *srcPixel = reinterpret_cast<UInt8*> (srcYUYV);

        // Increment by widthInPixels does not necessarily
        // mean that dstPixel is incremented by the stride, 
        // so we keep a separate pointer.
        UInt8 *dstPixel = destRGBA;

        for (int x = 0; (x + 1) < widthInPixels; x += 2)
        {
            // Byte order is Y0 U0 Y1 V0
            // Each word is a byte pair (Y, U/V)
            y0 = sAdjY[*srcPixel++];
            u = *srcPixel++;
            y1 = sAdjY[*srcPixel++];
            v = *srcPixel++;
            red = sAdjCrr[v];
            green = sAdjCrg[v] + sAdjCbg[u];
            blue = sAdjCbb[u];

            *dstPixel++ = 0xFF;
            *dstPixel++ = sClamp[y0 +  red];
            *dstPixel++ = sClamp[y0 - green];
            *dstPixel++ = sClamp[y0 + blue];
            *dstPixel++ = 0xFF;
            *dstPixel++ = sClamp[y1 + red];
            *dstPixel++ = sClamp[y1 - green];
            *dstPixel++ = sClamp[y1 + blue];
        }
        destRGBA -= destStride;
        srcYUYV += srcStride;
    }

    #if PROFILE_YUV_CONVERSION
    __int64 time1 = GetCpuTicks();
    {
        __int64 deltaTime = (time1 - time0) / 1000;
        static __int64 accumTime = 0;
        static int counter = 0;
        accumTime += deltaTime;
        ++counter;
        if ( counter == 20 )
        {
            printf_console( "YUYV Kclocks per frame: %i\n", (int)(accumTime / counter) );
            counter = 0;
            accumTime = 0;
        }
    }
    #endif
}

以上实现的是三个转换函数,下面再补充几个函数和枚举定义,纹理格式的定义这个可以在C#中查到,在Unity引擎内部也定义了如下所示:

typedef UInt32 TextureFormat;
enum
{
    kTexFormatAlpha8 = 1,
    kTexFormatARGB4444 = 2,
    kTexFormatRGB24 = 3,
    kTexFormatRGBA32 = 4,
    kTexFormatARGB32 = 5,
    kTexFormatARGBFloat = 6, // only for internal use at runtime
    kTexFormatRGB565 = 7,
    kTexFormatBGR24 = 8,
    // This one is for internal use; storage is 16 bits/pixel; samples
    // as Alpha (OpenGL) or RGB (D3D9). Can be reduced to 8 bit alpha/luminance on lower hardware.
    // Why it's not Luminance on GL: for some reason alpha seems to be faster.
    kTexFormatAlphaLum16 = 9,
    kTexFormatDXT1 = 10,
    kTexFormatDXT3 = 11,
    kTexFormatDXT5 = 12,
    kTexFormatRGBA4444 = 13,

    kTexFormatPCCount = 14,

    kTexReserved1 = 14, // Use reservedX when adding a new 'PC' texture format
    kTexReserved2 = 15,
    kTexReserved3 = 16,
    kTexReserved4 = 17,
    kTexReserved5 = 18,
    kTexReserved6 = 19,
    // [20..27] used to be Wii-specific formats before Unity 4.0
    kTexReserved11 = 28,
    kTexReserved12 = 29,

    // iPhone
    kTexFormatPVRTC_RGB2 = 30,
    kTexFormatPVRTC_RGBA2 = 31,

    kTexFormatPVRTC_RGB4 = 32,
    kTexFormatPVRTC_RGBA4 = 33,

    kTexFormatETC_RGB4 = 34,

    kTexFormatATC_RGB4 = 35,
    kTexFormatATC_RGBA8 = 36,

    // Pixels returned by iPhone camera
    kTexFormatBGRA32 = 37,

    kTexFormatFlashATF_RGB_DXT1 = 38,
    kTexFormatFlashATF_RGBA_JPG = 39,
    kTexFormatFlashATF_RGB_JPG = 40,

    // EAC and ETC2 compressed formats, mandated by OpenGL ES 3.0
    kTexFormatEAC_R = 41,
    kTexFormatEAC_R_SIGNED = 42,
    kTexFormatEAC_RG = 43,
    kTexFormatEAC_RG_SIGNED = 44,
    kTexFormatETC2_RGB = 45,
    kTexFormatETC2_RGBA1 = 46,
    kTexFormatETC2_RGBA8 = 47,

    // ASTC. The RGB and RGBA formats are internally identical, we just need to carry the has-alpha information somehow
    kTexFormatASTC_RGB_4x4 = 48,
    kTexFormatASTC_RGB_5x5 = 49,
    kTexFormatASTC_RGB_6x6 = 50,
    kTexFormatASTC_RGB_8x8 = 51,
    kTexFormatASTC_RGB_10x10 = 52,
    kTexFormatASTC_RGB_12x12 = 53,

    kTexFormatASTC_RGBA_4x4 = 54,
    kTexFormatASTC_RGBA_5x5 = 55,
    kTexFormatASTC_RGBA_6x6 = 56,
    kTexFormatASTC_RGBA_8x8 = 57,
    kTexFormatASTC_RGBA_10x10 = 58,
    kTexFormatASTC_RGBA_12x12 = 59,

    kTexFormatTotalCount    = 60 // keep this last!
};

另外在上述三个转换函数中内部封装了几个接口,在这里为了显示其完整性也给读者介绍一下:

    int GetPaddedHeight() const {return m_PaddedHeight; }
    int GetPaddedWidth() const {return m_PaddedWidth; }
    virtual TextureFormat   GetBufferTextureFormat() const      { return kTexFormatARGB32; }

最后一个接口函数代码如下所示:

UInt32 GetBytesFromTextureFormat (TextureFormat inFormat)
{
    AssertMsg (inFormat < kTexFormatDXT1 || inFormat == kTexFormatBGRA32 || inFormat == kTexFormatRGBA4444, "Invalid texture format: %d", (int)inFormat);
    return (inFormat == kTexFormatBGRA32) ? 4 : kTextureByteTable[inFormat];
}

UInt32 GetMaxBytesPerPixel (TextureFormat inFormat)
{
    Assert (GetBytesFromTextureFormat (inFormat) <= kTextureByteTable[kTexFormatARGBFloat]);
    return kTextureByteTable[kTexFormatARGBFloat];
}

int GetRowBytesFromWidthAndFormat (int width, TextureFormat inFormat)
{
    AssertMsg (inFormat < kTexFormatDXT1 || inFormat == kTexFormatBGRA32 || inFormat == kTexFormatRGBA4444, "Invalid texture format: %d", (int)inFormat);
    return GetBytesFromTextureFormat (inFormat) * width;
}

这样YUV转RGB转换接口就完整了,希望对大家有所帮助。

笔者简介:姜雪伟个人主页

评论