rtsp视频在使用unity三维融合播放后的修正

1 rtsp 接入

我们使用unity UE 等三维渲染引擎中使用c++编写插件来接入rtsp 视频。同时做融合的时候，和背景的三维颜色要一致，这就要使用视频融合修正技术。包括亮度，对比度，饱和度的修正。在单纯颜色上的修正可以简单使用rgb->hsv去修改，这里不做累赘说明了。

2 播放技术

使用unity 的纹理渲染来播放多路 视频，视频接入最大可达到30路，同时在untiy ， UE5 中渲染。播放后有很多视频的颜色不一致，如何调整颜色成了一个问题

3 untiy 渲染过程

建立一个材质，如命名为m2

将m2 赋值给我们的面片，如pp

接下来创建一个shader，控制渲染的亮度，对比度，饱和度，分别是 Brightness, Saturation, Contrast, unity shader的基础不再啰嗦，各位可以稍稍看一下，不难。

Shader "self/BrightnessSaturationAndContrast"
{
    Properties
    {
        _MainTex ("Base(RGB", 2D) = "white" {}
        //从脚本传递更好，这里可以直接省略这些值的展示
        _Brightness ("Brightness", float) = 1
        _Saturation ("Saturation", float) = 1
        _Contrast ("Contrast", float) = 1
    }
    SubShader
    {
        Pass
        {
            //关闭深度写入
            //ZTest Always Cull Off Zwrite Off

            CGPROGRAM

            #pragma vertex vert
            #pragma fragment frag
            #include "UnityCG.cginc"
            
            //properties
            sampler2D _MainTex;
            half _Brightness;
            half _Saturation;
            half _Contrast;

            struct v2f {
                float4 pos : SV_POSITION;
                half2 uv : TEXCOORD0; 
            };

            //使用了内置的appdata_img结构体作为顶点着色器的输入
            v2f vert(appdata_img  v) {
                v2f o;
                o.pos = UnityObjectToClipPos(v.vertex);
                o.uv = v.texcoord;
                return o;
            }

            fixed4 frag(v2f i) : SV_Target {
                //获得屏幕图像的采样
                fixed4 renderTex = tex2D(_MainTex, i.uv);

                //亮度
                fixed3 finalColor = renderTex.rgb * _Brightness;

                //饱和度
                fixed luminance = 0.2125 * renderTex.r + 0.7154 * renderTex.g + 0.0721 * renderTex.b;  //计算该像素的亮度值
                fixed3 luminanceColor = fixed3(luminance, luminance, luminance);  //创建饱和度为0的颜色
                finalColor = lerp(luminanceColor, finalColor, _Saturation);

                //contrast
                fixed3 avgColor = fixed3(0.5, 0.5, 0.5);
                finalColor = lerp(avgColor, finalColor, _Contrast);

                return fixed4(finalColor, renderTex.a);
            }
            ENDCG
        }
    }
    FallBack  Off
}

写好shader以后， 写一个脚本，调入我们的c#脚本，脚本一为控制我们的c++插件，二是可以修改,以下的插件脚本负责调入c++ 的插件，插件会在线程里面启动，获取图像，赋值给面片。插件本身可以接入实时视频，播放视频，倒放视频，支持国标gb28181和onvif协议，可以支持矫正视频，支持多组矫正参数，同时增加三个变量，改变亮度，对比度，饱和度，便于在里面实时修改观察。

//author: 钱波
using System;
using System.Text;
using System.Threading;
using System.Collections;
using UnityEngine;
using System.Runtime.InteropServices;
using UnityEngine.UI;
using System.Linq;
using System.Collections.Generic;
using Unity.VisualScripting;
using System.IO;


[StructLayout(LayoutKind.Sequential)]
public struct FRAME
{
    public int width;
    public int height;
    public int len;
    public IntPtr Frame;
    public IntPtr data;
    //public byte[] data;
    //[MarshalAs(UnmanagedType.LPArray)]
}
[StructLayout(LayoutKind.Sequential)]
public struct PARAM
{
    public double p1;
    public double p2;
    public double p3;
    public double p4;
    public double p5;
    public double p6;
    public double p7;
    public double p8;
    public double p9;
    public double c1;
    public double c2;
    public double c3;
    public double c4;
    public double c5;
}


public class rtspin : MonoBehaviour
{
    [DllImport("rtspPlugin")]
    public static extern bool rtsp_test([MarshalAs(UnmanagedType.LPStr)] string url, int isnv12);

    [DllImport("rtspPlugin")]
    public static extern bool rtsp_test_data([MarshalAs(UnmanagedType.LPStr)] string url,
        [MarshalAs(UnmanagedType.LPArray)] byte[] data, ref FRAME frame);

    [DllImport("rtspPlugin")]
    public static extern bool rtsp_test_data_nv12([MarshalAs(UnmanagedType.LPStr)] string url,
        [MarshalAs(UnmanagedType.LPArray)] byte[] data0, [MarshalAs(UnmanagedType.LPArray)] byte[] data1);

    [DllImport("rtspPlugin")]
    public static extern void rtsp_test_stop([MarshalAs(UnmanagedType.LPStr)] string url);

    [DllImport("rtspPlugin")]
    public static extern void rtsp_test_setparam([MarshalAs(UnmanagedType.LPStr)] string url, ref PARAM param);


    static string[] stringArray = {"rtsp://127.0.0.1/front.mkv" };
    static string[] stringPlanes = { "pp"};

    //Texture2D[] texture2Ds;
    int w1 = 1920;
    int h1 = 1080;
    int number = 1;


    [Range(0.0f, 3.0f)]
    public float brightness = 1.0f;

    [Range(0.0f, 3.0f)]
    public float saturation = 1.0f;

    [Range(0.0f, 3.0f)]
    public float contrast = 1.0f;
    public Material briSatConMaterial;
    class cmd
    {
        public int v_w1 = 1920;
        public int v_h1 = 1080;
        public byte[] v_data;
        public Texture2D v_texture2Ds;
        public cmd()
        {
            v_data = new byte[v_w1 * v_h1 * 3];
            v_texture2Ds = new Texture2D(v_w1, v_h1, TextureFormat.RGB24, false);//RGB24
        }
    }


    PARAM v_param;
    //byte[][] v_datas;
    Dictionary<int, cmd> hashMap_datas = new Dictionary<int, cmd>();
    //byte[][] hashMap_datas;


    void rtspThreading(string url)
    {
        Debug.Log(url);
        rtsp_test(url, 1);
    }


    void Start()
    {

        //string path = Application.dataPath + "/rtsp.txt";

        //path = path.Replace("/", "\\");
        //if (File.Exists(path))
        //{
        //    Debug.Log("FileExists");
        //    stringArray = File.ReadAllLines(path);
        //}
        //else
        //{
        //    Debug.Log("FileNotExists");
        //    File.CreateText(path);
        //}

        stringArray[0] = "rtsp://127.0.0.1/front.mkv";
        for (int i = 0; i < number; i++)
        {
            cmd c1 = new cmd();
            hashMap_datas.Add(i, c1);
        }
        v_param = new PARAM();

        //v_param.p1 = 6.5746697810243404e+002;
        //v_param.p2 = 0.0;
        //v_param.p3 = 3.1950000000000000e+002;
        //v_param.p4 = 0.0;
        //v_param.p5 = 6.5746697810243404e+002;
        //v_param.p6 = 2.3950000000000000e+002;
        //v_param.p7 = 0.0;
        //v_param.p8 = 0.0;
        //v_param.p9 = 1.0;
        //v_param.c1 = -0.5180232701824102559;
        //v_param.c2 = 0.5071524380583312119;
        //v_param.c3 = 0.0;
        //v_param.c4 = 0.0;
        //v_param.c5 = -0.5784359684793970446;

        //1281.48 0 975.5 0 1997.48 0 0 0 1 -0.6 0.4 0.1 0 -0.198
        v_param.p1 = 1281.48;
        v_param.p2 = 0.0;
        v_param.p3 = 975.5;
        v_param.p4 = 0.0;
        v_param.p5 = 1997.48;
        v_param.p6 = 0.0;
        v_param.p7 = 0.0;
        v_param.p8 = 0.0;
        v_param.p9 = 1.0;
        v_param.c1 = -0.6;
        v_param.c2 = 0.4;
        v_param.c3 = 0.1;
        v_param.c4 = 0.0;
        v_param.c5 = -0.198;


        GameObject go = GameObject.Find("pp");
        briSatConMaterial = go.GetComponent<MeshRenderer>().material;
    }
    private void OnGUI()
    {

        if (GUI.Button(new Rect(120, 10, 80, 30), "开始线程"))
        {
            Debug.Log("开始rtsp......");

            for (int i = 0; i < number; i++)
            {
                int currentIndex = i;
                Thread rtspthread1 = new Thread(() => rtspThreading(stringArray[currentIndex]));
                rtspthread1.Start();
                Thread.Sleep(1);
            }

        }

        //绘制按钮，以及按下断开连接按钮，发送断开连接请求
        if (GUI.Button(new Rect(210, 10, 80, 30), "结束线程"))
        {
            Debug.Log("结束rtsp......");
            for (int i = 0; i < number; i++)
            {
                int currentIndex = i;
                rtsp_test_stop(stringArray[currentIndex]);
            }
        }

    }


    //float delta_x, delta_y, delta_z;            //计算移动量
    //float distance = 5;
    //float ZoomSpeed = 5f;                  //拉近拉远速度
    //public bool isFar = true;
    void Update()
    {
        FRAME frame = new FRAME();
        if (briSatConMaterial != null)
        {
            briSatConMaterial.SetFloat("_Brightness", brightness);
            briSatConMaterial.SetFloat("_Saturation", saturation);
            briSatConMaterial.SetFloat("_Contrast", contrast);
        }
        for (int i = 0; i < number; i++)
        {
            if (rtsp_test_data(stringArray[i], hashMap_datas[i].v_data, ref frame))
            {
                 rtsp_test_setparam(stringArray[i], ref v_param);


                //Texture2D original = new Texture2D(w1, h1, TextureFormat.RGB24, false);
                //original.LoadRawTextureData(hashMap_datas[i].v_data);
                //FlipTexture(original, hashMap_datas[i].v_texture2Ds);
                hashMap_datas[i].v_texture2Ds.LoadRawTextureData(hashMap_datas[i].v_data);
                hashMap_datas[i].v_texture2Ds.Apply();
                GameObject go = GameObject.Find(stringPlanes[i]);
                go.GetComponent<MeshRenderer>().material.mainTexture = hashMap_datas[i].v_texture2Ds;
            }
        }
    }

    void OnDestroy()
    {
        Debug.Log("Destory, 结束rtsp......");
        for (int i = 0; i < number; i++)
        {
            int currentIndex = i;
            rtsp_test_stop(stringArray[currentIndex]);
        }
    }
}

结果

点击开始播放，rtsp线程开始播放，inspector里面可以调整三个值，可以直观地看到视频播放的亮度和对比度饱和度的改变。

修改一下亮度

同时修改三个参数的变化