Windows平台如何实现RTSP拉流添加动态水印|视频处理后转推RTMP或轻量级RTSP服务

技术背景

我们在做Windows平台流数据转发的时候，除了常规的RTSP转RTMP推送外，还有个场景就是，好多开发者希望拉取的RTSP流，做二次视频分析，或者加动态水印等，处理后的数据，再二次编码推送到RTMP服务或轻量级RTSP服务。

技术实现

本文就以Windows平台拉取RTSP流，回调yuv数据到上层，处理后的数据，二次投递到RTMP服务和轻量级RTSP服务，然后叠加动态水印，并实现处理后的数据实时录像功能，废话不多说，先上图：

上图拉取了RTSP流，然后左侧窗体显示，添加动态水印后，再在右侧预览，并把数据重新投递到推送端，考虑到编码性能，我们可选硬编码。

先说RTSP拉流，其他接口不表，这里主要是设置下video frame callback：

video_frame_call_back_ = new SP_SDKVideoFrameCallBack(SetVideoFrameCallBack);
NTSmartPlayerSDK.NT_SP_SetVideoFrameCallBack(player_handle_, (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FROMAT_I420, IntPtr.Zero, video_frame_call_back_);

回调上来的video数据，投递到推送端，当然如果需要二次处理的话，处理后再丢给推送端：

        public void SetVideoFrameCallBack(IntPtr handle, IntPtr userData, UInt32 status, IntPtr frame)
        {
            if (frame == IntPtr.Zero)
                return;

            NT_SP_VideoFrame video_frame = (NT_SP_VideoFrame)Marshal.PtrToStructure(frame, typeof(NT_SP_VideoFrame));

            if (publisher_wrapper_ != null) {
                         publisher_wrapper_.post_i420_layer_image(publisher_wrapper_.get_external_video_layer_index(), video_frame.plane0_, video_frame.stride0_, video_frame.plane1_, video_frame.stride1_,
                            video_frame.plane2_, video_frame.stride2_,
                            video_frame.width_, video_frame.height_);
            }
        }

audio的也是：

audio_pcm_frame_call_back_ = new SP_SDKAudioPCMFrameCallBack(SetAudioPCMFrameCallBack);
NTSmartPlayerSDK.NT_SP_SetAudioPCMFrameCallBack(player_handle_, IntPtr.Zero, audio_pcm_frame_call_back_);

回调上来的audio数据：

        public void SetAudioPCMFrameCallBack(IntPtr handle, IntPtr user_data,
             UInt32 status, IntPtr data, UInt32 size,
             Int32 sample_rate, Int32 channel, Int32 per_channel_sample_number)
        {
            if (data == IntPtr.Zero || size == 0)
                return;

            if (publisher_wrapper_ != null)
                publisher_wrapper_.post_audio_pcm_data(data, size, 0, sample_rate, channel, per_channel_sample_number);
        }

由于处理后的数据，需要重新编码，我们播放端会把原始视频宽高回调上来：

video_size_call_back_ = new SP_SDKVideoSizeCallBack(SP_SDKVideoSizeHandle);
NTSmartPlayerSDK.NT_SP_SetVideoSizeCallBack(player_handle_, IntPtr.Zero, video_size_call_back_);

拿到视频宽高后，我们可以把宽高投递到推送端，便于推送端推送相同分辨率出去，当然，二次编码，也可以设置其他期望的分辨率：

        private void PlaybackWindowResized(Int32 width, Int32 height)
        {
            width_ = width;
            height_ = height;

            if (publisher_wrapper_ != null)
                publisher_wrapper_.SetResolution(width, height);
        }

设置文字水印字体、字号：

        private void btn_set_font_Click(object sender, EventArgs e)
        {
            FontDialog font_dlg = new FontDialog();
            DialogResult result = font_dlg.ShowDialog();

            if (result == DialogResult.OK)
            {
                // 获取用户所选字体
                Font selectedFont = font_dlg.Font;
                btn_set_font.Text = "" + selectedFont.Name + ", " + selectedFont.Size + "pt";

                selected_osd_font_ = new Font(selectedFont.Name, selectedFont.Size, FontStyle.Regular, GraphicsUnit.Point);
            }
        }

如果需要添加文字水印：

        private async void btn_text_osd_Click(object sender, EventArgs e) {
            Bitmap bitmap = null;

            try
            {
                string format = "yyyy-MM-dd HH:mm:ss.fff";

                StringBuilder sb = new StringBuilder();
                sb.Append("施工单位：上海视沃信息科技有限公司(daniusdk.com)");
                sb.Append("\r\n");
                sb.Append("施工时间：");
                sb.Append(DateTime.Now.DayOfWeek.ToString());
                sb.Append(" ");
                sb.Append(DateTime.Now.ToString(format));
                sb.Append("\r\n");
                sb.Append("当前位置：上海市");
                string str = sb.ToString();
                bitmap = GenerateBitmap(str);
            }
            catch (Exception )
            {
                return;
            }

            if (null == bitmap)
                return;

            int x = 0;
            int y = 200;
            UpdateLayerRegion(publisher_wrapper_.get_text_layer_index(), x, y, bitmap);
            publisher_wrapper_.enable_layer(publisher_wrapper_.get_text_layer_index(), true);

            await Task.Delay(30);
            publisher_wrapper_.post_argb8888_layer_image(publisher_wrapper_.get_text_layer_index(), bitmap);
        }

开始推送RTMP：

        private void btn_publish_rtmp_Click(object sender, EventArgs e)
        {
            if (!OpenPublisherHandle())
                return;

            SetCommonOptionToPublisherSDK();

            String url = "rtmp://192.168.0.108:1935/hls/stream1";
            //String url = "rtmp://192.168.2.154:1935/live/stream1";

            if (url.Length < 8)
            {
                publisher_wrapper_.try_close_handle();
                MessageBox.Show("请输入推送地址");
                return;
            }

            if (!publisher_wrapper_.StartPublisher(url))
            {
                MessageBox.Show("调用StartPublisher失败..");
                return;
            }

            btn_publish_rtmp.Enabled = false;
            btn_stop_publish_rtmp.Enabled = true;
        }

publisher_wrapper接口封装：

        public bool StartPublisher(String url)
        {
            if (is_empty_handle() || is_rtmp_publishing())
                return false;

            if (!String.IsNullOrEmpty(url))
                NTSmartPublisherSDK.NT_PB_SetURL(handle_, url, IntPtr.Zero);

            if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_StartPublisher(handle_, IntPtr.Zero))
            {
                try_close_handle();
                return false;
            }

            shared_lock_.EnterWriteLock();
            try
            {
                handle_reference_count_++;
                is_rtmp_publishing_ = true;
            }
            finally
            {
                shared_lock_.ExitWriteLock();
            }

            return true;
        }

        public void StopPublisher()
        {
            if (is_empty_handle() || !is_rtmp_publishing())
                return;

            shared_lock_.EnterWriteLock();
            try
            {
                is_rtmp_publishing_ = false;
                handle_reference_count_--;
            }
            finally
            {
                shared_lock_.ExitWriteLock();
            }

            NTSmartPublisherSDK.NT_PB_StopPublisher(handle_);
            try_close_handle();
        }

对应的SetCommonOptionToPublisherSDK()实现：

        private void SetCommonOptionToPublisherSDK()
        {
            if (publisher_wrapper_.is_empty_handle())
                return;

            if (publisher_wrapper_.handle_reference_count() > 0)
                return;

            publisher_wrapper_.config_layers(true);

            publisher_wrapper_.SetFrameRate((uint)video_fps_);

            int cur_video_codec_id = (int)NTCommonMediaDefine.NT_MEDIA_CODEC_ID.NT_MEDIA_CODEC_ID_H264;

            bool is_h264_encoder = true;

            bool is_hw_encoder = false;

            if (btn_check_video_hardware_encoder_.Checked)
            {
                is_hw_encoder = true;
            }

            Int32 cur_sel_encoder_id = 0;
            Int32 cur_sel_gpu = 0;

            if (is_hw_encoder)
            {
                int cur_sel_hw = combobox_video_encoders_.SelectedIndex;
                if (cur_sel_hw >= 0)
                {
                    cur_sel_encoder_id = Convert.ToInt32(combobox_video_encoders_.SelectedValue);
                    cur_sel_gpu = -1;

                    int cur_sel_hw_dev = combobox_video_hardware_encoder_devices_.SelectedIndex;
                    if (cur_sel_hw_dev >= 0)
                    {
                        cur_sel_gpu = Convert.ToInt32(combobox_video_hardware_encoder_devices_.SelectedValue);
                    }
                }
                else
                {
                    is_hw_encoder = false;
                }
            }

            if (!is_hw_encoder)
            {
                if ((int)NTCommonMediaDefine.NT_MEDIA_CODEC_ID.NT_MEDIA_CODEC_ID_H264 == cur_video_codec_id)
                {
                    cur_sel_encoder_id = btn_check_openh264_encoder_.Checked ? 1 : 0;
                }
            }

            publisher_wrapper_.SetVideoEncoder((int)(is_hw_encoder ? 1 : 0), (int)cur_sel_encoder_id, (uint)cur_video_codec_id, (int)cur_sel_gpu);

            publisher_wrapper_.SetVideoQualityV2(publisher_wrapper_.CalVideoQuality(width_, height_, is_h264_encoder));

            publisher_wrapper_.SetVideoBitRate(publisher_wrapper_.CalBitRate(video_fps_, width_, height_));

            publisher_wrapper_.SetVideoMaxBitRate(publisher_wrapper_.CalMaxKBitRate(video_fps_, width_, height_, false));

            publisher_wrapper_.SetVideoKeyFrameInterval(key_frame_interval_);

            if (is_h264_encoder)
            {
                publisher_wrapper_.SetVideoEncoderProfile(3);
            }

            publisher_wrapper_.SetVideoEncoderSpeed(publisher_wrapper_.CalVideoEncoderSpeed(width_, height_, is_h264_encoder));

            publisher_wrapper_.SetPublisherAudioCodecType(1);   //1: AAC 2: Speex
        }

图层配置实现：

        public bool config_layers(bool is_add_rgbx_zero_layer)
        {
            if (video_option_ != (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER)
                return false;

            if (is_empty_handle())
                return false;

            int w = video_width_;
            int h = video_height_;

            if ((w & 0x1) != 0)
                --w;

            if ((h & 0x1) != 0)
                --h;

            if (w < 2 || h < 2)
                return false;

            NTSmartPublisherSDK.NT_PB_ClearLayersConfig(handle_, 0, 0, IntPtr.Zero);

            int type, index = 0;
            if (is_add_rgbx_zero_layer)
            {
                NT_PB_RGBARectangleLayerConfig rgba_layer = new NT_PB_RGBARectangleLayerConfig();
                type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_RGBA_RECTANGLE;
                fill_layer_base(rgba_layer, out rgba_layer.base_, type, index, true, 0, 0, w, h);
                rgba_layer.red_ = 0;
                rgba_layer.green_ = 0;
                rgba_layer.blue_ = 0;
                rgba_layer.alpha_ = 255;
                if (add_layer_config(rgba_layer, type))
                    index++;
            }

            NT_PB_ExternalVideoFrameLayerConfig external_video_layer = new NT_PB_ExternalVideoFrameLayerConfig();
            type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
            fill_layer_base(external_video_layer, out external_video_layer.base_, type, index, true, 0, 0, w, h);
            if (add_layer_config(external_video_layer, type))
                external_video_layer_index_ = index++;


            //叠加的文本层
            NT_PB_ExternalVideoFrameLayerConfig text_layer = new NT_PB_ExternalVideoFrameLayerConfig();
            type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
            fill_layer_base(text_layer, out text_layer.base_, type, index, false, 0, 0, 64, 64);
            if (add_layer_config(text_layer, type))
                text_layer_index_ = index++;

            return index > 0;
        }

推送端二次录像：

        private void btn_start_recorder_Click(object sender, EventArgs e)
        {
            if (!OpenPublisherHandle())
                return;

           SetCommonOptionToPublisherSDK();

            if (!publisher_wrapper_.StartRecorder())
            {
                MessageBox.Show("调用StartRecorder失败..");
                return;
            }

            btn_start_recorder.Enabled = false;
            btn_stop_recorder.Enabled = true;
        }

        private void btn_pause_recorder_Click(object sender, EventArgs e)
        {
            String btn_pause_rec_text = btn_pause_recorder.Text;

            if ("暂停录像" == btn_pause_rec_text)
            {
                UInt32 ret = publisher_wrapper_.PauseRecorder(true);

                if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
                {
                    MessageBox.Show("暂停录像失败, 请重新尝试!");
                    return;
                }
                else if (NTBaseCodeDefine.NT_ERC_OK == ret)
                {
                    btn_pause_recorder.Text = "恢复录像";
                }
            }
            else
            {
                UInt32 ret = publisher_wrapper_.PauseRecorder(false);
                if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
                {
                    MessageBox.Show("恢复录像失败, 请重新尝试!");
                    return;
                }
                else if (NTBaseCodeDefine.NT_ERC_OK == ret)
                {
                    btn_pause_recorder.Text = "暂停录像";
                }
            }
        }

        private void btn_stop_recorder_Click(object sender, EventArgs e)
        {
            if (publisher_wrapper_.is_recording()) {
                publisher_wrapper_.StopRecorder();

                btn_start_recorder.Enabled = true;
                btn_stop_recorder.Enabled = false;
            }
        }

启动轻量级RTSP服务：

        private void btn_rtsp_service_Click(object sender, EventArgs e)
        {
            if(publisher_wrapper_.IsRTSPSerivceRunning())
            {
                publisher_wrapper_.StopRtspService();
                btn_rtsp_service.Text = "启动RTSP服务";
                btn_rtsp_stream.Enabled = false;
            }
            else
            {
                if(publisher_wrapper_.StartRtspService())
                {
                    btn_rtsp_service.Text = "停止RTSP服务";
                    btn_rtsp_stream.Enabled = true;
                }
            }
        }

发布RTSP流：

       private void btn_rtsp_stream_Click(object sender, EventArgs e)
        {
            if (publisher_wrapper_.is_rtsp_publishing())
            {
                publisher_wrapper_.StopRtspStream();
                btn_rtsp_stream.Text = "发布RTSP流";
                btn_get_rtsp_session_numbers.Enabled = false;
                btn_rtsp_service.Enabled = true;
            }
            else
            {
                if (!OpenPublisherHandle())
                    return;

                SetCommonOptionToPublisherSDK();

                if (!publisher_wrapper_.StartRtspStream())
                {
                    MessageBox.Show("调用StartRtspStream失败..");
                    return;
                }

                btn_rtsp_stream.Text = "停止RTSP流";
                btn_get_rtsp_session_numbers.Enabled = true;
                btn_rtsp_service.Enabled = false;
            }
        }

获取RTSP session会话数：
?

        private void btn_get_rtsp_session_numbers_Click(object sender, EventArgs e)
        {
            if (publisher_wrapper_.is_rtsp_publishing())
            {
               int session_numbers = publisher_wrapper_.GetRtspSessionNumbers();

               MessageBox.Show(session_numbers.ToString(), "当前RTSP连接会话数");
            }
        }

叠加后的数据，本地预览：

       private void btn_preview_Click(object sender, EventArgs e)
        {
            if (publisher_wrapper_.is_previewing())
            {
                publisher_wrapper_.StopPreview();

                btn_preview.Text = "开始预览";
            }
            else
            {
               if (!OpenPublisherHandle())
                   return;

                SetCommonOptionToPublisherSDK();
                       
                if (!publisher_wrapper_.StartPreview())
                {
                    MessageBox.Show("调用StartPreview失败..");
                    return;
                }

                btn_preview.Text = "停止预览";
            }
        }

对应的预览封装实现：

        public bool StartPreview()
        {
            if (is_empty_handle() || is_previewing())
                return false;

            video_preview_image_callback_ = new NT_PB_SDKVideoPreviewImageCallBack(SDKVideoPreviewImageCallBack);
            NTSmartPublisherSDK.NT_PB_SetVideoPreviewImageCallBack(handle_, (int)NTSmartPublisherDefine.NT_PB_E_IMAGE_FORMAT.NT_PB_E_IMAGE_FORMAT_RGB32, IntPtr.Zero, video_preview_image_callback_);

            if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_StartPreview(handle_, 0x800000, IntPtr.Zero))
            {
                try_close_handle();
                return false;
            }

            shared_lock_.EnterWriteLock();
            try
            {
                handle_reference_count_++;
                is_previewing_ = true;
            }
            finally
            {
                shared_lock_.ExitWriteLock();
            }

            return true;
        }

        public void StopPreview()
        {
            if (is_empty_handle() || !is_previewing())
                return;

            shared_lock_.EnterWriteLock();
            try
            {
                is_previewing_ = false;
                handle_reference_count_--;
            }
            finally
            {
                shared_lock_.ExitWriteLock();
            }

            NTSmartPublisherSDK.NT_PB_StopPreview(handle_);
            try_close_handle();

            if (render_wnd_ != null)
                render_wnd_.Invalidate();
        }

预览数据回调：

        //预览数据回调
        public void SDKVideoPreviewImageCallBack(IntPtr handle, IntPtr user_data, IntPtr image)
        {
            NT_PB_Image pb_image = (NT_PB_Image)Marshal.PtrToStructure(image, typeof(NT_PB_Image));

            NT_VideoFrame pVideoFrame = new NT_VideoFrame();

            pVideoFrame.width_ = pb_image.width_;
            pVideoFrame.height_ = pb_image.height_;

            pVideoFrame.stride_ = pb_image.stride_[0];

            Int32 argb_size = pb_image.stride_[0] * pb_image.height_;

            pVideoFrame.plane_ = Marshal.AllocHGlobal(argb_size);

            CopyMemory(pVideoFrame.plane_, pb_image.plane_[0], (UInt32)argb_size);

            if (sync_invoke_ != null)
            {
                System.ComponentModel.ISynchronizeInvoke sync_invoke_target = sync_invoke_.Target as System.ComponentModel.ISynchronizeInvoke;

                if (sync_invoke_target != null)
                {

                    if (sync_invoke_target.InvokeRequired)
                    {
                        sync_invoke_target.BeginInvoke(set_video_preview_image_callback_, new object[] { pVideoFrame });
                    }
                    else
                    {
                        set_video_preview_image_callback_(pVideoFrame);
                    }
                }
            }
        }

总结

以上就是RTSP流二次编辑（如增加动态水印）或视频分析（视觉算法处理）后，再录像、转推至RTMP或轻量级RTSP服务流程，经过二次处理后的流数据，配合我们的SmartPlayer，依然可以整体毫秒级的延迟体验，感兴趣的开发者，可以和我探讨。