diff --git a/JianGongYun/JianGongYun.csproj b/JianGongYun/JianGongYun.csproj index 55aa31d..f189d0c 100644 --- a/JianGongYun/JianGongYun.csproj +++ b/JianGongYun/JianGongYun.csproj @@ -56,6 +56,6 @@ - + diff --git a/JianGongYun/TRTC/Components/TXLiteAVVideoView.cs b/JianGongYun/TRTC/Components/TXLiteAVVideoView.cs index c2d7000..e2c5275 100644 --- a/JianGongYun/TRTC/Components/TXLiteAVVideoView.cs +++ b/JianGongYun/TRTC/Components/TXLiteAVVideoView.cs @@ -163,6 +163,9 @@ namespace JianGongYun.TRTC.Components { if (!mFirstFrame) mFirstFrame = true; + + OnRenderVideoFrameHandler?.Invoke(data, width, height); + if (mPause) return false; if (data == null || data.Length <= 0) @@ -237,7 +240,6 @@ namespace JianGongYun.TRTC.Components private void RenderFillMode(DrawingContext dc, byte[] data, int width, int height, int rotation) { - OnRenderVideoFrameHandler?.Invoke(data, width, height); int viewWidth = (int)this.ActualWidth, viewHeight = (int)this.ActualHeight; PixelFormat pixelFormat = PixelFormats.Pbgra32; int bytesPerPixel = (pixelFormat.BitsPerPixel + 7) / 8; @@ -271,7 +273,6 @@ namespace JianGongYun.TRTC.Components private void RenderFitMode(DrawingContext dc, byte[] data, int width, int height, int rotation) { - OnRenderVideoFrameHandler?.Invoke(data, width, height); int viewWidth = (int)this.ActualWidth, viewHeight = (int)this.ActualHeight; PixelFormat pixelFormat = PixelFormats.Pbgra32; int bytesPerPixel = (pixelFormat.BitsPerPixel + 7) / 8; diff --git a/JianGongYun/TRTC/LiveClassroom.cs b/JianGongYun/TRTC/LiveClassroom.cs index 2bc38f4..6b610de 100644 --- a/JianGongYun/TRTC/LiveClassroom.cs +++ b/JianGongYun/TRTC/LiveClassroom.cs @@ -99,7 +99,6 @@ namespace JianGongYun.TRTC if (!string.IsNullOrEmpty(currentMic)) { var res = lTXDeviceManager.setCurrentDevice(TRTCDeviceType.TXMediaDeviceTypeMic, currentMic); - Console.WriteLine($"设置麦克风：{res}"); } lTXDeviceManager.setCurrentDeviceVolume(TRTCDeviceType.TXMediaDeviceTypeMic, settingWindowViewModel.MicVolume);//麦克风采集音量 lTRTCCloud.setSystemAudioLoopbackVolume(settingWindowViewModel.SytemGatherVolume);//系统声音采集音量 @@ -165,11 +164,8 @@ namespace JianGongYun.TRTC var view = AddCustomVideoView(parent, CurrentClassroomEntity.TeacherId, TRTCVideoStreamType.TRTCVideoStreamTypeBig, true); if (liveWinMode.IsLive) { - //Stopwatch sw = new Stopwatch(); - //VideoRecordTask(view, TRTCVideoStreamType.TRTCVideoStreamTypeBig); view.OnRenderVideoFrameHandler += (data, w, h) => { - //sw.Restart(); lock (MainFrame) { if (MainFrame.Cols != w || MainFrame.Rows != h) @@ -178,8 +174,6 @@ namespace JianGongYun.TRTC } Marshal.Copy(data, 0, MainFrame.Data, data.Length); } - //sw.Stop(); - //Debug.Print("main" + sw.ElapsedMilliseconds.ToString()); }; } return view; @@ -221,11 +215,8 @@ namespace JianGongYun.TRTC var view = AddCustomVideoView(parent, CurrentClassroomEntity.TeacherId, TRTCVideoStreamType.TRTCVideoStreamTypeSub, true); if (liveWinMode.IsLive) { - //Stopwatch sw = new Stopwatch(); - //VideoRecordTask(view, TRTCVideoStreamType.TRTCVideoStreamTypeSub); view.OnRenderVideoFrameHandler += (data, w, h) => { - //sw.Restart(); lock (SubFrame) { if (SubFrame.Cols != w || SubFrame.Rows != h) @@ -234,8 +225,6 @@ namespace JianGongYun.TRTC } Marshal.Copy(data, 0, SubFrame.Data, data.Length); } - //sw.Stop(); - //Debug.Print("sub" + sw.ElapsedMilliseconds.ToString()); }; } return view; @@ -280,25 +269,27 @@ namespace JianGongYun.TRTC { liveWinMode.MicRunning = true; lTRTCCloud.startLocalAudio(settingWindowViewModel.LiveAudioLevel); - //if (liveWinMode.IsLive && !liveWinMode.AudioRecordRunning) - //{ - // liveWinMode.AudioRecordRunning = true; - // //var time = Util.TimeStr(); - // var pars = new TRTCAudioRecordingParams { filePath = Path.Combine(RecoderDir, $"temp_audio.aac") }; - // var res = lTRTCCloud.startAudioRecording(ref pars); - // //Console.WriteLine(res); - //} + StartRecordAudio(); } } + /// + /// 录音 + /// public static void StartRecordAudio() { if (liveWinMode.IsLive && !liveWinMode.AudioRecordRunning) { liveWinMode.AudioRecordRunning = true; - //var time = Util.TimeStr(); var pars = new TRTCAudioRecordingParams { filePath = Path.Combine(RecoderDir, $"temp_audio.aac") }; - var res = lTRTCCloud.startAudioRecording(ref pars); - //Console.WriteLine(res); + lTRTCCloud.startAudioRecording(ref pars); + } + } + public static void StopRecordAudio() + { + if (liveWinMode.AudioRecordRunning) + { + liveWinMode.AudioRecordRunning = false; + lTRTCCloud.stopAudioRecording(); } } /// @@ -309,11 +300,7 @@ namespace JianGongYun.TRTC if (liveWinMode.MicRunning) { liveWinMode.MicRunning = false; - if (liveWinMode.AudioRecordRunning) - { - liveWinMode.AudioRecordRunning = false; - lTRTCCloud.stopAudioRecording(); - } + StopRecordAudio(); lTRTCCloud.stopLocalAudio(); } } @@ -337,31 +324,41 @@ namespace JianGongYun.TRTC var fps = settingWindowViewModel.LiveFps;//视频采集的fps BackgroundFrame = new Mat(int.Parse(resolution[2]), int.Parse(resolution[1]), MatType.CV_8UC4, backColor);//合成双路视频的背景 var delay = 1000 / (int)fps;//每帧时间 - var delayEqualize = 0;//每帧时间补偿，在性能和其他因素影响下delay的时间不一定充足 var _recoderDir = RecoderDir; var videoFile = Path.Combine(_recoderDir, $"temp_video.avi"); //var videoFrameTemp = Path.Combine(_recoderDir, $"videoFrameTemp.bmp"); var backHeight = BackgroundFrame.Rows;//画面高度 var backWidth = BackgroundFrame.Cols;//画面宽度 - VideoWriter vw = new VideoWriter(videoFile, FourCC.H264, fps, BackgroundFrame.Size()); + var end = false; + + + ConcurrentQueue mats = new ConcurrentQueue(); + + //int runFps = 0;//实时帧 + //Timer timer = new Timer((a) => + //{ + // Debug.Print($"runFps {runFps}. leavingsMat {mats.Count}."); + // Interlocked.Exchange(ref runFps, 0); + //}, null, 0, 1000); + onEnd = () => { end = true; + //timer.Dispose(); }; - - - + //帧合并线程 Task.Factory.StartNew(() => { - Stopwatch stopwatch = new Stopwatch();//计时器 + Stopwatch stopwatch = new Stopwatch();//计时器 bool onlyCameraInit = false;//只有摄像头的话背景填充为backColor，变量标记只需设置一次 bool noImgInit = false;//没有画面背景也填充backColor，变量标记只需设置一次 + var delayEqualize = 0;//每帧时间补偿，在性能和其他因素影响下delay的时间不一定充足 //屏幕分享画面 var screenRoi = BackgroundFrame[new OpenCvSharp.Rect(0, 0, backWidth, backHeight)]; @@ -376,7 +373,10 @@ namespace JianGongYun.TRTC //摄像头大画面位置 var bigRoi = BackgroundFrame[new OpenCvSharp.Rect((backWidth - backHeight) / 2, 0, backHeight, backHeight)]; - StartRecordAudio(); + + + + Debug.Print("开始处理帧线程"); while (!end) { stopwatch.Restart(); @@ -422,7 +422,7 @@ namespace JianGongYun.TRTC } } - Skip1: + Skip1: if (liveWinMode.CameraRunning)//摄像头分享中 { @@ -450,40 +450,91 @@ namespace JianGongYun.TRTC } } - Skip2: + Skip2: - var temp = BackgroundFrame.CvtColor(ColorConversionCodes.BGRA2BGR); - vw.Write(temp); - temp.Dispose(); + mats.Enqueue(BackgroundFrame.CvtColor(ColorConversionCodes.BGRA2BGR)); + //Interlocked.Increment(ref runFps); stopwatch.Stop(); - var aa = $"video frame run {stopwatch.ElapsedMilliseconds}"; - //Debug.Print(aa); - Console.WriteLine(aa); var sleep = delay - (int)stopwatch.ElapsedMilliseconds;//每帧时间减去每帧处理时间为sleep时间 - if (sleep < 0)//如果处理时间超过了每帧时间，记录下来 + //if (sleep < 0)//如果处理时间超过了每帧时间，记录下来 + //{ + // delayEqualize += sleep; + //} + //if (delayEqualize < 0 && sleep > 0) + //{ + // delayEqualize += sleep; + // if (delayEqualize > 0) + // { + // delayEqualize = 0; + // } + //} + //var lastsleep = sleep + delayEqualize;//理论休眠时间再去掉补偿时间 + //Debug.Print($"video frame run {stopwatch.ElapsedMilliseconds}. lastsleep {lastsleep}. sum {(stopwatch.ElapsedMilliseconds + lastsleep)}"); + //if (lastsleep > 0) + //{ + // Thread.Sleep(lastsleep); + //} + if (liveWinMode.IsLive && sleep > 0) { - delayEqualize += sleep; - } - if (delayEqualize < 0 && sleep > 0) - { - delayEqualize += sleep; - if (delayEqualize > 0) - { - delayEqualize = 0; - } - } - var lastsleep = sleep + delayEqualize;//理论休眠时间再去掉补偿时间 - if (lastsleep > 0) - { - Thread.Sleep(lastsleep); + Thread.Sleep(sleep); } } BackgroundFrame?.Dispose(); BackgroundFrame = null; - vw.Dispose(); - Debug.Print("录制结束"); + Debug.Print("结束处理帧线程"); }, TaskCreationOptions.LongRunning);//新开线程 + //新开线程写文件，减少帧处理时间 + Task.Factory.StartNew(() => + { + var delayEqualize = 0;//每帧时间补偿，在性能和其他因素影响下delay的时间不一定充足 + VideoWriter vw = new VideoWriter(videoFile, FourCC.H264, fps, BackgroundFrame.Size()); + Stopwatch stopwatch = new Stopwatch();//计时器 + Debug.Print("开始写入视频线程"); + while (!end || mats.Count > 0) + { + if (mats.Count > 0) + { + if (mats.TryDequeue(out var frame)) + { + stopwatch.Restart(); + vw.Write(frame); + frame.Dispose(); + stopwatch.Stop(); + Debug.Print($"video frame write {stopwatch.ElapsedMilliseconds}"); + //var sleep = delay - (int)stopwatch.ElapsedMilliseconds;//每帧时间减去每帧处理时间为sleep时间 + //if (sleep < 0)//如果处理时间超过了每帧时间，记录下来 + //{ + // delayEqualize += sleep; + //} + //if (delayEqualize < 0 && sleep > 0) + //{ + // delayEqualize += sleep; + // if (delayEqualize > 0) + // { + // delayEqualize = 0; + // } + //} + //var lastsleep = sleep + delayEqualize;//理论休眠时间再去掉补偿时间 + //if (lastsleep > 0) + //{ + // Thread.Sleep(lastsleep); + //} + } + else + { + Thread.Sleep(delay); + } + } + else + { + Thread.Sleep(delay); + } + } + vw.Dispose(); + Debug.Print("结束写入视频线程"); + }, TaskCreationOptions.LongRunning); + } /// diff --git a/JianGongYun/TRTC/Windows/LiveWindow.xaml.cs b/JianGongYun/TRTC/Windows/LiveWindow.xaml.cs index d4c7243..032a938 100644 --- a/JianGongYun/TRTC/Windows/LiveWindow.xaml.cs +++ b/JianGongYun/TRTC/Windows/LiveWindow.xaml.cs @@ -52,14 +52,14 @@ namespace JianGongYun.TRTC.Windows Rad2.Foreground = color; Rad3.Foreground = color; Rad4.Foreground = color; - //LiveClassroom.PauseAllView(false);//切前台启动实时预览渲染 + LiveClassroom.PauseAllView(false);//切前台启动实时预览渲染 } protected override void OnDeactivated(EventArgs e) { base.OnDeactivated(e); - //LiveClassroom.PauseAllView(true);//切后台停止实时预览渲染 + LiveClassroom.PauseAllView(true);//切后台停止实时预览渲染 } private Window settingWindow; diff --git a/JianGongYun/TRTC/Windows/SettingWindow.xaml b/JianGongYun/TRTC/Windows/SettingWindow.xaml index bf4a294..d0a287c 100644 --- a/JianGongYun/TRTC/Windows/SettingWindow.xaml +++ b/JianGongYun/TRTC/Windows/SettingWindow.xaml @@ -103,6 +103,7 @@ 录制倒计时 + @@ -136,8 +137,8 @@ 帧率(fps) - - + diff --git a/JianGongYun/TRTC_SDK/Win32/lib/ffmpeg.exe b/JianGongYun/TRTC_SDK/Win32/lib/ffmpeg.exe new file mode 100644 index 0000000..8df69e8 Binary files /dev/null and b/JianGongYun/TRTC_SDK/Win32/lib/ffmpeg.exe differ diff --git a/JianGongYun/TRTC_SDK/Win64/lib/ffmpeg.exe b/JianGongYun/TRTC_SDK/Win64/lib/ffmpeg.exe new file mode 100644 index 0000000..bbd81aa Binary files /dev/null and b/JianGongYun/TRTC_SDK/Win64/lib/ffmpeg.exe differ