From 8144bba32aaac4fd114184f74e043467ac5da2fe Mon Sep 17 00:00:00 2001 From: SteveGilvarry Date: Mon, 31 Aug 2015 21:52:16 +1000 Subject: [PATCH 1/3] Add videostore to remote_rtsp, still need to check stream is h264 --- src/zm_remote_camera_rtsp.cpp | 153 ++++++++++++++++++++++++++++++++++ src/zm_remote_camera_rtsp.h | 8 +- 2 files changed, 160 insertions(+), 1 deletion(-) diff --git a/src/zm_remote_camera_rtsp.cpp b/src/zm_remote_camera_rtsp.cpp index a07d97a8e..675e496b9 100644 --- a/src/zm_remote_camera_rtsp.cpp +++ b/src/zm_remote_camera_rtsp.cpp @@ -52,11 +52,13 @@ RemoteCameraRtsp::RemoteCameraRtsp( int p_id, const std::string &p_method, const mFormatContext = NULL; mVideoStreamId = -1; + mAudioStreamId = -1; mCodecContext = NULL; mCodec = NULL; mRawFrame = NULL; mFrame = NULL; frameCount = 0; + wasRecording = false; #if HAVE_LIBSWSCALE mConvertContext = NULL; @@ -364,6 +366,157 @@ int RemoteCameraRtsp::Capture( Image &image ) return (0) ; } +//Function to handle capture and store +int RemoteCameraRtsp::CaptureAndRecord( Image &image, bool recording, char* event_file ) +{ + AVPacket packet; + uint8_t* directbuffer; + int frameComplete = false; + + /* Request a writeable buffer of the target image */ + directbuffer = image.WriteBuffer(width, height, colours, subpixelorder); + if(directbuffer == NULL) { + Error("Failed requesting writeable buffer for the captured image."); + return (-1); + } + + while ( true ) + { + buffer.clear(); + if ( !rtspThread->isRunning() ) + return (-1); + + if ( rtspThread->getFrame( buffer ) ) + { + Debug( 3, "Read frame %d bytes", buffer.size() ); + Debug( 4, "Address %p", buffer.head() ); + Hexdump( 4, buffer.head(), 16 ); + + if ( !buffer.size() ) + return( -1 ); + + if(mCodecContext->codec_id == AV_CODEC_ID_H264) + { + // SPS and PPS frames should be saved and appended to IDR frames + int nalType = (buffer.head()[3] & 0x1f); + + // SPS + if(nalType == 7) + { + lastSps = buffer; + continue; + } + // PPS + else if(nalType == 8) + { + lastPps = buffer; + continue; + } + // IDR + else if(nalType == 5) + { + buffer += lastSps; + buffer += lastPps; + } + } + + av_init_packet( &packet ); + + while ( !frameComplete && buffer.size() > 0 ) + { + packet.data = buffer.head(); + packet.size = buffer.size(); +#if LIBAVCODEC_VERSION_CHECK(52, 23, 0, 23, 0) + int len = avcodec_decode_video2( mCodecContext, mRawFrame, &frameComplete, &packet ); +#else + int len = avcodec_decode_video( mCodecContext, mRawFrame, &frameComplete, packet.data, packet.size ); +#endif + if ( len < 0 ) + { + Error( "Error while decoding frame %d", frameCount ); + Hexdump( Logger::ERROR, buffer.head(), buffer.size()>256?256:buffer.size() ); + buffer.clear(); + continue; + } + Debug( 2, "Frame: %d - %d/%d", frameCount, len, buffer.size() ); + //if ( buffer.size() < 400 ) + //Hexdump( 0, buffer.head(), buffer.size() ); + + buffer -= len; + + } + if ( frameComplete ) { + + Debug( 3, "Got frame %d", frameCount ); + + avpicture_fill( (AVPicture *)mFrame, directbuffer, imagePixFormat, width, height); + + //Video recording + if(recording && !wasRecording){ + //Instantiate the video storage module + + videoStore = new VideoStore((const char *)event_file, "mp4", mFormatContext->streams[mVideoStreamId],mAudioStreamId==-1?NULL:mFormatContext->streams[mAudioStreamId],startTime); + wasRecording = true; + strcpy(oldDirectory, event_file); + + }else if(!recording && wasRecording && videoStore){ + Info("Deleting videoStore instance"); + delete videoStore; + videoStore = NULL; + } + + //The directory we are recording to is no longer tied to the current event. Need to re-init the videostore with the correct directory and start recording again + if(recording && wasRecording && (strcmp(oldDirectory, event_file)!=0) && (packet.flags & AV_PKT_FLAG_KEY) ){ //don't open new videostore until we're on a key frame..would this require an offset adjustment for the event as a result?...if we store our key frame location with the event will that be enough? + Info("Re-starting video storage module"); + if(videoStore){ + delete videoStore; + videoStore = NULL; + } + + videoStore = new VideoStore((const char *)event_file, "mp4", mFormatContext->streams[mVideoStreamId],mAudioStreamId==-1?NULL:mFormatContext->streams[mAudioStreamId],startTime); + strcpy(oldDirectory, event_file); + } + + if(videoStore && recording){ + //Write the packet to our video store + int ret = videoStore->writeVideoFramePacket(&packet, mFormatContext->streams[mVideoStreamId]);//, &lastKeyframePkt); + if(ret<0){//Less than zero and we skipped a frame + av_free_packet( &packet ); + return 0; + } + } + +#if HAVE_LIBSWSCALE + if(mConvertContext == NULL) { + if(config.cpu_extensions && sseversion >= 20) { + mConvertContext = sws_getContext( mCodecContext->width, mCodecContext->height, mCodecContext->pix_fmt, width, height, imagePixFormat, SWS_BICUBIC | SWS_CPU_CAPS_SSE2, NULL, NULL, NULL ); + } else { + mConvertContext = sws_getContext( mCodecContext->width, mCodecContext->height, mCodecContext->pix_fmt, width, height, imagePixFormat, SWS_BICUBIC, NULL, NULL, NULL ); + } + if(mConvertContext == NULL) + Fatal( "Unable to create conversion context"); + } + + if ( sws_scale( mConvertContext, mRawFrame->data, mRawFrame->linesize, 0, mCodecContext->height, mFrame->data, mFrame->linesize ) < 0 ) + Fatal( "Unable to convert raw format %u to target format %u at frame %d", mCodecContext->pix_fmt, imagePixFormat, frameCount ); +#else // HAVE_LIBSWSCALE + Fatal( "You must compile ffmpeg with the --enable-swscale option to use RTSP cameras" ); +#endif // HAVE_LIBSWSCALE + + frameCount++; + + } /* frame complete */ + + av_free_packet( &packet ); + } /* getFrame() */ + + if(frameComplete) + return (0); + + } + return (0) ; +} + int RemoteCameraRtsp::PostCapture() { return( 0 ); diff --git a/src/zm_remote_camera_rtsp.h b/src/zm_remote_camera_rtsp.h index 1e2bba7f8..257df4266 100644 --- a/src/zm_remote_camera_rtsp.h +++ b/src/zm_remote_camera_rtsp.h @@ -26,6 +26,7 @@ #include "zm_utils.h" #include "zm_rtsp.h" #include "zm_ffmpeg.h" +#include "zm_videostore.h" // // Class representing 'rtsp' cameras, i.e. those which are @@ -55,12 +56,17 @@ protected: #if HAVE_LIBAVFORMAT AVFormatContext *mFormatContext; int mVideoStreamId; + int mAudioStreamId; AVCodecContext *mCodecContext; AVCodec *mCodec; AVFrame *mRawFrame; AVFrame *mFrame; _AVPIXELFORMAT imagePixFormat; #endif // HAVE_LIBAVFORMAT + bool wasRecording; + VideoStore *videoStore; + char oldDirectory[4096]; + int64_t startTime; #if HAVE_LIBSWSCALE struct SwsContext *mConvertContext; @@ -79,7 +85,7 @@ public: int PreCapture(); int Capture( Image &image ); int PostCapture(); - int CaptureAndRecord( Image &image, bool recording, char* event_directory ) {return(0);}; + int CaptureAndRecord( Image &image, bool recording, char* event_directory ); }; #endif // ZM_REMOTE_CAMERA_RTSP_H From 612e326917e6569dee4982c9d03f740d4c33844b Mon Sep 17 00:00:00 2001 From: SteveGilvarry Date: Fri, 16 Oct 2015 23:32:54 +1100 Subject: [PATCH 2/3] Fixes for remote h264 passthrough --- src/zm_camera.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zm_camera.h b/src/zm_camera.h index 5d54919f1..9f901e17f 100755 --- a/src/zm_camera.h +++ b/src/zm_camera.h @@ -74,7 +74,7 @@ public: bool CanCapture() const { return( capture ); } - bool SupportsNativeVideo() const { return( type == FFMPEG_SRC ); } + bool SupportsNativeVideo() const { return( (type == FFMPEG_SRC )||(type == REMOTE_SRC)); } virtual int PrimeCapture() { return( 0 ); } virtual int PreCapture()=0; From 4f3664b7f8425bc47a7e6e25bfb30f5e36b6d35b Mon Sep 17 00:00:00 2001 From: SteveGilvarry Date: Wed, 2 Dec 2015 07:17:50 +1100 Subject: [PATCH 3/3] Bring is more of the h264 code and add some master changes to the duplicated code --- src/zm_remote_camera_rtsp.cpp | 161 +++++++++++++++++++--------------- 1 file changed, 90 insertions(+), 71 deletions(-) diff --git a/src/zm_remote_camera_rtsp.cpp b/src/zm_remote_camera_rtsp.cpp index 675e496b9..640ab0356 100644 --- a/src/zm_remote_camera_rtsp.cpp +++ b/src/zm_remote_camera_rtsp.cpp @@ -59,6 +59,7 @@ RemoteCameraRtsp::RemoteCameraRtsp( int p_id, const std::string &p_method, const mFrame = NULL; frameCount = 0; wasRecording = false; + startTime=0; #if HAVE_LIBSWSCALE mConvertContext = NULL; @@ -422,92 +423,110 @@ int RemoteCameraRtsp::CaptureAndRecord( Image &image, bool recording, char* even av_init_packet( &packet ); - while ( !frameComplete && buffer.size() > 0 ) - { - packet.data = buffer.head(); - packet.size = buffer.size(); + if ( packet.stream_index == mVideoStreamId ) + { + + while ( !frameComplete && buffer.size() > 0 ) + { + packet.data = buffer.head(); + packet.size = buffer.size(); #if LIBAVCODEC_VERSION_CHECK(52, 23, 0, 23, 0) - int len = avcodec_decode_video2( mCodecContext, mRawFrame, &frameComplete, &packet ); + int len = avcodec_decode_video2( mCodecContext, mRawFrame, &frameComplete, &packet ); #else - int len = avcodec_decode_video( mCodecContext, mRawFrame, &frameComplete, packet.data, packet.size ); + int len = avcodec_decode_video( mCodecContext, mRawFrame, &frameComplete, packet.data, packet.size ); #endif - if ( len < 0 ) - { - Error( "Error while decoding frame %d", frameCount ); - Hexdump( Logger::ERROR, buffer.head(), buffer.size()>256?256:buffer.size() ); - buffer.clear(); - continue; - } - Debug( 2, "Frame: %d - %d/%d", frameCount, len, buffer.size() ); - //if ( buffer.size() < 400 ) - //Hexdump( 0, buffer.head(), buffer.size() ); - - buffer -= len; + if ( len < 0 ) + { + Error( "Error while decoding frame %d", frameCount ); + Hexdump( Logger::ERROR, buffer.head(), buffer.size()>256?256:buffer.size() ); + buffer.clear(); + continue; + } + Debug( 2, "Frame: %d - %d/%d", frameCount, len, buffer.size() ); + //if ( buffer.size() < 400 ) + //Hexdump( 0, buffer.head(), buffer.size() ); - } - if ( frameComplete ) { - - Debug( 3, "Got frame %d", frameCount ); - - avpicture_fill( (AVPicture *)mFrame, directbuffer, imagePixFormat, width, height); - - //Video recording - if(recording && !wasRecording){ - //Instantiate the video storage module + buffer -= len; - videoStore = new VideoStore((const char *)event_file, "mp4", mFormatContext->streams[mVideoStreamId],mAudioStreamId==-1?NULL:mFormatContext->streams[mAudioStreamId],startTime); - wasRecording = true; - strcpy(oldDirectory, event_file); - - }else if(!recording && wasRecording && videoStore){ - Info("Deleting videoStore instance"); - delete videoStore; - videoStore = NULL; } - - //The directory we are recording to is no longer tied to the current event. Need to re-init the videostore with the correct directory and start recording again - if(recording && wasRecording && (strcmp(oldDirectory, event_file)!=0) && (packet.flags & AV_PKT_FLAG_KEY) ){ //don't open new videostore until we're on a key frame..would this require an offset adjustment for the event as a result?...if we store our key frame location with the event will that be enough? - Info("Re-starting video storage module"); - if(videoStore){ + if ( frameComplete ) { + + Debug( 3, "Got frame %d", frameCount ); + + avpicture_fill( (AVPicture *)mFrame, directbuffer, imagePixFormat, width, height); + + //Video recording + if(recording && !wasRecording){ + //Instantiate the video storage module + + videoStore = new VideoStore((const char *)event_file, "mp4", mFormatContext->streams[mVideoStreamId],mAudioStreamId==-1?NULL:mFormatContext->streams[mAudioStreamId],startTime); + wasRecording = true; + strcpy(oldDirectory, event_file); + + }else if(!recording && wasRecording && videoStore){ + Info("Deleting videoStore instance"); delete videoStore; videoStore = NULL; } - videoStore = new VideoStore((const char *)event_file, "mp4", mFormatContext->streams[mVideoStreamId],mAudioStreamId==-1?NULL:mFormatContext->streams[mAudioStreamId],startTime); - strcpy(oldDirectory, event_file); - } - - if(videoStore && recording){ + //The directory we are recording to is no longer tied to the current event. Need to re-init the videostore with the correct directory and start recording again + if(recording && wasRecording && (strcmp(oldDirectory, event_file)!=0) && (packet.flags & AV_PKT_FLAG_KEY) ){ //don't open new videostore until we're on a key frame..would this require an offset adjustment for the event as a result?...if we store our key frame location with the event will that be enough? + Info("Re-starting video storage module"); + if(videoStore){ + delete videoStore; + videoStore = NULL; + } + + videoStore = new VideoStore((const char *)event_file, "mp4", mFormatContext->streams[mVideoStreamId],mAudioStreamId==-1?NULL:mFormatContext->streams[mAudioStreamId],startTime); + strcpy(oldDirectory, event_file); + } + + if(videoStore && recording){ + //Write the packet to our video store + int ret = videoStore->writeVideoFramePacket(&packet, mFormatContext->streams[mVideoStreamId]);//, &lastKeyframePkt); + if(ret<0){//Less than zero and we skipped a frame + av_free_packet( &packet ); + return 0; + } + } + +#if HAVE_LIBSWSCALE + if(mConvertContext == NULL) { + mConvertContext = sws_getContext( mCodecContext->width, mCodecContext->height, mCodecContext->pix_fmt, width, height, imagePixFormat, SWS_BICUBIC, NULL, NULL, NULL ); + + if(mConvertContext == NULL) + Fatal( "Unable to create conversion context"); + } + + if ( sws_scale( mConvertContext, mRawFrame->data, mRawFrame->linesize, 0, mCodecContext->height, mFrame->data, mFrame->linesize ) < 0 ) + Fatal( "Unable to convert raw format %u to target format %u at frame %d", mCodecContext->pix_fmt, imagePixFormat, frameCount ); +#else // HAVE_LIBSWSCALE + Fatal( "You must compile ffmpeg with the --enable-swscale option to use RTSP cameras" ); +#endif // HAVE_LIBSWSCALE + + frameCount++; + + } /* frame complete */ + } + else if(packet.stream_index == mAudioStreamId) + { + if(videoStore && recording) + { //Write the packet to our video store - int ret = videoStore->writeVideoFramePacket(&packet, mFormatContext->streams[mVideoStreamId]);//, &lastKeyframePkt); - if(ret<0){//Less than zero and we skipped a frame + int ret = videoStore->writeAudioFramePacket(&packet, mFormatContext->streams[packet.stream_index]); //FIXME no relevance of last key frame + if(ret<0) //Less than zero and we skipped a frame + { av_free_packet( &packet ); - return 0; + return 0; } } - -#if HAVE_LIBSWSCALE - if(mConvertContext == NULL) { - if(config.cpu_extensions && sseversion >= 20) { - mConvertContext = sws_getContext( mCodecContext->width, mCodecContext->height, mCodecContext->pix_fmt, width, height, imagePixFormat, SWS_BICUBIC | SWS_CPU_CAPS_SSE2, NULL, NULL, NULL ); - } else { - mConvertContext = sws_getContext( mCodecContext->width, mCodecContext->height, mCodecContext->pix_fmt, width, height, imagePixFormat, SWS_BICUBIC, NULL, NULL, NULL ); - } - if(mConvertContext == NULL) - Fatal( "Unable to create conversion context"); - } - - if ( sws_scale( mConvertContext, mRawFrame->data, mRawFrame->linesize, 0, mCodecContext->height, mFrame->data, mFrame->linesize ) < 0 ) - Fatal( "Unable to convert raw format %u to target format %u at frame %d", mCodecContext->pix_fmt, imagePixFormat, frameCount ); -#else // HAVE_LIBSWSCALE - Fatal( "You must compile ffmpeg with the --enable-swscale option to use RTSP cameras" ); -#endif // HAVE_LIBSWSCALE - - frameCount++; - - } /* frame complete */ + } - av_free_packet( &packet ); +#if LIBAVCODEC_VERSION_CHECK(57, 8, 0, 12, 100) + av_packet_unref( &packet); +#else + av_free_packet( &packet ); +#endif } /* getFrame() */ if(frameComplete)