音視訊傳輸中時間戳平滑處理
阿新 • • 發佈:2018-12-13
在音視訊中一般時間戳從裝置中系統時間得來,通常是以毫秒作為單位的linux時間。因為網路傳輸或者時間有時候突變的因為,造成了時間戳混亂。有必要對時間戳做一下處理。包括突變時候平滑處理,包括音視訊不同步的時候的處理,下面演算法解決了時間戳計算問題,在移動裝置上很有好處:
static int RTMP_CanculateTimestamp(CarEye_RTMP_Handle handle, MediaFrameInfo& FrameInfo, unsigned int MediaType) { int Diff; if(handle == NULL) { return -1; } CERTMPInfo* thiz = (CERTMPInfo*)handle; if(MediaType == CAREYE_VFRAME_FLAG) { if(thiz->fVideoIndex < 0) { thiz->fVideoIndex++;//skip first frame } else { if(FrameInfo.TimestampInternal == 0 && thiz->fVideoPTSIncrement > 0 && thiz->fVideoIndex >= TIMESTAMP_CANCULATE_VIDEO_THRESHOLD_SIZE) { unsigned int preIndex = (thiz->fVideoIndex + TIMESTAMP_CANCULATE_BUFFER_SIZE - 1)%TIMESTAMP_CANCULATE_BUFFER_SIZE; long pts = (long)(FrameInfo.Timestamp - thiz->fVideoTimestamp[preIndex]); if(pts > (thiz->fVideoPTSIncrement << 1) || pts < -(thiz->fVideoPTSIncrement << 1)) { /*printf("Have an abnormal video frame! drop the Timestamp!! fVideoIndex=%d, preIndex=%d, Timestamp=%ul, oldTimestamp=%ul, fVideoPTSIncrement=%d, pts=%ld\n", thiz->fVideoIndex, preIndex, FrameInfo.Timestamp, thiz->fVideoTimestamp[preIndex], thiz->fVideoPTSIncrement, pts);*/ if((++thiz->fVideoFrameAbnornalCounts) > 1) { thiz->fVideoIndex = 0; thiz->fVideoFrameAbnornalCounts = 0; thiz->fVideoPTSIncrement = 0; //重新賦值 } else { return -2; } } } thiz->fVideoTimestamp[thiz->fVideoIndex++ % TIMESTAMP_CANCULATE_BUFFER_SIZE] = FrameInfo.Timestamp; if(thiz->fVideoIndex == (TIMESTAMP_CANCULATE_BUFFER_SIZE << 8)) { thiz->fVideoIndex = TIMESTAMP_CANCULATE_BUFFER_SIZE; } if(TIMESTAMP_CANCULATE_VIDEO_THRESHOLD_SIZE <= thiz->fVideoIndex) { int frameCount = (thiz->fVideoIndex >= TIMESTAMP_CANCULATE_BUFFER_SIZE)?TIMESTAMP_CANCULATE_BUFFER_SIZE:thiz->fVideoIndex; if(frameCount >= TIMESTAMP_CANCULATE_BUFFER_SIZE) { thiz->fVideoPTSIncrement = (FrameInfo.Timestamp - thiz->fVideoTimestamp[thiz->fVideoIndex%TIMESTAMP_CANCULATE_BUFFER_SIZE] + (TIMESTAMP_CANCULATE_BUFFER_SIZE>>1))/(TIMESTAMP_CANCULATE_BUFFER_SIZE-1); } else { thiz->fVideoPTSIncrement = (FrameInfo.Timestamp - thiz->fVideoTimestamp[0] + (frameCount>>1)) / (frameCount-1); } } } } else if(MediaType == CAREYE_AFRAME_FLAG) { if(thiz->fAudioIndex < 0) { thiz->fAudioIndex++;//skip first frame } else { if(FrameInfo.TimestampInternal == 0 && thiz->fAudioPTSIncrement > 0 && thiz->fAudioIndex >= TIMESTAMP_CANCULATE_AUDIO_THRESHOLD_SIZE) { unsigned int preIndex = (thiz->fAudioIndex + TIMESTAMP_CANCULATE_BUFFER_SIZE - 1)%TIMESTAMP_CANCULATE_BUFFER_SIZE; long pts = (long)(FrameInfo.Timestamp - thiz->fAudioTimestamp[preIndex]); if(pts > (thiz->fAudioPTSIncrement << 1) || pts < -(thiz->fAudioPTSIncrement << 1)) { /*printf("Have an abnormal audio frame! drop the Timestamp!! fAudioIndex=%d, preIndex=%d, Timestamp=%ul, oldTimestamp=%ul, fAudioPTSIncrement=%d, pts=%ld\n", thiz->fAudioIndex, preIndex, FrameInfo.Timestamp, thiz->fAudioTimestamp[preIndex], thiz->fAudioPTSIncrement, pts);*/ if((++thiz->fAudioFrameAbnornalCounts) > 1) { thiz->fAudioIndex = 0; thiz->fAudioFrameAbnornalCounts = 0; thiz->fAudioPTSIncrement = 0; } else { return -2; } } } thiz->fAudioTimestamp[thiz->fAudioIndex++ % TIMESTAMP_CANCULATE_BUFFER_SIZE] = FrameInfo.Timestamp; if(thiz->fAudioIndex == (TIMESTAMP_CANCULATE_BUFFER_SIZE << 8)) { thiz->fAudioIndex = TIMESTAMP_CANCULATE_BUFFER_SIZE; } if(TIMESTAMP_CANCULATE_AUDIO_THRESHOLD_SIZE <= thiz->fAudioIndex) { int frameCount = (thiz->fAudioIndex >= TIMESTAMP_CANCULATE_BUFFER_SIZE)?TIMESTAMP_CANCULATE_BUFFER_SIZE:thiz->fAudioIndex; if(frameCount >= TIMESTAMP_CANCULATE_BUFFER_SIZE) { thiz->fAudioPTSIncrement = (FrameInfo.Timestamp - thiz->fAudioTimestamp[thiz->fAudioIndex % TIMESTAMP_CANCULATE_BUFFER_SIZE] + (TIMESTAMP_CANCULATE_BUFFER_SIZE>>1))/(TIMESTAMP_CANCULATE_BUFFER_SIZE-1); } else { thiz->fAudioPTSIncrement = (FrameInfo.Timestamp - thiz->fAudioTimestamp[0] + (frameCount>>1)) / (frameCount-1); } } } } return 0; }
#define MAX_AUDIO_VEDIO_DIFF 1000 int CRTMPPublisher::SendH264Packet(unsigned char *data, UINT32 size,bool bIsKeyFrame, UINT32 nTimeStamp, UINT32 ptzIncrement) { UINT32 Diff; if(data == NULL && size<11) { return -1; } if (data[0]==0x00 && data[1]==0x00 && data[2]==0x00 && data[3]==0x01) { data +=4; size -= 4; } if (data[0]==0x00 && data[1]==0x00 && data[2]==0x01) { data +=3; size -= 3; } if (m_bSentMetadata == false) { if (!RTMP_IsConnected(m_pRtmp)/* && (Reconnect() == false)*/) { return -2; } if(SendMetadata()==true) { m_bSentMetadata = true; } else { m_bSentMetadata = false; return -3; } } int nalu_type = data[0] & 0x1F; int nalu_offset = 0; int parser_offset = 0; while (nalu_type == 0x06 || nalu_type == 0x07 || nalu_type == 0x08) { nalu_offset = FindNaluHead(data + parser_offset, size - parser_offset); if (nalu_offset == -1) { return -4; } parser_offset += nalu_offset; nalu_type = data[parser_offset] & 0x1F; } data += parser_offset; size -= parser_offset; if (size > m_nH264BuffSize - 10 - 30) { _TRACE_LOG("SendH264Packet error! packet too large! size =%ld.\n", size); return -5; } unsigned char *body = (unsigned char*)(m_h264Buff+30); int i = 0; if(bIsKeyFrame) { body[i++] = 0x17;// 1:Iframe 7:AVC if (m_bWaitingKeyFrame) { m_bWaitingKeyFrame = false; } } else { body[i++] = 0x27;// 2:Pframe 7:AVC } body[i++] = 0x01;// AVC NALU body[i++] = 0x00; body[i++] = 0x00; body[i++] = 0x00; // NALU size body[i++] = size>>24; body[i++] = size>>16; body[i++] = size>>8; body[i++] = size&0xff; // NALU data memcpy(&body[i],data,size); if (!RTMP_IsConnected(m_pRtmp)/* && (Reconnect() == false)*/) { return -6; } if (m_bWaitingKeyFrame) { return -7; } if(ptzIncrement <= 0) { //int nTick = 0; //if (m_uLastVideoPTS == 0) //{ // m_uLastVideoPTS = nTimeStamp; // m_uVideoPTS = 0; //} //else //{ // nTick = nTimeStamp - m_uLastVideoPTS; // if (nTick < 0 || nTick>500) // { // nTick = 40; // //printf("H264 TimeStamp Exception : Tick = 0x%x\n", nTimeStamp-m_uLastVideoPTS); // } //} //m_uVideoPTS += nTick; //m_uLastVideoPTS = nTimeStamp; //m_uVideoPTS += 100; m_uAudioPTS = m_uVideoPTS = nTimeStamp; //printf("%s m_uAudioPTS=%u, m_uVideoPTS=%u, ptzIncrement=%d\n", __FUNCTION__, m_uAudioPTS, m_uVideoPTS, ptzIncrement); } else { m_uVideoPTS += ptzIncrement; } //增加橫向音視訊同步 Diff = m_uAudioPTS > m_uVideoPTS ? m_uAudioPTS - m_uVideoPTS : m_uVideoPTS - m_uAudioPTS; if (Diff > MAX_AUDIO_VEDIO_DIFF) { m_uAudioPTS = m_uVideoPTS = nTimeStamp; } //printf("%s m_uAudioPTS=%u, m_uVideoPTS=%u, ptzIncrement=%d\n", __FUNCTION__, m_uAudioPTS, m_uVideoPTS, ptzIncrement); int iRet = SendPacket(RTMP_PACKET_TYPE_VIDEO, body, i+size, m_uVideoPTS); return iRet; }
第一步時間時間戳的平滑處理和突變的判斷,第二步對音視訊時間戳不同步進行了處理。相關程式碼可以從開源網站下載:https://github.com/Car-eye-team