@ -41,6 +41,8 @@
# include "asterisk/utils.h"
# include "asterisk/pbx.h"
# include "asterisk/timing.h"
# include "asterisk/rtp_engine.h"
# include "asterisk/format_cache.h"
# include "asterisk/abstract_jb.h"
# include "fixedjitterbuf.h"
@ -53,6 +55,9 @@ enum {
JB_CREATED = ( 1 < < 2 )
} ;
/*! The maximum size we allow the early frame buffer to get */
# define MAXIMUM_EARLY_FRAME_COUNT 200
/* Implementation functions */
/* fixed */
@ -568,6 +573,8 @@ int ast_jb_read_conf(struct ast_jb_conf *conf, const char *varname, const char *
}
} else if ( ! strcasecmp ( name , AST_JB_CONF_LOG ) ) {
ast_set2_flag ( conf , ast_true ( value ) , AST_JB_LOG ) ;
} else if ( ! strcasecmp ( name , AST_JB_CONF_SYNC_VIDEO ) ) {
ast_set2_flag ( conf , ast_true ( value ) , AST_JB_SYNC_VIDEO ) ;
} else {
return - 1 ;
}
@ -832,6 +839,11 @@ static int jb_is_late_adaptive(void *jb, long ts)
# define DEFAULT_RESYNC 1000
# define DEFAULT_TYPE AST_JB_FIXED
struct jb_stream_sync {
unsigned int timestamp ;
struct timeval ntp ;
} ;
struct jb_framedata {
const struct ast_jb_impl * jb_impl ;
struct ast_jb_conf jb_conf ;
@ -841,11 +853,21 @@ struct jb_framedata {
int timer_interval ; /* ms between deliveries */
int timer_fd ;
int first ;
int audio_stream_id ;
struct jb_stream_sync audio_stream_sync ;
int video_stream_id ;
struct jb_stream_sync video_stream_sync ;
AST_LIST_HEAD_NOLOCK ( , ast_frame ) early_frames ;
unsigned int early_frame_count ;
struct timeval last_audio_ntp_timestamp ;
int audio_flowing ;
void * jb_obj ;
} ;
static void jb_framedata_destroy ( struct jb_framedata * framedata )
{
struct ast_frame * frame ;
if ( framedata - > timer ) {
ast_timer_close ( framedata - > timer ) ;
framedata - > timer = NULL ;
@ -859,11 +881,15 @@ static void jb_framedata_destroy(struct jb_framedata *framedata)
framedata - > jb_obj = NULL ;
}
ao2_cleanup ( framedata - > last_format ) ;
while ( ( frame = AST_LIST_REMOVE_HEAD ( & framedata - > early_frames , frame_list ) ) ) {
ast_frfree ( frame ) ;
}
ast_free ( framedata ) ;
}
void ast_jb_conf_default ( struct ast_jb_conf * conf )
{
ast_clear_flag ( conf , AST_FLAGS_ALL ) ;
conf - > max_size = DEFAULT_SIZE ;
conf - > resync_threshold = DEFAULT_RESYNC ;
ast_copy_string ( conf - > impl , " fixed " , sizeof ( conf - > impl ) ) ;
@ -886,6 +912,44 @@ static void hook_destroy_cb(void *framedata)
jb_framedata_destroy ( ( struct jb_framedata * ) framedata ) ;
}
static struct timeval jitterbuffer_frame_get_ntp_timestamp ( const struct jb_stream_sync * stream_sync , const struct ast_frame * frame )
{
int timestamp_diff ;
unsigned int rate ;
/* It's possible for us to receive frames before we receive the information allowing
* us to do NTP / RTP timestamp calculations . Since the information isn ' t available we
* can ' t generate one and give an empty timestamp .
*/
if ( ast_tvzero ( stream_sync - > ntp ) ) {
return ast_tv ( 0 , 0 ) ;
}
/* Convert the Asterisk timestamp into an RTP timestamp, and then based on the difference we can
* determine how many samples are in the frame and how long has elapsed since the synchronization
* RTP and NTP timestamps were received giving us the NTP timestamp for this frame .
*/
if ( frame - > frametype = = AST_FRAME_VOICE ) {
rate = ast_rtp_get_rate ( frame - > subclass . format ) ;
timestamp_diff = ( frame - > ts * ( rate / 1000 ) ) - stream_sync - > timestamp ;
} else {
/* Video is special - internally we reference it as 1000 to preserve the RTP timestamp but
* it is actualy 90000 , this is why we can just directly subtract the timestamp .
*/
rate = 90000 ;
timestamp_diff = frame - > ts - stream_sync - > timestamp ;
}
if ( timestamp_diff < 0 ) {
/* It's possible for us to be asked for an NTP timestamp from before our latest
* RTCP SR report . To handle this we subtract so we go back in time .
*/
return ast_tvsub ( stream_sync - > ntp , ast_samp2tv ( abs ( timestamp_diff ) , rate ) ) ;
} else {
return ast_tvadd ( stream_sync - > ntp , ast_samp2tv ( timestamp_diff , rate ) ) ;
}
}
static struct ast_frame * hook_event_cb ( struct ast_channel * chan , struct ast_frame * frame , enum ast_framehook_event event , void * data )
{
struct jb_framedata * framedata = data ;
@ -928,6 +992,77 @@ static struct ast_frame *hook_event_cb(struct ast_channel *chan, struct ast_fram
return frame ;
}
if ( ast_test_flag ( & framedata - > jb_conf , AST_JB_SYNC_VIDEO ) ) {
if ( frame - > frametype = = AST_FRAME_VOICE ) {
/* Store the stream identifier for the audio stream so we can associate the incoming RTCP SR
* with the correct stream sync structure .
*/
framedata - > audio_stream_id = frame - > stream_num ;
} else if ( frame - > frametype = = AST_FRAME_RTCP & & frame - > subclass . integer = = AST_RTP_RTCP_SR ) {
struct ast_rtp_rtcp_report * rtcp_report = frame - > data . ptr ;
struct jb_stream_sync * stream_sync = NULL ;
/* Determine which stream this RTCP is in regards to */
if ( framedata - > audio_stream_id = = frame - > stream_num ) {
stream_sync = & framedata - > audio_stream_sync ;
} else if ( framedata - > video_stream_id = = frame - > stream_num ) {
stream_sync = & framedata - > video_stream_sync ;
}
if ( stream_sync ) {
/* Store the RTP and NTP timestamp mapping so we can derive an NTP timestamp for each frame */
stream_sync - > timestamp = rtcp_report - > sender_information . rtp_timestamp ;
stream_sync - > ntp = rtcp_report - > sender_information . ntp_timestamp ;
}
} else if ( frame - > frametype = = AST_FRAME_VIDEO ) {
/* If a video frame is late according to the audio timestamp don't stash it away, just return it.
* If however it is ahead then we keep it until such time as the audio catches up .
*/
struct ast_frame * jbframe ;
framedata - > video_stream_id = frame - > stream_num ;
/* If no timing information is available we can't store this away, so just let it through now */
if ( ! ast_test_flag ( frame , AST_FRFLAG_HAS_TIMING_INFO ) ) {
return frame ;
}
/* To ensure that the video starts when the audio starts we only start allowing frames through once
* audio starts flowing .
*/
if ( framedata - > audio_flowing ) {
struct timeval video_timestamp ;
video_timestamp = jitterbuffer_frame_get_ntp_timestamp ( & framedata - > video_stream_sync , frame ) ;
if ( ast_tvdiff_ms ( framedata - > last_audio_ntp_timestamp , video_timestamp ) > = 0 ) {
return frame ;
}
}
/* To prevent the early frame buffer from growing uncontrolled we impose a maximum count that it can
* get to . If this is reached then we drop a video frame , which should cause the receiver to ask for a
* new key frame .
*/
if ( framedata - > early_frame_count = = MAXIMUM_EARLY_FRAME_COUNT ) {
jbframe = AST_LIST_REMOVE_HEAD ( & framedata - > early_frames , frame_list ) ;
framedata - > early_frame_count - - ;
ast_frfree ( jbframe ) ;
}
jbframe = ast_frisolate ( frame ) ;
if ( ! jbframe ) {
/* If we can't isolate the frame the safest thing we can do is return it, even if the A/V sync
* may be off .
*/
return frame ;
}
AST_LIST_INSERT_TAIL ( & framedata - > early_frames , jbframe , frame_list ) ;
framedata - > early_frame_count + + ;
return & ast_null_frame ;
}
}
now_tv = ast_tvnow ( ) ;
now = ast_tvdiff_ms ( now_tv , framedata - > start_tv ) ;
@ -1022,6 +1157,8 @@ static struct ast_frame *hook_event_cb(struct ast_channel *chan, struct ast_fram
}
if ( frame - > frametype = = AST_FRAME_CONTROL ) {
struct ast_frame * early_frame ;
switch ( frame - > subclass . integer ) {
case AST_CONTROL_HOLD :
case AST_CONTROL_UNHOLD :
@ -1029,12 +1166,50 @@ static struct ast_frame *hook_event_cb(struct ast_channel *chan, struct ast_fram
case AST_CONTROL_SRCUPDATE :
case AST_CONTROL_SRCCHANGE :
framedata - > jb_impl - > force_resync ( framedata - > jb_obj ) ;
/* Since we are resyncing go ahead and clear out the video frames too */
while ( ( early_frame = AST_LIST_REMOVE_HEAD ( & framedata - > early_frames , frame_list ) ) ) {
ast_frfree ( early_frame ) ;
}
framedata - > audio_flowing = 0 ;
framedata - > early_frame_count = 0 ;
break ;
default :
break ;
}
}
/* If a voice frame is being passed through see if we need to add any additional frames to it */
if ( ast_test_flag ( & framedata - > jb_conf , AST_JB_SYNC_VIDEO ) & & frame - > frametype = = AST_FRAME_VOICE ) {
AST_LIST_HEAD_NOLOCK ( , ast_frame ) additional_frames ;
struct ast_frame * early_frame ;
/* We store the last NTP timestamp for the audio given to the core so that subsequents frames which
* are late can be passed immediately through ( this will occur for video frames which are returned here )
*/
framedata - > last_audio_ntp_timestamp = jitterbuffer_frame_get_ntp_timestamp ( & framedata - > audio_stream_sync , frame ) ;
framedata - > audio_flowing = 1 ;
AST_LIST_HEAD_INIT_NOLOCK ( & additional_frames ) ;
AST_LIST_TRAVERSE_SAFE_BEGIN ( & framedata - > early_frames , early_frame , frame_list ) {
struct timeval early_timestamp = jitterbuffer_frame_get_ntp_timestamp ( & framedata - > video_stream_sync , early_frame ) ;
int diff = ast_tvdiff_ms ( framedata - > last_audio_ntp_timestamp , early_timestamp ) ;
/* If this frame is from the past we need to include it with the audio frame that is going
* out .
*/
if ( diff > = 0 ) {
AST_LIST_REMOVE_CURRENT ( frame_list ) ;
framedata - > early_frame_count - - ;
AST_LIST_INSERT_TAIL ( & additional_frames , early_frame , frame_list ) ;
}
}
AST_LIST_TRAVERSE_SAFE_END ;
/* Append any additional frames we may want to include (such as video) */
AST_LIST_NEXT ( frame , frame_list ) = AST_LIST_FIRST ( & additional_frames ) ;
}
return frame ;
}
@ -1066,6 +1241,9 @@ static int jb_framedata_init(struct jb_framedata *framedata, struct ast_jb_conf
return - 1 ;
}
framedata - > audio_stream_id = - 1 ;
framedata - > video_stream_id = - 1 ;
AST_LIST_HEAD_INIT_NOLOCK ( & framedata - > early_frames ) ;
framedata - > timer_fd = ast_timer_fd ( framedata - > timer ) ;
framedata - > timer_interval = DEFAULT_TIMER_INTERVAL ;
ast_timer_set_rate ( framedata - > timer , 1000 / framedata - > timer_interval ) ;
Joshua Colp
6 years ago