Update ffmpeg, windows libs depended on pthreads. Add keyframes in h264 codec, fix start-end bits in FU-A, set desired video size in capture datasource.

cusax-fix
Damian Minkov 18 years ago
parent cbfddfe03a
commit d6dd75f4a0

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

@ -399,6 +399,9 @@ private void initCaptureDevices()
if (videoDeviceInfo != null)
{
videoDataSource = createDataSource(videoDeviceInfo.getLocator());
// we will check video sizes and will set the most appropriate one
selectVideoSize(videoDataSource);
}
// Create the av data source
@ -1289,4 +1292,49 @@ public void setMute(boolean mute)
if (muteAudioDataSource != null)
muteAudioDataSource.setMute(mute);
}
/**
* Selects the nearest size supported by the capture device,
* to make drivers scale the images.
*
* @param videoDS the video datasource
*/
private void selectVideoSize(DataSource videoDS)
{
FormatControl fControl =
(FormatControl)videoDS.getControl(FormatControl.class.getName());
if(fControl == null)
return;
// On MacOSX isight camera reports two sizes 640x480 and 320x240
// if we use the default size 352x288 we must use source format 640x480
// in this situation we suffer from high cpu usage as every frame is
// scaled, so we use the non standard format 320x240.
String osName = System.getProperty("os.name");
if (osName.startsWith("Mac"))
{
Constants.VIDEO_WIDTH = 320;
Constants.VIDEO_HEIGHT = 240;
}
Format targetFormat = null;
double targetWidth = Constants.VIDEO_WIDTH;
Format[] fs = fControl.getSupportedFormats();
for (int i = 0; i < fs.length; i++)
{
VideoFormat format = (VideoFormat)fs[i];
Dimension size = format.getSize();
if(size.getWidth() >= Constants.VIDEO_WIDTH
&& size.getWidth() <= targetWidth)
{
targetFormat = format;
}
}
if(targetFormat != null)
fControl.setFormat(targetFormat);
}
}

@ -27,4 +27,7 @@ public class Constants
* 30 - 30 ms
*/
public static int ILBC_MODE = 30;
public static int VIDEO_WIDTH = 352;
public static int VIDEO_HEIGHT = 288;
}

@ -37,11 +37,14 @@ public class ImageScaler extends AbstractCodec implements Codec
new RGBFormat(new Dimension(704, 576), -1, Format.intArray, -1.0f, 32, -1, -1, -1),
//CIF
new RGBFormat(new Dimension(352, 288), -1, Format.intArray, -1.0f, 32, -1, -1, -1),
new RGBFormat(new Dimension(320, 240), -1, Format.intArray, -1.0f, 32, -1, -1, -1),
//QCIF
new RGBFormat(new Dimension(176, 144), -1, Format.intArray, -1.0f, 32, -1, -1, -1),
//SQCIF
new RGBFormat(new Dimension(128, 96), -1, Format.intArray, -1.0f, 32, -1, -1, -1),
};
private boolean toProcess = true;
@Override
public Format[] getSupportedInputFormats()
@ -72,6 +75,14 @@ public Format setInputFormat(Format format)
return null; // must set a size.
// logger.fine("FORMAT: " + MediaCGUtils.formatToStr(format));
if(outputFormat != null)
{
VideoFormat outVFormat = (VideoFormat)outputFormat;
if(outVFormat.getSize() != null)
toProcess = !outVFormat.getSize().equals(videoFormat.getSize());
}
// TODO: check VideoFormat and compatibility
bufferToImage = new BufferToImage((VideoFormat) format);
return super.setInputFormat(format);
@ -91,6 +102,18 @@ public int process(Buffer input, Buffer output)
return BUFFER_PROCESSED_OK;
}
// sometimes format sizes are the same but some other field is different
// and jmf use the scaler (in my case length field was not sent in
// one of the formats) the check for sizes is made in method
// setInputFormat
if(!toProcess)
{
output.setData(input.getData());
output.setLength(input.getLength());
output.setOffset(input.getOffset());
return BUFFER_PROCESSED_OK;
}
final BufferedImage image = (BufferedImage) bufferToImage.createImage(input);
final Dimension inputSize = ((VideoFormat) inputFormat).getSize();

@ -27,7 +27,7 @@ public class H264Parser
private byte[] encodedFrame = new byte[MAX_FRAME_SIZE];
// the size of the result data
private int encodedFrameLen;
/**
* New rtp packet is received. We push it to the parser to extract the data.
* @param inputBuffer the data from the rtp packet
@ -36,12 +36,12 @@ public class H264Parser
public boolean pushRTPInput(Buffer inputBuffer)
{
long currentStamp = inputBuffer.getTimeStamp();
// the rtp marker field points that this is the last packet of
// the received frame
boolean hasMarker =
(inputBuffer.getFlags() & Buffer.FLAG_RTP_MARKER) != 0;
// if the timestamp changes we are starting receiving a new frame
if(!(currentStamp == lastTimestamp))
{
@ -50,33 +50,40 @@ public boolean pushRTPInput(Buffer inputBuffer)
}
// the new frame timestamp
lastTimestamp = currentStamp;
byte[] inData = (byte[]) inputBuffer.getData();
int inputOffset = inputBuffer.getOffset();
byte fByte = inData[inputOffset];
int type = fByte & 0x1f;
// types from 1 to 23 are treated the same way
if (type >= 1 && type <= 23)
{
System.arraycopy(startSequence, 0, encodedFrame, encodedFrameLen, startSequence.length);
encodedFrameLen += startSequence.length;
int len = inputBuffer.getLength();
System.arraycopy(inData, inputOffset, encodedFrame, encodedFrameLen, len);
encodedFrameLen += len;
}
else if (type == 24)
{
//return deencapsulateSTAP(inputBuffer);
}
else if (type == 28)
try
{
deencapsulateFU(fByte, inputBuffer);
// types from 1 to 23 are treated the same way
if (type >= 1 && type <= 23)
{
System.arraycopy(startSequence, 0, encodedFrame, encodedFrameLen, startSequence.length);
encodedFrameLen += startSequence.length;
int len = inputBuffer.getLength();
System.arraycopy(inData, inputOffset, encodedFrame, encodedFrameLen, len);
encodedFrameLen += len;
}
else if (type == 24)
{
//return deencapsulateSTAP(inputBuffer);
}
else if (type == 28)
{
deencapsulateFU(fByte, inputBuffer);
}
else
{
logger.warn("Skipping unsupported NAL unit type");
return false;
}
}
else
catch(Exception ex)
{
logger.warn("Skipping unsupported NAL unit type");
return false;
logger.warn("Cannot parse incoming " + ex.getMessage());
return true;
}
if(hasMarker)
@ -88,7 +95,7 @@ else if (type == 28)
return false;
}
}
/**
* Extract data from FU packet. This are packets accross several
* rtp packets, the fisrt has a start bit set, we store all data
@ -101,10 +108,10 @@ private void deencapsulateFU (byte nal, Buffer inputBuffer)
byte[] buf = (byte[])inputBuffer.getData();
int len = inputBuffer.getLength();
int offset = inputBuffer.getOffset();
offset++;
len--;
byte fu_indicator = nal;
byte fu_header = buf[offset];
boolean start_bit = (fu_header >> 7) != 0;
@ -115,11 +122,11 @@ private void deencapsulateFU (byte nal, Buffer inputBuffer)
//the original nal forbidden bit and NRI are stored in this packet's nal;
reconstructed_nal = (byte)(fu_indicator & (byte)0xe0);
reconstructed_nal |= nal_type;
// skip the fu_header...
offset++;
len--;
if(start_bit)
{
// copy in the start sequence, and the reconstructed nal....

@ -24,12 +24,12 @@ public class NativeDecoder
extends VideoCodec
{
private final Logger logger = Logger.getLogger(NativeDecoder.class);
// Instances for the ffmpeg lib
private AVFormatLibrary AVFORMAT;
private AVCodecLibrary AVCODEC;
private AVUtilLibrary AVUTIL;
// The codec we will use
private AVCodec avcodec;
private AVCodecContext avcontext;
@ -37,10 +37,10 @@ public class NativeDecoder
private AVFrame avpicture;
// Used to convert decoded data to RGB
private AVFrame frameRGB;
// The parser used to parse rtp content
private H264Parser parser = new H264Parser();
// supported sizes by the codec
private Dimension[] supportedSizes = new Dimension[]
{
@ -50,15 +50,19 @@ public class NativeDecoder
new Dimension(704, 576),
//CIF
new Dimension(352, 288),
new Dimension(320, 240),
//QCIF
new Dimension(176, 144),
//SQCIF
new Dimension(128, 96)
};
// index of default size (output format)
private static final int defaultSizeIx = 2;
private static int defaultSizeIx = 2;
// current width of video, so we can detect changes in video size
private double currentVideoWidth;
/**
* Constructs new h264 decoder
*/
@ -69,17 +73,24 @@ public NativeDecoder()
{
new VideoFormat(Constants.H264_RTP)
};
defaultOutputFormats = new VideoFormat[]
{
new RGBFormat()
};
supportedOutputFormats = new VideoFormat[supportedSizes.length];
Dimension targetVideoSize =
new Dimension(Constants.VIDEO_WIDTH, Constants.VIDEO_HEIGHT);
for (int i = 0; i < supportedSizes.length; i++)
{
Dimension size = supportedSizes[i];
if(size.equals(targetVideoSize))
defaultSizeIx = i;
supportedOutputFormats[i] =
//PIX_FMT_RGB32
new RGBFormat(
@ -96,7 +107,10 @@ public NativeDecoder()
Format.FALSE,
Format.NOT_SPECIFIED);
}
currentVideoWidth =
supportedOutputFormats[defaultSizeIx].getSize().getWidth();
PLUGIN_NAME = "H.264 Decoder";
AVFORMAT = AVFormatLibrary.INSTANCE;
@ -111,7 +125,7 @@ public NativeDecoder()
protected Format[] getMatchingOutputFormats(Format in)
{
VideoFormat ivf = (VideoFormat) in;
// return the default size/currently decoder and encoder
//set to transmit/receive at this size
if(ivf.getSize() == null)
@ -143,6 +157,13 @@ public Format setInputFormat(Format format)
else
return null;
}
@Override
public Format setOutputFormat(Format format)
{
return super.setOutputFormat(format);
}
/**
* Init the codec instances.
@ -150,27 +171,27 @@ public Format setInputFormat(Format format)
public void open() throws ResourceUnavailableException
{
avcodec = AVCODEC.avcodec_find_decoder(AVCodecLibrary.CODEC_ID_H264);
avcontext = AVCODEC.avcodec_alloc_context();
if (AVCODEC.avcodec_open(avcontext, avcodec) < 0)
throw new RuntimeException("Could not open codec ");
avpicture = AVCODEC.avcodec_alloc_frame();
avcontext.lowres = 1;
avcontext.workaround_bugs = 1;
AVUTIL.av_log_set_callback(new AVUtilLibrary.LogCallback()
{
avcontext.lowres = 1;
public void callback(Pointer p, int l, String fmtS, Pointer va_list)
{
logger.info("decoder: " + fmtS);
avcontext.workaround_bugs = 1;
}
});
// AVUTIL.av_log_set_callback(new AVUtilLibrary.LogCallback()
// {
//
// public void callback(Pointer p, int l, String fmtS, Pointer va_list)
// {
// logger.info("decoder: " + fmtS);
//
// }
// });
frameRGB = AVCODEC.avcodec_alloc_frame();
@ -192,7 +213,7 @@ public void close()
}
public int process(Buffer inputBuffer, Buffer outputBuffer)
{
{
if (!checkInputBuffer(inputBuffer))
{
return BUFFER_PROCESSED_FAILED;
@ -203,7 +224,7 @@ public int process(Buffer inputBuffer, Buffer outputBuffer)
propagateEOM(outputBuffer);
return BUFFER_PROCESSED_OK;
}
if(!parser.pushRTPInput(inputBuffer))
{
return OUTPUT_BUFFER_NOT_FILLED;
@ -213,15 +234,25 @@ public int process(Buffer inputBuffer, Buffer outputBuffer)
IntByReference got_picture = new IntByReference();
Pointer encBuf = AVUTIL.av_malloc(parser.getEncodedFrameLen());
arraycopy(parser.getEncodedFrame(), 0, encBuf, 0, parser.getEncodedFrameLen());
// decodes the data
AVCODEC.avcodec_decode_video(
avcontext, avpicture, got_picture, encBuf, parser.getEncodedFrameLen());
if(currentVideoWidth != avcontext.width)
{
currentVideoWidth = avcontext.width;
VideoFormat format = getVideoFormat(currentVideoWidth);
if(format != null)
{
outputBuffer.setFormat(format);
outputFormat = format;
}
}
if(got_picture.getValue() == 0)
{
outputBuffer.setDiscard(true);
AVUTIL.av_free(encBuf);
return BUFFER_PROCESSED_OK;
@ -231,7 +262,7 @@ public int process(Buffer inputBuffer, Buffer outputBuffer)
int numBytes = AVCODEC.avpicture_get_size(
AVCodecLibrary.PIX_FMT_RGB32, avcontext.width, avcontext.height);
Pointer buffer = AVUTIL.av_malloc(numBytes);
AVCODEC.avpicture_fill(
frameRGB, buffer, AVCodecLibrary.PIX_FMT_RGB32, avcontext.width, avcontext.height);
@ -239,19 +270,19 @@ public int process(Buffer inputBuffer, Buffer outputBuffer)
AVCODEC.img_convert(frameRGB, AVCodecLibrary.PIX_FMT_RGB32,
avpicture, avcontext.pix_fmt, avcontext.width,
avcontext.height);
int[] data =
frameRGB.data0.getIntArray(0, avcontext.height * avcontext.width);
int[] outData = validateIntArraySize(outputBuffer, data.length);
System.arraycopy(data, 0, outData, 0, data.length);
outputBuffer.setOffset(0);
outputBuffer.setLength(outData.length);
outputBuffer.setData(outData);
AVUTIL.av_free(encBuf);
AVUTIL.av_free(buffer);
return BUFFER_PROCESSED_OK;
}
@ -285,4 +316,15 @@ public boolean checkFormat(Format format)
return super.checkFormat(format);
}
}
private VideoFormat getVideoFormat(double width)
{
for (int i = 0; i < supportedOutputFormats.length; i++)
{
VideoFormat vf = supportedOutputFormats[i];
if(vf.getSize().getWidth() == width)
return vf;
}
return null;
}
}

@ -24,25 +24,14 @@ public class NativeEncoder
{
private final Logger logger = Logger.getLogger(NativeEncoder.class);
// supported formats
// private final static int P720_WIDTH = 720;
// private final static int P720_HEIGHT = 480;
// private final static int CIF4_WIDTH = 704;
// private final static int CIF4_HEIGHT = 576;
private final static int CIF_WIDTH = 352;
private final static int CIF_HEIGHT = 288;
// private final static int QCIF_WIDTH = 176;
// private final static int QCIF_HEIGHT = 144;
// private final static int SQCIF_WIDTH = 128;
// private final static int SQCIF_HEIGHT = 96;
private final static String PLUGIN_NAME = "H264 Encoder";
private final static int DEF_WIDTH = CIF_WIDTH;
private final static int DEF_HEIGHT = CIF_HEIGHT;
private static int DEF_WIDTH = 352;
private static int DEF_HEIGHT = 288;
// without the headers
private final static int MAX_PAYLOAD_SIZE = 1400;
// private final static int MAX_PAYLOAD_SIZE = 1400;
private final static int MAX_PAYLOAD_SIZE = 984;
private final static int INPUT_BUFFER_PADDING_SIZE = 8;
@ -80,11 +69,18 @@ public class NativeEncoder
// the current rtp sequence
private int seq = 0;
private static int IFRAME_INTERVAL = 125;
private int framesSinceLastIFrame = 0;
/**
* Constructor
*/
public NativeEncoder()
{
DEF_WIDTH = Constants.VIDEO_WIDTH;
DEF_HEIGHT = Constants.VIDEO_HEIGHT;
int strideY = DEF_WIDTH;
int strideUV = strideY / 2;
int offsetU = strideY * DEF_HEIGHT;
@ -93,23 +89,22 @@ public NativeEncoder()
int inputYuvLength = (strideY + strideUV) * DEF_HEIGHT;
float sourceFrameRate = TARGET_FRAME_RATE;
inputFormats = new Format[]{
new YUVFormat(new Dimension(DEF_WIDTH, DEF_HEIGHT),
inputYuvLength + INPUT_BUFFER_PADDING_SIZE,
Format.byteArray, sourceFrameRate, YUVFormat.YUV_420, strideY,
strideUV, 0, offsetU, offsetV)
};
inputFormat = null;
outputFormat = null;
AVFORMAT = AVFormatLibrary.INSTANCE;
AVCODEC = AVCodecLibrary.INSTANCE;
AVUTIL = AVUtilLibrary.INSTANCE;
AVFORMAT.av_register_all();
AVCODEC.avcodec_init();
}
@ -218,31 +213,36 @@ public int process(Buffer inBuffer, Buffer outBuffer)
{
int bufOfset = 0;
int size = buf.length;
int nri = buf[bufOfset] & 0x60;
byte[] tmp = new byte[MAX_PAYLOAD_SIZE];
tmp[0] = 28; /* FU Indicator; Type = 28 ---> FU-A */
tmp[0] |= nri;
tmp[1] = buf[bufOfset];
// set start bit
tmp[1] |= 1 << 7;
// remove end bit
tmp[1] &= ~(1 << 6);
bufOfset += 1;
size -= 1;
int currentSIx = 0;
while (size + 2 > MAX_PAYLOAD_SIZE)
{
System.arraycopy(buf, bufOfset, tmp, 2, MAX_PAYLOAD_SIZE - 2);
nals.add(currentSIx++, tmp.clone());
bufOfset += MAX_PAYLOAD_SIZE - 2;
size -= MAX_PAYLOAD_SIZE - 2;
tmp[1] &= ~(1 << 7);
}
byte[] tmp2 = new byte[size + 2];
tmp2[0] = tmp[0];
tmp2[1] = tmp[1];
tmp2[1] |= 1 << 6;
System.arraycopy(buf, bufOfset, tmp2, 2, size);
nals.add(currentSIx++, tmp2);
@ -268,7 +268,7 @@ public int process(Buffer inBuffer, Buffer outBuffer)
return BUFFER_PROCESSED_OK;
}
}
if (isEOM(inBuffer))
{
propagateEOM(outBuffer);
@ -299,12 +299,23 @@ public int process(Buffer inBuffer, Buffer outBuffer)
// copy data to avpicture
rawFrameBuffer.write(0,
(byte[])inBuffer.getData(), inBuffer.getOffset(), encFrameLen);
if(framesSinceLastIFrame >= IFRAME_INTERVAL )
{
avpicture.key_frame = 1;
framesSinceLastIFrame = 0;
}
else
{
framesSinceLastIFrame++;
avpicture.key_frame = 0;
}
// encode data
int encLen =
AVCODEC.avcodec_encode_video(
avcontext, encFrameBuffer, encFrameLen, avpicture);
byte[] r = encFrameBuffer.getByteArray(0, encLen);
// split encoded data to nals
@ -321,8 +332,15 @@ public int process(Buffer inBuffer, Buffer outBuffer)
prevIx = ix;
ix = prevIx + 3;
}
int len = ix - prevIx;
if(len < 0)
{
logger.warn("aaaa");
outBuffer.setDiscard(true);
return BUFFER_PROCESSED_OK;
}
byte[] b = new byte[len - 3];
System.arraycopy(r, prevIx+ 3, b, 0, len-3);
nals.add(b);
@ -365,65 +383,100 @@ public synchronized void open() throws ResourceUnavailableException
if (outputFormat == null)
throw new ResourceUnavailableException(
"No output format selected");
AVCODEC.avcodec_init();
avcodec = AVCODEC.avcodec_find_encoder(AVCodecLibrary.CODEC_ID_H264);
avcontext = AVCODEC.avcodec_alloc_context();
avpicture = AVCODEC.avcodec_alloc_frame();
avcontext.crf = 1f;
avcontext.pix_fmt = AVFormatLibrary.PIX_FMT_YUV420P;
avcontext.width = DEF_WIDTH;
avcontext.height = DEF_HEIGHT;
avcontext.time_base = new FFMPEGLibrary.AVRational(1, TARGET_FRAME_RATE);
avpicture.linesize[0] = DEF_WIDTH;
avpicture.linesize[1] = DEF_WIDTH / 2;
avpicture.linesize[2] = DEF_WIDTH / 2;
//avpicture.quality = (int)10;
avcontext.qcompress = 1;
int _bitRate = 48000;
avcontext.bit_rate = _bitRate; // average bit rate
avcontext.bit_rate_tolerance = _bitRate;// so to be 1 in x264
avcontext.rc_max_rate = _bitRate;
avcontext.sample_aspect_ratio.den = 0;
avcontext.sample_aspect_ratio.num = 0;
avcontext.thread_count = 0;
avcontext.time_base.den = 15;//25500; //???
avcontext.time_base.num = 1000;
avcontext.qmin = 10;
avcontext.qmax = 22;
avcontext.max_qdiff = 4;
int _bitRate = 267000;
avcontext.bit_rate = (_bitRate * 3) >> 2; // average bit rate
avcontext.partitions |= 0x111;
//X264_PART_I4X4 0x001
//X264_PART_P8X8 0x010
//X264_PART_B8X8 0x100
AVUTIL.av_log_set_callback(new AVUtilLibrary.LogCallback(){
public void callback(Pointer p, int l, String fmtS,
Pointer va_list)
{
logger.info("encoder: " + fmtS);
}});
avcontext.flags |= AVCodecLibrary.CODEC_FLAG_PASS1;
avcontext.mb_decision = AVCodecContext.FF_MB_DECISION_SIMPLE;
avcontext.rc_eq = "blurCplx^(1-qComp)";
avcontext.flags |= AVCodecLibrary.CODEC_FLAG_LOOP_FILTER;
avcontext.me_method = 1;
avcontext.me_subpel_quality = 5;
avcontext.me_range = 16;
avcontext.me_cmp |= AVCodecContext.FF_CMP_CHROMA;
avcontext.thread_count = 1;
avcontext.scenechange_threshold = 40;
avcontext.crf = 0;// Constant quality mode (also known as constant ratefactor)
//avcontext.rc_buffer_size = _bitRate * 64;
avcontext.gop_size = IFRAME_INTERVAL;
framesSinceLastIFrame = IFRAME_INTERVAL + 1;
avcontext.i_quant_factor = 1f/1.4f;
// AVUTIL.av_log_set_callback(new AVUtilLibrary.LogCallback(){
//
// public void callback(Pointer p, int l, String fmtS,
// Pointer va_list)
// {
// logger.info("encoder: " + fmtS);
// }});
if (AVCODEC.avcodec_open(avcontext, avcodec) < 0)
throw new RuntimeException("Could not open codec ");
opened = true;
encFrameLen = (DEF_WIDTH * DEF_HEIGHT * 3) / 2;
rawFrameBuffer = AVUTIL.av_malloc(encFrameLen);
encFrameBuffer = AVUTIL.av_malloc(encFrameLen);
int size = DEF_WIDTH * DEF_HEIGHT;
avpicture.data0 = rawFrameBuffer;
avpicture.data1 = avpicture.data0.share(size);
avpicture.data2 = avpicture.data1.share(size/4);
}
}
public synchronized void close()
{
if (opened)
{
super.close();
AVCODEC.avcodec_close(avcontext);
AVUTIL.av_free(avpicture.getPointer());
AVUTIL.av_free(avcontext.getPointer());
AVUTIL.av_free(rawFrameBuffer);
AVUTIL.av_free(encFrameBuffer);
}

Loading…
Cancel
Save