From 4e39c83ba9029f8fc01ac942bfa40e3a6676c77f Mon Sep 17 00:00:00 2001 From: Damian Minkov Date: Fri, 17 Feb 2012 10:19:07 +0000 Subject: [PATCH] Makes possible making video call without audio, and sharing the desktop from machine without audio device. Updates zrtp to use the first available stream as master, if more than one stream use the audio one. --- .../impl/neomedia/ZrtpControlImpl.java | 35 ++++-- .../neomedia/notify/PortAudioClipImpl.java | 8 +- .../transform/sdes/SDesControlImpl.java | 17 ++- .../jabber/CallPeerMediaHandlerGTalkImpl.java | 56 +++++++-- .../CallPeerMediaHandlerJabberImpl.java | 114 ++++++++++++++++-- .../sip/CallPeerMediaHandlerSipImpl.java | 46 ++++++- .../service/neomedia/SrtpControl.java | 11 +- .../protocol/media/CallPeerMediaHandler.java | 14 ++- .../protocol/media/MediaAwareCall.java | 5 +- 9 files changed, 264 insertions(+), 42 deletions(-) diff --git a/src/net/java/sip/communicator/impl/neomedia/ZrtpControlImpl.java b/src/net/java/sip/communicator/impl/neomedia/ZrtpControlImpl.java index cd24643ff..3e90c7109 100644 --- a/src/net/java/sip/communicator/impl/neomedia/ZrtpControlImpl.java +++ b/src/net/java/sip/communicator/impl/neomedia/ZrtpControlImpl.java @@ -51,6 +51,11 @@ public static enum ZRTPCustomInfoCodes */ private AbstractRTPConnector zrtpConnector = null; + /** + * Whether current is master session. + */ + private boolean masterSession = false; + /** * Creates the control. */ @@ -139,14 +144,25 @@ public ZRTPTransformEngine getTransformEngine() } /** - * Starts and enables zrtp in the stream holding this control. - * @param masterSession whether this stream is master for the current - * media session. + * When in multistream mode, enables the master session. + * @param masterSession whether current control, controls the master session. */ - public void start(boolean masterSession) + public void setMasterSession(boolean masterSession) { + // by default its not master, change only if set to be master + // sometimes (jingle) streams are re-initing and + // we must reuse old value (true) event that false is submitted + if(masterSession) + this.masterSession = masterSession; + } - boolean zrtpAutoStart = false; + /** + * Starts and enables zrtp in the stream holding this control. + * @param mediaType the media type of the stream this control controls. + */ + public void start(MediaType mediaType) + { + boolean zrtpAutoStart; // ZRTP engine initialization ZRTPTransformEngine engine = getTransformEngine(); @@ -170,7 +186,10 @@ public void start(boolean masterSession) // we know that audio is considered as master for zrtp securityEventManager.setSessionType( - SecurityEventManager.AUDIO_SESSION); + mediaType.equals(MediaType.AUDIO) ? + SecurityEventManager.AUDIO_SESSION + : SecurityEventManager.VIDEO_SESSION + ); } else { @@ -180,7 +199,9 @@ public void start(boolean masterSession) // initially engine has value enableZrtp = false zrtpAutoStart = zrtpEngine.isEnableZrtp(); securityEventManager.setSessionType( - SecurityEventManager.VIDEO_SESSION); + mediaType.equals(MediaType.AUDIO) ? + SecurityEventManager.AUDIO_SESSION + : SecurityEventManager.VIDEO_SESSION); } // tells the engine whether to autostart(enable) diff --git a/src/net/java/sip/communicator/impl/neomedia/notify/PortAudioClipImpl.java b/src/net/java/sip/communicator/impl/neomedia/notify/PortAudioClipImpl.java index a22a8fac8..e449aeb37 100644 --- a/src/net/java/sip/communicator/impl/neomedia/notify/PortAudioClipImpl.java +++ b/src/net/java/sip/communicator/impl/neomedia/notify/PortAudioClipImpl.java @@ -193,12 +193,16 @@ private boolean runOnceInPlayThread( * If the user has configured PortAudio to use no notification device, * don't try to play this clip. */ - MediaLocator rendererLocator + CaptureDeviceInfo audioNotifyDeviceInfo = audioNotifier - .getDeviceConfiguration().getAudioNotifyDevice().getLocator(); + .getDeviceConfiguration().getAudioNotifyDevice(); + if(audioNotifyDeviceInfo == null) + return false; + MediaLocator rendererLocator = audioNotifyDeviceInfo.getLocator(); if (rendererLocator == null) return false; + renderer.setLocator(rendererLocator); AudioInputStream audioStream = null; diff --git a/src/net/java/sip/communicator/impl/neomedia/transform/sdes/SDesControlImpl.java b/src/net/java/sip/communicator/impl/neomedia/transform/sdes/SDesControlImpl.java index ceb051bce..666bbc074 100644 --- a/src/net/java/sip/communicator/impl/neomedia/transform/sdes/SDesControlImpl.java +++ b/src/net/java/sip/communicator/impl/neomedia/transform/sdes/SDesControlImpl.java @@ -14,9 +14,9 @@ import net.java.sip.communicator.impl.neomedia.*; import net.java.sip.communicator.impl.neomedia.transform.*; +import net.java.sip.communicator.impl.neomedia.transform.zrtp.*; import net.java.sip.communicator.service.neomedia.*; import net.java.sip.communicator.service.neomedia.event.*; -import net.java.sip.communicator.service.protocol.event.*; /** * Default implementation of {@link SDesControl} that supports the crypto suites @@ -93,12 +93,19 @@ public boolean getSecureCommunicationStatus() return engine != null; } - public void start(boolean masterSession) + /** + * Not used. + * @param masterSession not used. + */ + public void setMasterSession(boolean masterSession) + {} + + public void start(MediaType type) { srtpListener.securityTurnedOn( - masterSession ? - CallPeerSecurityStatusEvent.AUDIO_SESSION : - CallPeerSecurityStatusEvent.VIDEO_SESSION, + type.equals(MediaType.AUDIO) ? + SecurityEventManager.AUDIO_SESSION + : SecurityEventManager.VIDEO_SESSION, selectedInAttribute.getCryptoSuite().encode(), this); } diff --git a/src/net/java/sip/communicator/impl/protocol/jabber/CallPeerMediaHandlerGTalkImpl.java b/src/net/java/sip/communicator/impl/protocol/jabber/CallPeerMediaHandlerGTalkImpl.java index acd387209..2e2907e7d 100644 --- a/src/net/java/sip/communicator/impl/protocol/jabber/CallPeerMediaHandlerGTalkImpl.java +++ b/src/net/java/sip/communicator/impl/protocol/jabber/CallPeerMediaHandlerGTalkImpl.java @@ -230,7 +230,8 @@ public RtpDescriptionPacketExtension generateSessionAccept( List lst = localContentMap.get("audio"); description.setNamespace(SessionIQProvider.GTALK_AUDIO_NAMESPACE); - + + boolean masterStreamSet = false; for(MediaType mediaType : MediaType.values()) { MediaFormat format = null; @@ -291,8 +292,27 @@ public RtpDescriptionPacketExtension generateSessionAccept( new ArrayList(); MediaDirection direction = MediaDirection.SENDRECV; + boolean masterStream = false; + // if we have more than one stream, lets the audio be the master + if(!masterStreamSet) + { + if(MediaType.values().length > 1) + { + if(mediaType.equals(MediaType.AUDIO)) + { + masterStream = true; + masterStreamSet = true; + } + } + else + { + masterStream = true; + masterStreamSet = true; + } + } + initStream(mediaName, connector, dev, format, target, - direction, rtpExtensions); + direction, rtpExtensions, masterStream); } return description; @@ -319,6 +339,7 @@ public void processAnswer(RtpDescriptionPacketExtension answer) { List lst = answer.getPayloadTypes(); + boolean masterStreamSet = true; for(MediaType mediaType : MediaType.values()) { String ns = getNamespaceForMediaType(mediaType); @@ -360,8 +381,27 @@ public void processAnswer(RtpDescriptionPacketExtension answer) List rtpExtensions = new ArrayList(); MediaDirection direction = MediaDirection.SENDRECV; + boolean masterStream = false; + // if we have more than one stream, lets the audio be the master + if(!masterStreamSet) + { + if(MediaType.values().length > 1) + { + if(mediaType.equals(MediaType.AUDIO)) + { + masterStream = true; + masterStreamSet = true; + } + } + else + { + masterStream = true; + masterStreamSet = true; + } + } + initStream(mediaName, connector, dev, format, target, - direction, rtpExtensions); + direction, rtpExtensions, masterStream); } } @@ -565,8 +605,7 @@ else if(mediaType == MediaType.VIDEO) * Create list of payload types for device. * * @param supportedFormats supported formats of a device - * @param direction direction - * @param supportedExtensions supported RTP extensions + * @param name name of payload type * @return list of payload types for this device */ private List createPayloadTypesForOffer( @@ -663,6 +702,7 @@ private void wrapupConnectivityEstablishment() * stream to use (i.e. sendonly, sendrecv, recvonly, or inactive). * @param rtpExtensions the list of RTPExtensions that should be * enabled for this stream. + * @param masterStream whether the stream to be used as master if secured * * @return the newly created MediaStream. * @@ -675,7 +715,8 @@ protected MediaStream initStream(String streamName, MediaFormat format, MediaStreamTarget target, MediaDirection direction, - List rtpExtensions) + List rtpExtensions, + boolean masterStream) throws OperationFailedException { if(format instanceof VideoMediaFormat) @@ -696,7 +737,8 @@ protected MediaStream initStream(String streamName, format, target, direction, - rtpExtensions); + rtpExtensions, + masterStream); if(stream != null) stream.setName(streamName); diff --git a/src/net/java/sip/communicator/impl/protocol/jabber/CallPeerMediaHandlerJabberImpl.java b/src/net/java/sip/communicator/impl/protocol/jabber/CallPeerMediaHandlerJabberImpl.java index 8188c21b9..f4ac19eb6 100644 --- a/src/net/java/sip/communicator/impl/protocol/jabber/CallPeerMediaHandlerJabberImpl.java +++ b/src/net/java/sip/communicator/impl/protocol/jabber/CallPeerMediaHandlerJabberImpl.java @@ -193,6 +193,7 @@ public ContentPacketExtension getLocalContent(String contentType) * stream to use (i.e. sendonly, sendrecv, recvonly, or inactive). * @param rtpExtensions the list of RTPExtensions that should be * enabled for this stream. + * @param masterStream whether the stream to be used as master if secured * * @return the newly created MediaStream. * @@ -205,7 +206,8 @@ protected MediaStream initStream(String streamName, MediaFormat format, MediaStreamTarget target, MediaDirection direction, - List rtpExtensions) + List rtpExtensions, + boolean masterStream) throws OperationFailedException { MediaStream stream @@ -215,7 +217,8 @@ protected MediaStream initStream(String streamName, format, target, direction, - rtpExtensions); + rtpExtensions, + masterStream); if(stream != null) stream.setName(streamName); @@ -457,10 +460,27 @@ public Iterable generateSessionAccept() //user answered an incoming call so we go through whatever content //entries we are initializing and init their corresponding streams + + // First parse content so we know how may streams, + // and what type of content we have + Map contents + = new HashMap(); for(ContentPacketExtension ourContent : sessAccept) { RtpDescriptionPacketExtension description - = JingleUtils.getRtpDescription(ourContent); + = JingleUtils.getRtpDescription(ourContent); + contents.put(ourContent, description); + } + + boolean masterStreamSet = false; + for(Map.Entry en + : contents.entrySet()) + { + ContentPacketExtension ourContent = en.getKey(); + + RtpDescriptionPacketExtension description = en.getValue(); MediaType type = MediaType.parseString(description.getMedia()); // stream connector @@ -532,9 +552,28 @@ public Iterable generateSessionAccept() } } + boolean masterStream = false; + // if we have more than one stream, lets the audio be the master + if(!masterStreamSet) + { + if(contents.size() > 1) + { + if(type.equals(MediaType.AUDIO)) + { + masterStream = true; + masterStreamSet = true; + } + } + else + { + masterStream = true; + masterStreamSet = true; + } + } + // create the corresponding stream... initStream(ourContent.getName(), connector, dev, format, target, - direction, rtpExtensions); + direction, rtpExtensions, masterStream); // if remote peer requires inputevt, notify UI to capture mouse // and keyboard events @@ -887,12 +926,37 @@ public void reinitAllContents() throws OperationFailedException, IllegalArgumentException { + boolean masterStreamSet = false; for(String key : remoteContentMap.keySet()) { ContentPacketExtension ext = remoteContentMap.get(key); + boolean masterStream = false; + // if we have more than one stream, lets the audio be the master + if(!masterStreamSet) + { + RtpDescriptionPacketExtension description + = JingleUtils.getRtpDescription(ext); + MediaType mediaType + = MediaType.parseString( description.getMedia() ); + + if(remoteContentMap.size() > 1) + { + if(mediaType.equals(MediaType.AUDIO)) + { + masterStream = true; + masterStreamSet = true; + } + } + else + { + masterStream = true; + masterStreamSet = true; + } + } + if(ext != null) - processContent(ext, false); + processContent(ext, false, masterStream); } } @@ -924,13 +988,13 @@ public void reinitContent( { if(modify) { - processContent(content, modify); + processContent(content, modify, false); remoteContentMap.put(name, content); } else { ext.setSenders(content.getSenders()); - processContent(ext, modify); + processContent(ext, modify, false); remoteContentMap.put(name, ext); } } @@ -982,6 +1046,7 @@ private void removeContent( * * @param content a ContentPacketExtension * @param modify if it correspond to a content-modify for resolution change + * @param masterStream whether the stream to be used as master * @throws OperationFailedException if we fail to handle content * for reasons like failing to initialize media devices or streams. * @throws IllegalArgumentException if there's a problem with the syntax or @@ -991,7 +1056,8 @@ private void removeContent( * in this operation can synchronize to the mediaHandler instance to wait * processing to stop (method setState in CallPeer). */ - private void processContent(ContentPacketExtension content, boolean modify) + private void processContent(ContentPacketExtension content, boolean modify, + boolean masterStream) throws OperationFailedException, IllegalArgumentException { @@ -1114,7 +1180,8 @@ private void processContent(ContentPacketExtension content, boolean modify) // create the corresponding stream... initStream(content.getName(), connector, dev, - supportedFormats.get(0), target, direction, rtpExtensions); + supportedFormats.get(0), target, direction, rtpExtensions, + masterStream); } /** @@ -1141,12 +1208,37 @@ public void processAnswer(List answer) * information compatible with that carried in transport-info. */ processTransportInfo(answer); - + + boolean masterStreamSet = false; for (ContentPacketExtension content : answer) { remoteContentMap.put(content.getName(), content); - processContent(content, false); + boolean masterStream = false; + // if we have more than one stream, lets the audio be the master + if(!masterStreamSet) + { + RtpDescriptionPacketExtension description + = JingleUtils.getRtpDescription(content); + MediaType mediaType + = MediaType.parseString( description.getMedia() ); + + if(answer.size() > 1) + { + if(mediaType.equals(MediaType.AUDIO)) + { + masterStream = true; + masterStreamSet = true; + } + } + else + { + masterStream = true; + masterStreamSet = true; + } + } + + processContent(content, false, masterStream); } } diff --git a/src/net/java/sip/communicator/impl/protocol/sip/CallPeerMediaHandlerSipImpl.java b/src/net/java/sip/communicator/impl/protocol/sip/CallPeerMediaHandlerSipImpl.java index 92872475c..8da7ef854 100644 --- a/src/net/java/sip/communicator/impl/protocol/sip/CallPeerMediaHandlerSipImpl.java +++ b/src/net/java/sip/communicator/impl/protocol/sip/CallPeerMediaHandlerSipImpl.java @@ -415,6 +415,7 @@ private Vector createMediaDescriptionsForAnswer( .getAccountPropertyInt(ProtocolProviderFactory.SAVP_OPTION, ProtocolProviderFactory.SAVP_OFF); + boolean masterStreamSet = false; List seenMediaTypes = new ArrayList(); for (MediaDescription mediaDescription : remoteDescriptions) { @@ -590,7 +591,28 @@ private Vector createMediaDescriptionsForAnswer( // create the corresponding stream... MediaFormat fmt = findMediaFormat(remoteFormats, mutuallySupportedFormats.get(0)); - initStream(connector, dev, fmt, target, direction, rtpExtensions); + + boolean masterStream = false; + // if we have more than one stream, lets the audio be the master + if(!masterStreamSet) + { + if(remoteDescriptions.size() > 1) + { + if(mediaType.equals(MediaType.AUDIO)) + { + masterStream = true; + masterStreamSet = true; + } + } + else + { + masterStream = true; + masterStreamSet = true; + } + } + + initStream(connector, dev, fmt, target, direction, rtpExtensions, + masterStream); // create the answer description answerDescriptions.add(md); @@ -847,6 +869,7 @@ private synchronized void processAnswer(SessionDescription answer) this.setCallInfoURL(SdpUtils.getCallInfoURL(answer)); + boolean masterStreamSet = false; List seenMediaTypes = new ArrayList(); for (MediaDescription mediaDescription : remoteDescriptions) { @@ -992,9 +1015,28 @@ private synchronized void processAnswer(SessionDescription answer) } } + boolean masterStream = false; + // if we have more than one stream, lets the audio be the master + if(!masterStreamSet) + { + if(remoteDescriptions.size() > 1) + { + if(mediaType.equals(MediaType.AUDIO)) + { + masterStream = true; + masterStreamSet = true; + } + } + else + { + masterStream = true; + masterStreamSet = true; + } + } + // create the corresponding stream... initStream(connector, dev, supportedFormats.get(0), target, - direction, rtpExtensions); + direction, rtpExtensions, masterStream); } } diff --git a/src/net/java/sip/communicator/service/neomedia/SrtpControl.java b/src/net/java/sip/communicator/service/neomedia/SrtpControl.java index 85ca9387d..6d04d9251 100644 --- a/src/net/java/sip/communicator/service/neomedia/SrtpControl.java +++ b/src/net/java/sip/communicator/service/neomedia/SrtpControl.java @@ -46,12 +46,17 @@ public interface SrtpControl */ public boolean getSecureCommunicationStatus(); + /** + * When in multistream mode, enables the master session. + * @param masterSession whether current control, controls the master session. + */ + public void setMasterSession(boolean masterSession); + /** * Starts and enables zrtp in the stream holding this control. - * @param masterSession whether this stream is master for the current - * media session. + * @param mediaType the media type of the stream this control controls. */ - public void start(boolean masterSession); + public void start(MediaType mediaType); /** * Sets the multistream data, which means that the master stream diff --git a/src/net/java/sip/communicator/service/protocol/media/CallPeerMediaHandler.java b/src/net/java/sip/communicator/service/protocol/media/CallPeerMediaHandler.java index d19c4d9b8..a97bce439 100644 --- a/src/net/java/sip/communicator/service/protocol/media/CallPeerMediaHandler.java +++ b/src/net/java/sip/communicator/service/protocol/media/CallPeerMediaHandler.java @@ -1236,6 +1236,7 @@ protected Map getSrtpControls() * stream to use (i.e. sendonly, sendrecv, recvonly, or inactive). * @param rtpExtensions the list of RTPExtensions that should be * enabled for this stream. + * @param masterStream whether the stream to be used as master if secured * * @return the newly created MediaStream. * @@ -1247,7 +1248,8 @@ protected MediaStream initStream(StreamConnector connector, MediaFormat format, MediaStreamTarget target, MediaDirection direction, - List rtpExtensions) + List rtpExtensions, + boolean masterStream) throws OperationFailedException { MediaType mediaType = device.getMediaType(); @@ -1284,7 +1286,8 @@ protected MediaStream initStream(StreamConnector connector, return configureStream( - device, format, target, direction, rtpExtensions, stream); + device, format, target, direction, rtpExtensions, stream, + masterStream); } /** @@ -1303,6 +1306,7 @@ protected MediaStream initStream(StreamConnector connector, * @param rtpExtensions the list of RTPExtensions that should be * enabled for this stream. * @param stream the MediaStream that we'd like to configure. + * @param masterStream whether the stream to be used as master if secured * * @return the MediaStream that we received as a parameter (for * convenience reasons). @@ -1316,7 +1320,8 @@ protected MediaStream configureStream( MediaDevice device, MediaStreamTarget target, MediaDirection direction, List rtpExtensions, - MediaStream stream) + MediaStream stream, + boolean masterStream) throws OperationFailedException { registerDynamicPTsWithStream(stream); @@ -1355,8 +1360,9 @@ protected MediaStream configureStream( MediaDevice device, */ SrtpControl srtpControl = stream.getSrtpControl(); + srtpControl.setMasterSession(masterStream); srtpControl.setSrtpListener(srtpListener); - srtpControl.start(MediaType.AUDIO.equals(mediaType)); + srtpControl.start(mediaType); } return stream; diff --git a/src/net/java/sip/communicator/service/protocol/media/MediaAwareCall.java b/src/net/java/sip/communicator/service/protocol/media/MediaAwareCall.java index 9c6cb5bad..9109ff459 100644 --- a/src/net/java/sip/communicator/service/protocol/media/MediaAwareCall.java +++ b/src/net/java/sip/communicator/service/protocol/media/MediaAwareCall.java @@ -495,7 +495,10 @@ public MediaDevice getDefaultDevice(MediaType mediaType) */ if ((conferenceAudioMixer == null) && (device != null) - && (!OSUtils.IS_ANDROID || isConferenceFocus())) + && (!OSUtils.IS_ANDROID || isConferenceFocus()) + // we can use audio mixer only if we + // have capture device (device can send) + && (device.getDirection().allowsSending())) conferenceAudioMixer = mediaService.createMixer(device); if (conferenceAudioMixer != null) device = conferenceAudioMixer;