Commits

Anonymous committed b596472 Draft

Bug 801843: Change how video frames are inserted into getUserMedia streams to remove blocking r=roc,anant

  • Participants
  • Parent commits af26f70

Comments (0)

Files changed (7)

File content/media/webrtc/MediaEngine.h

    */
   virtual nsresult Snapshot(uint32_t aDuration, nsIDOMFile** aFile) = 0;
 
+  /* Called when the stream wants more data */
+  virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime) = 0;
+
   /* Stop the device and release the corresponding MediaStream */
   virtual nsresult Stop() = 0;
 

File content/media/webrtc/MediaEngineDefault.cpp

   return NS_OK;
 }
 
-NS_IMPL_THREADSAFE_ISUPPORTS1(MediaEngineDefaultAudioSource, nsITimerCallback)
+void
+MediaEngineDefaultVideoSource::NotifyPull(MediaStreamGraph* aGraph,
+                                          StreamTime aDesiredTime)
+{
+  // Ignore - we push video data
+}
+
+
 /**
  * Default audio source.
  */
+NS_IMPL_THREADSAFE_ISUPPORTS1(MediaEngineDefaultAudioSource, nsITimerCallback)
+
 MediaEngineDefaultAudioSource::MediaEngineDefaultAudioSource()
   : mTimer(nullptr)
 {
 {}
 
 void
+MediaEngineDefaultAudioSource::NotifyPull(MediaStreamGraph* aGraph,
+                                          StreamTime aDesiredTime)
+{
+  // Ignore - we push audio data
+}
+
+void
 MediaEngineDefaultAudioSource::GetName(nsAString& aName)
 {
   aName.Assign(NS_LITERAL_STRING("Default Audio Device"));

File content/media/webrtc/MediaEngineDefault.h

   virtual nsresult Start(SourceMediaStream*, TrackID);
   virtual nsresult Stop();
   virtual nsresult Snapshot(uint32_t aDuration, nsIDOMFile** aFile);
+  virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime);
 
   NS_DECL_ISUPPORTS
   NS_DECL_NSITIMERCALLBACK
   virtual nsresult Start(SourceMediaStream*, TrackID);
   virtual nsresult Stop();
   virtual nsresult Snapshot(uint32_t aDuration, nsIDOMFile** aFile);
+  virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime);
 
   NS_DECL_ISUPPORTS
   NS_DECL_NSITIMERCALLBACK

File content/media/webrtc/MediaEngineWebRTC.h

                                      public nsRunnable
 {
 public:
-  static const int DEFAULT_VIDEO_FPS = 30;
+  static const int DEFAULT_VIDEO_FPS = 60;
   static const int DEFAULT_MIN_VIDEO_FPS = 10;
 
   // ViEExternalRenderer.
     , mCapabilityChosen(false)
     , mWidth(640)
     , mHeight(480)
+    , mLastEndTime(0)
     , mMonitor("WebRTCCamera.Monitor")
     , mFps(DEFAULT_VIDEO_FPS)
     , mMinFps(aMinFps)
   virtual nsresult Start(SourceMediaStream*, TrackID);
   virtual nsresult Stop();
   virtual nsresult Snapshot(uint32_t aDuration, nsIDOMFile** aFile);
+  virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime);
 
   NS_DECL_ISUPPORTS
 
   bool mCapabilityChosen;
   int mWidth, mHeight;
   TrackID mTrackID;
+  TrackTicks mLastEndTime;
 
   mozilla::ReentrantMonitor mMonitor; // Monitor for processing WebRTC frames.
   SourceMediaStream* mSource;
   bool mInSnapshotMode;
   nsString* mSnapshotPath;
 
+  nsRefPtr<layers::Image> mImage;
   nsRefPtr<layers::ImageContainer> mImageContainer;
 
   PRLock* mSnapshotLock;
   virtual nsresult Start(SourceMediaStream*, TrackID);
   virtual nsresult Stop();
   virtual nsresult Snapshot(uint32_t aDuration, nsIDOMFile** aFile);
+  virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime);
 
   // VoEMediaProcess.
   void Process(const int channel, const webrtc::ProcessingTypes type,

File content/media/webrtc/MediaEngineWebRTCAudio.cpp

   segment->Init(CHANNELS);
   mSource->AddTrack(aID, SAMPLE_FREQUENCY, 0, segment);
   mSource->AdvanceKnownTracksTime(STREAM_TIME_MAX);
+  LOG(("Initial audio"));
   mTrackID = aID;
 
   if (mVoEBase->StartReceive(mChannel)) {
   return NS_OK;
 }
 
+void
+MediaEngineWebRTCAudioSource::NotifyPull(MediaStreamGraph* aGraph,
+                                         StreamTime aDesiredTime)
+{
+  // Ignore - we push audio data
+}
+
 nsresult
 MediaEngineWebRTCAudioSource::Snapshot(uint32_t aDuration, nsIDOMFile** aFile)
 {

File content/media/webrtc/MediaEngineWebRTCVideo.cpp

 MediaEngineWebRTCVideoSource::DeliverFrame(
    unsigned char* buffer, int size, uint32_t time_stamp, int64_t render_time)
 {
-  ReentrantMonitorAutoEnter enter(mMonitor);
-
   if (mInSnapshotMode) {
     // Set the condition variable to false and notify Snapshot().
     PR_Lock(mSnapshotLock);
 
   // Create a video frame and append it to the track.
   ImageFormat format = PLANAR_YCBCR;
+
   nsRefPtr<layers::Image> image = mImageContainer->CreateImage(&format, 1);
 
   layers::PlanarYCbCrImage* videoImage = static_cast<layers::PlanarYCbCrImage*>(image.get());
 
   videoImage->SetData(data);
 
+#ifdef LOG_ALL_FRAMES
+  static uint32_t frame_num = 0;
+  LOG(("frame %d; timestamp %u, render_time %lu", frame_num++, time_stamp, render_time));
+#endif
+
+  // we don't touch anything in 'this' until here (except for snapshot,
+  // which has it's own lock)
+  ReentrantMonitorAutoEnter enter(mMonitor);
+
+  // implicitly releases last image
+  mImage = image.forget();
+
+  return 0;
+}
+
+// Called if the graph thinks it's running out of buffered video; repeat
+// the last frame for whatever minimum period it think it needs.  Note that
+// this means that no *real* frame can be inserted during this period.
+void
+MediaEngineWebRTCVideoSource::NotifyPull(MediaStreamGraph* aGraph,
+                                         StreamTime aDesiredTime)
+{
   VideoSegment segment;
-  segment.AppendFrame(image.forget(), 1, gfxIntSize(mWidth, mHeight));
+
+  ReentrantMonitorAutoEnter enter(mMonitor);
+
+  if (mState != kStarted)
+    return;
+
+  // Note: we're not giving up mImage here
+  nsRefPtr<layers::Image> image = mImage;
+  TrackTicks target = TimeToTicksRoundUp(USECS_PER_S, aDesiredTime);
+  TrackTicks delta = target - mLastEndTime;
+#ifdef LOG_ALL_FRAMES
+  LOG(("NotifyPull, target = %lu, delta = %lu", (uint64_t) target, (uint64_t) delta));
+#endif
+  // NULL images are allowed
+  segment.AppendFrame(image ? image.forget() : nullptr, delta, gfxIntSize(mWidth, mHeight));
   mSource->AppendToTrack(mTrackID, &(segment));
-  return 0;
+  mLastEndTime = target;
 }
 
 void
   mTrackID = aID;
 
   mImageContainer = layers::LayerManager::CreateImageContainer();
-  mSource->AddTrack(aID, mFps, 0, new VideoSegment());
+
+  mSource->AddTrack(aID, USECS_PER_S, 0, new VideoSegment());
   mSource->AdvanceKnownTracksTime(STREAM_TIME_MAX);
+  mLastEndTime = 0;
 
   error = mViERender->AddRenderer(mCaptureIndex, webrtc::kVideoI420, (webrtc::ExternalRenderer*)this);
   if (error == -1) {

File dom/media/MediaManager.h

       nsresult rv;
 
       SourceMediaStream* stream = mStream->GetStream()->AsSourceStream();
+      stream->SetPullEnabled(true);
+
       if (mAudioSource) {
         rv = mAudioSource->Start(stream, kAudioTrack);
         if (NS_FAILED(rv)) {
           MM_LOG(("Starting video failed, rv=%d",rv));
         }
       }
+
       MM_LOG(("started all sources"));
       nsCOMPtr<GetUserMediaNotificationEvent> event =
         new GetUserMediaNotificationEvent(GetUserMediaNotificationEvent::STARTING);
     return;
   }
 
+  // Proxy NotifyPull() to sources
+  void
+  NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime)
+  {
+    // Currently audio sources ignore NotifyPull, but they could
+    // watch it especially for fake audio.
+    if (mAudioSource) {
+      mAudioSource->NotifyPull(aGraph, aDesiredTime);
+    }
+    if (mVideoSource) {
+      mVideoSource->NotifyPull(aGraph, aDesiredTime);
+    }
+  }
+
 private:
   nsRefPtr<MediaEngineSource> mAudioSource;
   nsRefPtr<MediaEngineSource> mVideoSource;