Commits

Steve Borho committed eb2b3e3

encoder: fix deadlocks/crashes when picture is a single CTU row

1. if only one row per frame, disable WPP it is pointless
2. if only one row per frame, frame parallelism is (mostly) impossible
3. clamp rateControlUpdateStats() to be within row count range

  • Participants
  • Parent commits b835636
  • Branches stable

Comments (0)

Files changed (2)

File source/encoder/encoder.cpp

 {
     this->m_param = p;
 
+    uint32_t maxLog2CUSize = g_log2Size[p->maxCUSize];
+    int rows = (p->sourceHeight + p->maxCUSize - 1) >> maxLog2CUSize;
+
+    // Do not allow WPP if only one row, it is pointless and unstable
+    if (rows == 1)
+        p->bEnableWavefront = 0;
+
     // Trim the thread pool if WPP is disabled
     if (!p->bEnableWavefront)
         p->poolNumThreads = 1;
 
     setThreadPool(ThreadPool::allocThreadPool(p->poolNumThreads));
     int poolThreadCount = ThreadPool::getThreadPool()->getThreadCount();
-    uint32_t maxLog2CUSize = g_log2Size[p->maxCUSize];
-    int rows = (p->sourceHeight + p->maxCUSize - 1) >> maxLog2CUSize;
 
-    if (p->frameNumThreads == 0)
+    if (!p->frameNumThreads)
     {
         // auto-detect frame threads
         int cpuCount = getCpuCount();
         if (poolThreadCount <= 1)
-            p->frameNumThreads = X265_MIN(cpuCount, rows / 2);
+            p->frameNumThreads = X265_MIN(cpuCount, (rows + 1) / 2);
         else if (cpuCount > 32)
             p->frameNumThreads = 6; // dual-socket 10-core IvyBridge or higher
         else if (cpuCount >= 16)

File source/encoder/frameencoder.cpp

     if (m_param->rc.rateControlMode == X265_RC_ABR)
     {
         if ((uint32_t)m_rce.encodeOrder <= 2 * (m_param->fpsNum / m_param->fpsDenom))
-            rowCount = m_numRows/2;
+            rowCount = X265_MIN((m_numRows + 1) / 2, m_numRows - 1);
         else
-            rowCount = m_refLagRows;
+            rowCount = X265_MIN(m_refLagRows, m_numRows - 1);
     }
     if (row == rowCount)
     {