"Dither" option in the command line produces very heavy artifacts for 16bit input. Output unwatchable

[Former user]

While I was encoding 16bit Yuv 4:2:0 source material into 10bit hevc using x265 10bit build, I noticed that the "dither" option in the command line does not work for 16bit input. It produces very heavy artifacts in the 10bit encoded output.

While we are on this subject, I also want to ask a question that is related to how this bug should be handled. Should I convert 16bit input into 10bit (via third party tools) before I send it to x265? Or is it better to let x265 encode 10bit output video directly using the 16bit YUV input?

I did some visual comparisons but I could not decide which method is the best, quality wise. Please let me know what the best approach is (at least theoretically based on the way you designed x265).

If converting 16bit input to 10bit input is the best approach, I would appreciate if you can fix the "dither" option so that it can handle 16bit to 10bit conversion. If converting 16bit input to 10bit input is not necessary, please implement a sanity check into "dither" option so that it deactivates itself (with a warning message saying that dither is not necessary) when input bitrate is 16bits.

[Ma0]

You can try this patch:

diff -r c8ec86965e54 source/x265-extras.cpp
--- a/source/x265-extras.cpp    Fri Feb 19 14:36:52 2016 +0530
+++ b/source/x265-extras.cpp    Sun Mar 27 21:35:21 2016 +0200
@@ -290,15 +290,16 @@
     const int pixelMax = (1 << bitDepth) - 1;

     memset(errors, 0, (width + 1) * sizeof(int16_t));
-    int pitch = 1;
+    pixel pDst;
     for (int y = 0; y < height; y++, src += srcStride, dst += dstStride)
     {
         int16_t err = 0;
         for (int x = 0; x < width; x++)
         {
             err = err * 2 + errors[x] + errors[x + 1];
-            dst[x * pitch] = (pixel)x265_clip3(0, pixelMax, ((src[x * 1] << 2) + err + half) >> rShift);
-            errors[x] = err = src[x * pitch] - (dst[x * pitch] << lShift);
+            pDst = (pixel)x265_clip3(0, pixelMax, ((src[x] << 2) + err + half) >> rShift);
+            errors[x] = err = src[x] - (pDst << lShift);
+            dst[x] = pDst;
         }
     }
 } 

[Ma0]

The first patch is working only at default bit-depth. If you want to change bit-depth (in multilib build for example), you should use this patch:

diff -r c8ec86965e54 source/x265-extras.cpp
--- a/source/x265-extras.cpp    Fri Feb 19 14:36:52 2016 +0530
+++ b/source/x265-extras.cpp    Sun Mar 27 23:43:21 2016 +0200
@@ -281,8 +281,7 @@
 }

 /* The dithering algorithm is based on Sierra-2-4A error diffusion. */
-static void ditherPlane(pixel *dst, int dstStride, uint16_t *src, int srcStride,
-                        int width, int height, int16_t *errors, int bitDepth)
+static void ditherPlane(uint16_t *src, int srcStride, int width, int height, int16_t *errors, int bitDepth)
 {
     const int lShift = 16 - bitDepth;
     const int rShift = 16 - bitDepth + 2;
@@ -290,15 +289,36 @@
     const int pixelMax = (1 << bitDepth) - 1;

     memset(errors, 0, (width + 1) * sizeof(int16_t));
-    int pitch = 1;
-    for (int y = 0; y < height; y++, src += srcStride, dst += dstStride)
+
+    if (bitDepth == 8)
     {
-        int16_t err = 0;
-        for (int x = 0; x < width; x++)
+        for (int y = 0; y < height; y++, src += srcStride)
         {
-            err = err * 2 + errors[x] + errors[x + 1];
-            dst[x * pitch] = (pixel)x265_clip3(0, pixelMax, ((src[x * 1] << 2) + err + half) >> rShift);
-            errors[x] = err = src[x * pitch] - (dst[x * pitch] << lShift);
+            uint8_t *dst = (uint8_t*)src;
+            int16_t err = 0;
+            uint8_t pDst;
+            for (int x = 0; x < width; x++)
+            {
+                err = err * 2 + errors[x] + errors[x + 1];
+                pDst = (uint8_t)x265_clip3(0, pixelMax, ((src[x] << 2) + err + half) >> rShift);
+                errors[x] = err = src[x] - (pDst << lShift);
+                dst[x] = pDst;
+            }
+        }
+    }
+    else
+    {
+        for (int y = 0; y < height; y++, src += srcStride)
+        {
+            int16_t err = 0;
+            uint16_t pDst;
+            for (int x = 0; x < width; x++)
+            {
+                err = err * 2 + errors[x] + errors[x + 1];
+                pDst = (uint16_t)x265_clip3(0, pixelMax, ((src[x] << 2) + err + half) >> rShift);
+                errors[x] = err = src[x] - (pDst << lShift);
+                src[x] = pDst;
+            }
         }
     }
 }
@@ -339,7 +359,6 @@
         int height = (int)(picHeight >> x265_cli_csps[picIn.colorSpace].height[i]);
         int width = (int)(picWidth >> x265_cli_csps[picIn.colorSpace].width[i]);

-        ditherPlane(((pixel*)picIn.planes[i]), picIn.stride[i] / sizeof(pixel), ((uint16_t*)picIn.planes[i]),
-                    picIn.stride[i] / 2, width, height, errorBuf, bitDepth);
+        ditherPlane(((uint16_t*)picIn.planes[i]), picIn.stride[i] / 2, width, height, errorBuf, bitDepth);
     }
 } 

[Deepthi Nandakumar]

Fixed with 5b5374cae60f