citrus_viqr module wrongly stops to convert character if input ends with mnemonic chars.

[Takehiko NOZAKI]

try following:

$ echo "A." | iconv -f VIQR -t java
\u1ea0
$ echo -n "A." | iconv -f VIQR -t java
iconv: iconv(): Invalid argument

this is not only VIQR problem, UTF-5 encoding(not merged yet) have same problems.

[Takehiko NOZAKI]

suggested fix:

diff --git a/lib/libc/citrus/citrus_ctype_template.h b/lib/libc/citrus/citrus_ctype_template.h
index 9ab64c9..c5ecc73 100644
--- a/lib/libc/citrus/citrus_ctype_template.h
+++ b/lib/libc/citrus/citrus_ctype_template.h
@@ -131,6 +131,9 @@ static int _FUNCNAME(put_state_reset)(_ENCODING_INFO * __restrict,
                      char * __restrict, size_t,
                      _ENCODING_STATE * __restrict,
                      size_t * __restrict);
+static int _FUNCNAME(mbrtowc_state_reset)(_ENCODING_INFO * __restrict,
+    wchar_t * __restrict, _ENCODING_STATE * __restrict,
+    size_t * __restrict);
 #endif

 /*
diff --git a/lib/libc/citrus/citrus_namespace.h b/lib/libc/citrus/citrus_namespace.h
index b8cac9e..39e3f83 100644
--- a/lib/libc/citrus/citrus_namespace.h
+++ b/lib/libc/citrus/citrus_namespace.h
@@ -204,6 +204,7 @@
 #define _stdenc_cstomb     _citrus_stdenc_cstomb
 #define _stdenc_mbtowc     _citrus_stdenc_mbtowc
 #define _stdenc_wctomb     _citrus_stdenc_wctomb
+#define _stdenc_mbtocs_state_reset _citrus_stdenc_mbtocs_state_reset
 #define _stdenc_put_state_reset    _citrus_stdenc_put_state_reset
 #define _stdenc_get_state_size _citrus_stdenc_get_state_size
 #define _stdenc_get_mb_cur_max _citrus_stdenc_get_mb_cur_max
diff --git a/lib/libc/citrus/citrus_none.c b/lib/libc/citrus/citrus_none.c
index d6cf8e2..9c0c7fe 100644
--- a/lib/libc/citrus/citrus_none.c
+++ b/lib/libc/citrus/citrus_none.c
@@ -587,3 +587,13 @@ _citrus_NONE_stdenc_get_state_desc(struct _stdenc * __restrict ce,

    return ret;
 }
+
+static int
+/*ARGSUSED*/
+_citrus_NONE_stdenc_mbtocs_state_reset(struct _citrus_stdenc * __restrict ce,
+    _csid_t *csid, _index_t *idx, void * __restrict pspriv,
+    size_t * __restrict nresult)
+{
+   *nresult = 0;
+   return 0;
+}
diff --git a/lib/libc/citrus/citrus_stdenc.h b/lib/libc/citrus/citrus_stdenc.h
index ce198db..744830e 100644
--- a/lib/libc/citrus/citrus_stdenc.h
+++ b/lib/libc/citrus/citrus_stdenc.h
@@ -142,4 +142,13 @@ _citrus_stdenc_get_state_desc(struct _citrus_stdenc * __restrict ce,
    return (*ce->ce_ops->eo_get_state_desc)(ce, ps, id, d);
 }

+static __inline int
+_citrus_stdenc_mbtocs_state_reset(struct _citrus_stdenc * __restrict ce,
+    _citrus_csid_t * __restrict csid, _citrus_index_t * __restrict idx,
+    void * __restrict ps, size_t * __restrict nresult)
+{
+   _DIAGASSERT(ce && ce->ce_ops && ce->ce_ops->eo_mbtocs_state_reset);
+   return (*ce->ce_ops->eo_mbtocs_state_reset)(ce, csid, idx, ps, nresult);
+}
+
 #endif
diff --git a/lib/libc/citrus/citrus_stdenc_local.h b/lib/libc/citrus/citrus_stdenc_local.h
index bac89c6..67ae606 100644
--- a/lib/libc/citrus/citrus_stdenc_local.h
+++ b/lib/libc/citrus/citrus_stdenc_local.h
@@ -66,7 +66,11 @@ static int   _citrus_##_e_##_stdenc_put_state_reset          \
     void * __restrict, size_t * __restrict);           \
 static int _citrus_##_e_##_stdenc_get_state_desc           \
    (struct _citrus_stdenc * __restrict, void * __restrict, int,    \
-    struct _citrus_stdenc_state_desc * __restrict)
+    struct _citrus_stdenc_state_desc * __restrict);        \
+static int _citrus_##_e_##_stdenc_mbtocs_state_reset       \
+   (struct _citrus_stdenc * __restrict,                \
+    _citrus_csid_t * __restrict, _citrus_index_t * __restrict, \
+    void * __restrict, size_t * __restrict)

 #define _CITRUS_STDENC_DEF_OPS(_e_)                    \
 struct _citrus_stdenc_ops _citrus_##_e_##_stdenc_ops = {       \
@@ -79,7 +83,9 @@ struct _citrus_stdenc_ops _citrus_##_e_##_stdenc_ops = {      \
    /* eo_mbtowc */     &_citrus_##_e_##_stdenc_mbtowc,     \
    /* eo_wctomb */     &_citrus_##_e_##_stdenc_wctomb,     \
    /* eo_put_state_reset */&_citrus_##_e_##_stdenc_put_state_reset,\
-   /* eo_get_state_desc */ &_citrus_##_e_##_stdenc_get_state_desc  \
+   /* eo_get_state_desc */ &_citrus_##_e_##_stdenc_get_state_desc, \
+   /* eo_mbtocs_state_reset */                 \
+       &_citrus_##_e_##_stdenc_mbtocs_state_reset          \
 }

 typedef int    (*_citrus_stdenc_init_t)
@@ -111,12 +117,16 @@ typedef int   (*_citrus_stdenc_put_state_reset_t)
 typedef int    (*_citrus_stdenc_get_state_desc_t)
    (struct _citrus_stdenc * __restrict, void * __restrict, int,
     struct _citrus_stdenc_state_desc * __restrict);
+typedef int    (*_citrus_stdenc_mbtocs_state_reset_t)
+   (struct _citrus_stdenc *__restrict,
+    _citrus_csid_t * __restrict, _citrus_index_t * __restrict,
+    void * __restrict, size_t * __restrict);
 /*
  * ABI version change log
  *   0x00000001
  *     initial version
  */
-#define _CITRUS_STDENC_ABI_VERSION 0x00000002
+#define _CITRUS_STDENC_ABI_VERSION 0x00000003
 struct _citrus_stdenc_ops {
    uint32_t            eo_abi_version;
    /* version 0x00000001 */
@@ -130,6 +140,8 @@ struct _citrus_stdenc_ops {
    _citrus_stdenc_put_state_reset_t eo_put_state_reset;
    /* version 0x00000002 */
    _citrus_stdenc_get_state_desc_t eo_get_state_desc;
+   /* version 0x00000003 */
+   _citrus_stdenc_mbtocs_state_reset_t eo_mbtocs_state_reset;
 };

 struct _citrus_stdenc_traits {
diff --git a/lib/libc/citrus/citrus_stdenc_template.h b/lib/libc/citrus/citrus_stdenc_template.h
index 54088cd..39fa379 100644
--- a/lib/libc/citrus/citrus_stdenc_template.h
+++ b/lib/libc/citrus/citrus_stdenc_template.h
@@ -204,3 +204,22 @@ _FUNCNAME(stdenc_get_state_desc)(struct _citrus_stdenc * __restrict ce,

    return ret;
 }
+
+static int
+_FUNCNAME(stdenc_mbtocs_state_reset)(struct _citrus_stdenc * __restrict ce,
+    _citrus_csid_t * __restrict csid, _citrus_index_t * __restrict idx,
+    void * __restrict ps, size_t * __restrict nresult)
+{
+#if _ENCODING_IS_STATE_DEPENDENT
+   int ret;
+   wchar_t wc;
+   ret = _FUNCNAME(mbrtowc_state_reset)(_CE_TO_EI(ce),
+       &wc, _TO_STATE(ps), nresult);
+   if (!ret && *nresult > 0)
+       _FUNCNAME(stdenc_wctocs)(_CE_TO_EI(ce), csid, idx, wc);
+   return ret;
+#else
+   *nresult = 0;
+   return 0;
+#endif
+}
diff --git a/lib/libc/citrus/modules/citrus_hz.c b/lib/libc/citrus/modules/citrus_hz.c
index 6b9a73b..bdb6a0f 100644
--- a/lib/libc/citrus/modules/citrus_hz.c
+++ b/lib/libc/citrus/modules/citrus_hz.c
@@ -439,6 +439,15 @@ _citrus_HZ_put_state_reset(_HZEncodingInfo * __restrict ei,
 }

 static __inline int
+_citrus_HZ_mbrtowc_state_reset(_HZEncodingInfo * __restrict ei,
+    wchar_t * __restrict pwc, _HZState * __restrict psenc,
+    size_t * __restrict nresult)
+{
+   *nresult = (size_t)0;
+   return 0;
+}
+
+static __inline int
 _citrus_HZ_stdenc_get_state_desc_generic(_HZEncodingInfo * __restrict ei,
    _HZState * __restrict psenc, int * __restrict rstate)
 {
diff --git a/lib/libc/citrus/modules/citrus_iconv_std.c b/lib/libc/citrus/modules/citrus_iconv_std.c
index 72932b3..6dc19d3 100644
--- a/lib/libc/citrus/modules/citrus_iconv_std.c
+++ b/lib/libc/citrus/modules/citrus_iconv_std.c
@@ -128,6 +128,13 @@ wctombx(struct _citrus_iconv_std_encoding *se,
 }

 static __inline int
+mbtocs_state_resetx(struct _citrus_iconv_std_encoding *se,
+   _csid_t *csid, _index_t *idx, size_t *nresult)
+{
+   return _stdenc_mbtocs_state_reset(se->se_handle, csid, idx, se->se_ps, nresult);
+}
+
+static __inline int
 put_state_resetx(struct _citrus_iconv_std_encoding *se,
         char *s, size_t n, size_t *nresult)
 {
@@ -536,8 +543,13 @@ _citrus_iconv_std_iconv_convert(struct _citrus_iconv * __restrict cv,
                /* fetch shift sequences only. */
                goto next;
            }
-           ret = EINVAL;
-           goto err;
+           ret = mbtocs_state_resetx(&sc->sc_src_encoding, &csid, &idx, &szrin);
+           if (ret)
+               goto err;
+           if (szrin == 0) {
+               ret = EINVAL;
+               goto err;
+           }
        }
        /* convert the character */
        ret = do_conv(is, sc, &csid, &idx);
diff --git a/lib/libc/citrus/modules/citrus_iso2022.c b/lib/libc/citrus/modules/citrus_iso2022.c
index 3c375e2..c89cfe3 100644
--- a/lib/libc/citrus/modules/citrus_iso2022.c
+++ b/lib/libc/citrus/modules/citrus_iso2022.c
@@ -1210,6 +1210,15 @@ _citrus_ISO2022_put_state_reset(_ISO2022EncodingInfo * __restrict ei,
    return (0);
 }

+static __inline int
+_citrus_ISO2022_mbrtowc_state_reset(_ISO2022EncodingInfo * __restrict ei,
+    wchar_t * __restrict pwc, _ISO2022State * __restrict psenc,
+    size_t * __restrict nresult)
+{
+   *nresult = (size_t)0;
+   return 0;
+}
+
 static int
 _citrus_ISO2022_wcrtomb_priv(_ISO2022EncodingInfo * __restrict ei,
                 char * __restrict s, size_t n, wchar_t wc,
diff --git a/lib/libc/citrus/modules/citrus_softbank.c b/lib/libc/citrus/modules/citrus_softbank.c
index 293d8ee..bf9d502 100644
--- a/lib/libc/citrus/modules/citrus_softbank.c
+++ b/lib/libc/citrus/modules/citrus_softbank.c
@@ -304,6 +304,15 @@ _citrus_SoftBank_put_state_reset(_SoftBankEncodingInfo * __restrict ei,
 }

 static __inline int
+_citrus_SoftBank_mbrtowc_state_reset(_SoftBankEncodingInfo * __restrict ei,
+    wchar_t * __restrict pwc, _SoftBankState * __restrict psenc,
+    size_t * __restrict nresult)
+{
+   *nresult = (size_t)0;
+   return 0;
+}
+
+static __inline int
 /*ARGSUSED*/
 _citrus_SoftBank_stdenc_wctocs(_SoftBankEncodingInfo * __restrict ei,
     _csid_t * __restrict csid, _index_t * __restrict idx, wchar_t wc)
diff --git a/lib/libc/citrus/modules/citrus_utf7.c b/lib/libc/citrus/modules/citrus_utf7.c
index 0205a62..8673917 100644
--- a/lib/libc/citrus/modules/citrus_utf7.c
+++ b/lib/libc/citrus/modules/citrus_utf7.c
@@ -450,6 +450,15 @@ _citrus_UTF7_put_state_reset(_UTF7EncodingInfo * __restrict ei,
 }

 static __inline int
+_citrus_UTF7_mbrtowc_state_reset(_UTF7EncodingInfo * __restrict ei,
+    wchar_t * __restrict pwc, _UTF7State * __restrict psenc,
+    size_t * __restrict nresult)
+{
+   *nresult = (size_t)0;
+   return 0;
+}
+
+static __inline int
 /*ARGSUSED*/
 _citrus_UTF7_stdenc_wctocs(_UTF7EncodingInfo * __restrict ei,
               _csid_t * __restrict csid,
diff --git a/lib/libc/citrus/modules/citrus_viqr.c b/lib/libc/citrus/modules/citrus_viqr.c
index 5418e48..54a04bc 100644
--- a/lib/libc/citrus/modules/citrus_viqr.c
+++ b/lib/libc/citrus/modules/citrus_viqr.c
@@ -426,6 +426,17 @@ _citrus_VIQR_put_state_reset(_VIQREncodingInfo * __restrict ei,
 }

 static __inline int
+/* ARGSUSED */
+_citrus_VIQR_mbrtowc_state_reset(_VIQREncodingInfo * __restrict ei,
+    wchar_t * __restrict pwc, _VIQRState * __restrict psenc,
+    size_t * __restrict nresult)
+{
+   /* XXX: FIXME */
+   const char *s = "";
+   return _citrus_VIQR_mbrtowc_priv(ei, pwc, &s, 1, psenc, nresult);
+}
+
+static __inline int
 /*ARGSUSED*/
 _citrus_VIQR_stdenc_wctocs(_VIQREncodingInfo * __restrict ei,
    _csid_t * __restrict csid, _index_t * __restrict idx, wchar_t wc)
diff --git a/lib/libc/citrus/modules/citrus_zw.c b/lib/libc/citrus/modules/citrus_zw.c
index 7e9a2cf..8662ca2 100644
--- a/lib/libc/citrus/modules/citrus_zw.c
+++ b/lib/libc/citrus/modules/citrus_zw.c
@@ -382,6 +382,15 @@ _citrus_ZW_put_state_reset(_ZWEncodingInfo * __restrict ei,
 }

 static __inline int
+_citrus_ZW_mbrtowc_state_reset(_ZWEncodingInfo * __restrict ei,
+    wchar_t * __restrict pwc, _ZWState * __restrict psenc,
+    size_t * __restrict nresult)
+{
+   *nresult = (size_t)0;
+   return 0;
+}
+
+static __inline int
 /*ARGSUSED*/
 _citrus_ZW_stdenc_get_state_desc_generic(_ZWEncodingInfo * __restrict ei,
    _ZWState * __restrict psenc, int * __restrict rstate)

[Takehiko NOZAKI]

oops, UTF-5 is stateless encoding, so mbrtowc_state_reset should not use _ENCODING_IS_STATE_DEPENDENT.

[Takehiko NOZAKI]

BUGFIX: Issue #63 -- citrus_viqr module wrongly stops to convert character if input ends with mnemonic chars.

→ <<cset 264f89b87b20>>

[Takehiko NOZAKI]

i dicided that there is no merit to change libencoding's ABI. so i don't change stdenc, add kludge to iconv_std only.

[Takehiko NOZAKI]

Revert "BUGFIX: Issue #63 -- citrus_viqr module wrongly stops to convert character if input ends with mnemonic chars."

This reverts commit 264f89b87b20ff699b3d031eb13000a45469788f.

→ <<cset 28c68479b7fc>>

[Takehiko NOZAKI]

previous fix is break other encoding conversion, so i revert and start to rework.

[Takehiko NOZAKI]

Issue #69 was marked as a duplicate of this issue.

[Takehiko NOZAKI]

BUGFIX: Issue #63 -- citrus_viqr module wrongly stops to convert character if input ends with mnemonic chars. (DUPLICATE: Issue #69 -- citrus_utf5 module UTF-5 -> other encoding may causes EILSEQ.)

→ <<cset bf4720bdbc5e>>

[Takehiko NOZAKI]

BUGFIX: Issue #72 -- after fix Issue #63, iconv(3) can't detect incomplete input or shift state.

→ <<cset acbef0854e4c>>