Wrong behavior for '\xff\n'.decode('gb2312', 'ignore')

Modules/cjkcodecs/_codecs_jp.c

 /*
  * _codecs_jp.c: Codecs collection for Japanese encodings
  *
  * Written by Hye-Shik Chang <perky@FreeBSD.org>
  */
 
 #define USING_BINARY_PAIR_SEARCH
 #define EMPBASE 0x20000
 
 #include "cjkcodecs.h"
@@ skipping 94 matching lines @@
             NEXT(1, 1)
             continue;
         }
 
         REQUIRE_INBUF(2)
         c2 = IN2;
 
         TRYMAP_DEC(cp932ext, **outbuf, c, c2);
         else if ((c >= 0x81 && c <= 0x9f) || (c >= 0xe0 && c <= 0xea)){
             if (c2 < 0x40 || (c2 > 0x7e && c2 < 0x80) || c2 > 0xfc)
-                return 2;
+                return 1;
 
             c = (c < 0xe0 ? c - 0x81 : c - 0xc1);
             c2 = (c2 < 0x80 ? c2 - 0x40 : c2 - 0x41);
             c = (2 * c + (c2 < 0x5e ? 0 : 1) + 0x21);
             c2 = (c2 < 0x5e ? c2 : c2 - 0x5e) + 0x21;
 
             TRYMAP_DEC(jisx0208, **outbuf, c, c2);
-            else return 2;
+            else return 1;
         }
         else if (c >= 0xf0 && c <= 0xf9) {
             if ((c2 >= 0x40 && c2 <= 0x7e) ||
                 (c2 >= 0x80 && c2 <= 0xfc))
                 OUT1(0xe000 + 188 * (c - 0xf0) +
                      (c2 < 0x80 ? c2 - 0x40 : c2 - 0x41))
             else
-                return 2;
+                return 1;
         }
         else
-            return 2;
+            return 1;
 
         NEXT(2, 1)
     }
 
     return 0;
 }
 
 
 /*
  * EUC-JIS-2004 codec
@@ skipping 104 matching lines @@
             /* JIS X 0201 half-width katakana */
             unsigned char c2;
 
             REQUIRE_INBUF(2)
             c2 = IN2;
             if (c2 >= 0xa1 && c2 <= 0xdf) {
                 OUT1(0xfec0 + c2)
                 NEXT(2, 1)
             }
             else
-                return 2;
+                return 1;
         }
         else if (c == 0x8f) {
             unsigned char c2, c3;
 
             REQUIRE_INBUF(3)
             c2 = IN2 ^ 0x80;
             c3 = IN3 ^ 0x80;
 
             /* JIS X 0213 Plane 2 or JIS X 0212 (see NOTES) */
             EMULATE_JISX0213_2000_DECODE_PLANE2(**outbuf, c2, c3)
             else TRYMAP_DEC(jisx0213_2_bmp, **outbuf, c2, c3) ;
             else TRYMAP_DEC(jisx0213_2_emp, code, c2, c3) {
                 WRITEUCS4(EMPBASE | code)
                 NEXT_IN(3)
                 continue;
             }
             else TRYMAP_DEC(jisx0212, **outbuf, c2, c3) ;
-            else return 3;
+            else return 1;
             NEXT(3, 1)
         }
         else {
             unsigned char c2;
 
             REQUIRE_INBUF(2)
             c ^= 0x80;
             c2 = IN2 ^ 0x80;
 
             /* JIS X 0213 Plane 1 */
             EMULATE_JISX0213_2000_DECODE_PLANE1(**outbuf, c, c2)
             else if (c == 0x21 && c2 == 0x40) **outbuf = 0xff3c;
             else if (c == 0x22 && c2 == 0x32) **outbuf = 0xff5e;
             else TRYMAP_DEC(jisx0208, **outbuf, c, c2);
             else TRYMAP_DEC(jisx0213_1_bmp, **outbuf, c, c2);
             else TRYMAP_DEC(jisx0213_1_emp, code, c, c2) {
                 WRITEUCS4(EMPBASE | code)
                 NEXT_IN(2)
                 continue;
             }
             else TRYMAP_DEC(jisx0213_pair, code, c, c2) {
                 WRITE2(code >> 16, code & 0xffff)
                 NEXT(2, 2)
                 continue;
             }
-            else return 2;
+            else return 1;
             NEXT(2, 1)
         }
     }
 
     return 0;
 }
 
 
 /*
  * EUC-JP codec
@@ skipping 67 matching lines @@
             /* JIS X 0201 half-width katakana */
             unsigned char c2;
 
             REQUIRE_INBUF(2)
             c2 = IN2;
             if (c2 >= 0xa1 && c2 <= 0xdf) {
                 OUT1(0xfec0 + c2)
                 NEXT(2, 1)
             }
             else
-                return 2;
+                return 1;
         }
         else if (c == 0x8f) {
             unsigned char c2, c3;
 
             REQUIRE_INBUF(3)
             c2 = IN2;
             c3 = IN3;
             /* JIS X 0212 */
             TRYMAP_DEC(jisx0212, **outbuf, c2 ^ 0x80, c3 ^ 0x80) {
                 NEXT(3, 1)
             }
             else
-                return 3;
+                return 1;
         }
         else {
             unsigned char c2;
 
             REQUIRE_INBUF(2)
             c2 = IN2;
             /* JIS X 0208 */
 #ifndef STRICT_BUILD
             if (c == 0xa1 && c2 == 0xc0)
                 /* FULL-WIDTH REVERSE SOLIDUS */
                 **outbuf = 0xff3c;
             else
 #endif
                 TRYMAP_DEC(jisx0208, **outbuf,
                            c ^ 0x80, c2 ^ 0x80) ;
-            else return 2;
+            else return 1;
             NEXT(2, 1)
         }
     }
 
     return 0;
 }
 
 
 /*
  * SHIFT_JIS codec
@@ skipping 64 matching lines @@
 #else
         if (c < 0x80) **outbuf = c;
 #endif
         else JISX0201_K_DECODE(c, **outbuf)
         else if ((c >= 0x81 && c <= 0x9f) || (c >= 0xe0 && c <= 0xea)){
             unsigned char c1, c2;
 
             REQUIRE_INBUF(2)
             c2 = IN2;
             if (c2 < 0x40 || (c2 > 0x7e && c2 < 0x80) || c2 > 0xfc)
-                return 2;
+                return 1;
 
             c1 = (c < 0xe0 ? c - 0x81 : c - 0xc1);
             c2 = (c2 < 0x80 ? c2 - 0x40 : c2 - 0x41);
             c1 = (2 * c1 + (c2 < 0x5e ? 0 : 1) + 0x21);
             c2 = (c2 < 0x5e ? c2 : c2 - 0x5e) + 0x21;
 
 #ifndef STRICT_BUILD
             if (c1 == 0x21 && c2 == 0x40) {
                 /* FULL-WIDTH REVERSE SOLIDUS */
                 OUT1(0xff3c)
                 NEXT(2, 1)
                 continue;
             }
 #endif
             TRYMAP_DEC(jisx0208, **outbuf, c1, c2) {
                 NEXT(2, 1)
                 continue;
             }
             else
-                return 2;
+                return 1;
         }
         else
-            return 2;
+            return 1;
 
         NEXT(1, 1) /* JIS X 0201 */
     }
 
     return 0;
 }
 
 
 /*
  * SHIFT_JIS-2004 codec
@@ skipping 99 matching lines @@
 
         REQUIRE_OUTBUF(1)
         JISX0201_DECODE(c, **outbuf)
         else if ((c >= 0x81 && c <= 0x9f) || (c >= 0xe0 && c <= 0xfc)){
             unsigned char c1, c2;
             ucs4_t code;
 
             REQUIRE_INBUF(2)
             c2 = IN2;
             if (c2 < 0x40 || (c2 > 0x7e && c2 < 0x80) || c2 > 0xfc)
-                return 2;
+                return 1;
 
             c1 = (c < 0xe0 ? c - 0x81 : c - 0xc1);
             c2 = (c2 < 0x80 ? c2 - 0x40 : c2 - 0x41);
             c1 = (2 * c1 + (c2 < 0x5e ? 0 : 1));
             c2 = (c2 < 0x5e ? c2 : c2 - 0x5e) + 0x21;
 
             if (c1 < 0x5e) { /* Plane 1 */
                 c1 += 0x21;
                 EMULATE_JISX0213_2000_DECODE_PLANE1(**outbuf,
                                 c1, c2)
                 else TRYMAP_DEC(jisx0208, **outbuf, c1, c2) {
                     NEXT_OUT(1)
                 }
                 else TRYMAP_DEC(jisx0213_1_bmp, **outbuf,
                                 c1, c2) {
                     NEXT_OUT(1)
                 }
                 else TRYMAP_DEC(jisx0213_1_emp, code, c1, c2) {
                     WRITEUCS4(EMPBASE | code)
                 }
                 else TRYMAP_DEC(jisx0213_pair, code, c1, c2) {
                     WRITE2(code >> 16, code & 0xffff)
                     NEXT_OUT(2)
                 }
                 else
-                    return 2;
+                    return 1;
                 NEXT_IN(2)
             }
             else { /* Plane 2 */
                 if (c1 >= 0x67) c1 += 0x07;
                 else if (c1 >= 0x63 || c1 == 0x5f) c1 -= 0x37;
                 else c1 -= 0x3d;
 
                 EMULATE_JISX0213_2000_DECODE_PLANE2(**outbuf,
                                 c1, c2)
                 else TRYMAP_DEC(jisx0213_2_bmp, **outbuf,
                                 c1, c2) ;
                 else TRYMAP_DEC(jisx0213_2_emp, code, c1, c2) {
                     WRITEUCS4(EMPBASE | code)
                     NEXT_IN(2)
                     continue;
                 }
                 else
-                    return 2;
+                    return 1;
                 NEXT(2, 1)
             }
             continue;
         }
         else
-            return 2;
+            return 1;
 
         NEXT(1, 1) /* JIS X 0201 */
     }
 
     return 0;
 }
 
 
 BEGIN_MAPPINGS_LIST
   MAPPING_DECONLY(jisx0208)
@@ skipping 13 matching lines @@
   CODEC_STATELESS(shift_jis)
   CODEC_STATELESS(cp932)
   CODEC_STATELESS(euc_jp)
   CODEC_STATELESS(shift_jis_2004)
   CODEC_STATELESS(euc_jis_2004)
   { "euc_jisx0213", (void *)2000, NULL, _STATELESS_METHODS(euc_jis_2004) },
   { "shift_jisx0213", (void *)2000, NULL, _STATELESS_METHODS(shift_jis_2004) },
 END_CODECS_LIST
 
 I_AM_A_MODULE_FOR(jp)