banananetwork
/
roundcubemail
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improved rcube_charset::detect() with BOM checks from rcube_vcard::detect_encoding() - removed code duplication

Browse Source
pull/45/merge
Aleksander Machniak 12 years ago
parent 3c047d0e87
commit 0679b22150
2 changed files with 45 additions and 51 deletions
Show Stats Download Patch File Download Diff File

57
program/include/rcube_charset.php
Unescape Escape View File

@ -655,22 +655,49 @@ class rcube_charset
*/ */
public static function detect($string, $failover='') public static function detect($string, $failover='')
{ {
if (!function_exists('mb_detect_encoding')) { if (substr($string, 0, 4) == "\0\0\xFE\xFF") return 'UTF-32BE'; // Big Endian
return $failover; if (substr($string, 0, 4) == "\xFF\xFE\0\0") return 'UTF-32LE'; // Little Endian
} if (substr($string, 0, 2) == "\xFE\xFF") return 'UTF-16BE'; // Big Endian
if (substr($string, 0, 2) == "\xFF\xFE") return 'UTF-16LE'; // Little Endian
// FIXME: the order is important, because sometimes if (substr($string, 0, 3) == "\xEF\xBB\xBF") return 'UTF-8';
// iso string is detected as euc-jp and etc.
$enc = array( // heuristics
'UTF-8', 'SJIS', 'BIG5', 'GB2312', if ($string[0] == "\0" && $string[1] == "\0" && $string[2] == "\0" && $string[3] != "\0") return 'UTF-32BE';
'ISO-8859-1', 'ISO-8859-2', 'ISO-8859-3', 'ISO-8859-4', if ($string[0] != "\0" && $string[1] == "\0" && $string[2] == "\0" && $string[3] == "\0") return 'UTF-32LE';
'ISO-8859-5', 'ISO-8859-6', 'ISO-8859-7', 'ISO-8859-8', 'ISO-8859-9', if ($string[0] == "\0" && $string[1] != "\0" && $string[2] == "\0" && $string[3] != "\0") return 'UTF-16BE';
'ISO-8859-10', 'ISO-8859-13', 'ISO-8859-14', 'ISO-8859-15', 'ISO-8859-16', if ($string[0] != "\0" && $string[1] == "\0" && $string[2] != "\0" && $string[3] == "\0") return 'UTF-16LE';
'WINDOWS-1252', 'WINDOWS-1251', 'EUC-JP', 'EUC-TW', 'KOI8-R',
'ISO-2022-KR', 'ISO-2022-JP' if (function_exists('mb_detect_encoding')) {
); // FIXME: the order is important, because sometimes
// iso string is detected as euc-jp and etc.
$enc = array(
'UTF-8', 'SJIS', 'BIG5', 'GB2312',
'ISO-8859-1', 'ISO-8859-2', 'ISO-8859-3', 'ISO-8859-4',
'ISO-8859-5', 'ISO-8859-6', 'ISO-8859-7', 'ISO-8859-8', 'ISO-8859-9',
'ISO-8859-10', 'ISO-8859-13', 'ISO-8859-14', 'ISO-8859-15', 'ISO-8859-16',
'WINDOWS-1252', 'WINDOWS-1251', 'EUC-JP', 'EUC-TW', 'KOI8-R',
'ISO-2022-KR', 'ISO-2022-JP'
);
$result = mb_detect_encoding($string, join(',', $enc)); $result = mb_detect_encoding($string, join(',', $enc));
}
else {
// No match, check for UTF-8
// from http://w3.org/International/questions/qa-forms-utf-8.html
if (preg_match('/\A(
[\x09\x0A\x0D\x20-\x7E]
| [\xC2-\xDF][\x80-\xBF]
| \xE0[\xA0-\xBF][\x80-\xBF]
| [\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}
| \xED[\x80-\x9F][\x80-\xBF]
| \xF0[\x90-\xBF][\x80-\xBF]{2}
| [\xF1-\xF3][\x80-\xBF]{3}
| \xF4[\x80-\x8F][\x80-\xBF]{2}
)*\z/xs', substr($string, 0, 2048))
) {
return 'UTF-8';
}
}
return $result ? $result : $failover; return $result ? $result : $failover;
} }

39
program/include/rcube_vcard.php
Unescape Escape View File

@ -784,42 +784,9 @@ class rcube_vcard
*/ */
private static function detect_encoding($string) private static function detect_encoding($string)
{ {
if (substr($string, 0, 4) == "\0\0\xFE\xFF") return 'UTF-32BE'; // Big Endian $fallback = rcube::get_instance()->config->get('default_charset', 'ISO-8859-1'); // fallback to Latin-1
if (substr($string, 0, 4) == "\xFF\xFE\0\0") return 'UTF-32LE'; // Little Endian
if (substr($string, 0, 2) == "\xFE\xFF") return 'UTF-16BE'; // Big Endian return rcube_charset::detect($string, $fallback);
if (substr($string, 0, 2) == "\xFF\xFE") return 'UTF-16LE'; // Little Endian
if (substr($string, 0, 3) == "\xEF\xBB\xBF") return 'UTF-8';
// heuristics
if ($string[0] == "\0" && $string[1] == "\0" && $string[2] == "\0" && $string[3] != "\0") return 'UTF-32BE';
if ($string[0] != "\0" && $string[1] == "\0" && $string[2] == "\0" && $string[3] == "\0") return 'UTF-32LE';
if ($string[0] == "\0" && $string[1] != "\0" && $string[2] == "\0" && $string[3] != "\0") return 'UTF-16BE';
if ($string[0] != "\0" && $string[1] == "\0" && $string[2] != "\0" && $string[3] == "\0") return 'UTF-16LE';
// use mb_detect_encoding()
$encodings = array('UTF-8', 'ISO-8859-1', 'ISO-8859-2', 'ISO-8859-3',
'ISO-8859-4', 'ISO-8859-5', 'ISO-8859-6', 'ISO-8859-7', 'ISO-8859-8', 'ISO-8859-9',
'ISO-8859-10', 'ISO-8859-13', 'ISO-8859-14', 'ISO-8859-15', 'ISO-8859-16',
'WINDOWS-1252', 'WINDOWS-1251', 'BIG5', 'GB2312');
if (function_exists('mb_detect_encoding') && ($enc = mb_detect_encoding($string, $encodings)))
return $enc;
// No match, check for UTF-8
// from http://w3.org/International/questions/qa-forms-utf-8.html
if (preg_match('/\A(
[\x09\x0A\x0D\x20-\x7E]
| [\xC2-\xDF][\x80-\xBF]
| \xE0[\xA0-\xBF][\x80-\xBF]
| [\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}
| \xED[\x80-\x9F][\x80-\xBF]
| \xF0[\x90-\xBF][\x80-\xBF]{2}
| [\xF1-\xF3][\x80-\xBF]{3}
| \xF4[\x80-\x8F][\x80-\xBF]{2}
)*\z/xs', substr($string, 0, 2048)))
return 'UTF-8';
return rcube::get_instance()->config->get('default_charset', 'ISO-8859-1'); # fallback to Latin-1
} }
} }

Write Preview
Loading…
Cancel
Save