"""Character-encoding detection and the catalogue of supported text encodings."""
from dataclasses import dataclass
from typing import List
import chardet
_DETECTOR_BUFFER_SIZE = 64
[docs]
def detect_text_encoding(file) -> str:
"""Detect the character encoding of a binary file-like object.
Reads up to ``_DETECTOR_BUFFER_SIZE`` lines using ``chardet``'s universal
detector and stops early when confidence is high enough.
Args:
file: A binary file-like object to inspect.
Returns:
str: The detected encoding name (e.g. ``"utf-8"``).
"""
detector = chardet.universaldetector.UniversalDetector()
for line in file:
detector.feed(line)
if detector.done:
break
detector.close()
return detector.result["encoding"]
[docs]
@dataclass
class Encoding:
"""Descriptor for a single character encoding."""
name: str
aliases: List[str]
languages: List[str]
[docs]
class SupportedEncodings:
"""Catalogue of character encodings offered as user-selectable form choices."""
[docs]
@classmethod
def choices(cls):
"""Return Django-style ``(value, label)`` tuples for all supported encodings.
Returns:
list[tuple[str, str]]: Pairs of ``(encoding_name, human_readable_label)``.
"""
return [
(
encoding.name,
"%s (%s) for %s"
% (
encoding.name,
", ".join(encoding.aliases),
", ".join(encoding.languages),
),
)
for encoding in cls._encodings
]
_encodings: List[Encoding] = [
Encoding(
name="ascii",
aliases=["646", "us-ascii"],
languages=["English"],
),
Encoding(
name="big5",
aliases=["big5-tw", "csbig5"],
languages=["Traditional Chinese"],
),
Encoding(
name="big5hkscs",
aliases=["big5-hkscs", "hkscs"],
languages=["Traditional Chinese"],
),
Encoding(
name="cp037",
aliases=["IBM037", "IBM039"],
languages=["English"],
),
Encoding(
name="cp273",
aliases=["273", "IBM273", "csIBM273"],
languages=["German"],
),
Encoding(
name="cp424",
aliases=["EBCDIC-CP-HE", "IBM424"],
languages=["Hebrew"],
),
Encoding(
name="cp437",
aliases=["437", "IBM437"],
languages=["English"],
),
Encoding(
name="cp500",
aliases=["EBCDIC-CP-BE", "EBCDIC-CP-CH", "IBM500"],
languages=["Western Europe"],
),
Encoding(
name="cp720",
aliases=[""],
languages=["Arabic"],
),
Encoding(
name="cp737",
aliases=[""],
languages=["Greek"],
),
Encoding(
name="cp775",
aliases=["IBM775"],
languages=["Baltic languages"],
),
Encoding(
name="cp850",
aliases=["850", "IBM850"],
languages=["Western Europe"],
),
Encoding(
name="cp852",
aliases=["852", "IBM852"],
languages=["Central and Eastern Europe"],
),
Encoding(
name="cp855",
aliases=["855", "IBM855"],
languages=["Bulgarian", "Byelorussian", "Macedonian", "Russian", "Serbian"],
),
Encoding(
name="cp856",
aliases=[""],
languages=["Hebrew"],
),
Encoding(
name="cp857",
aliases=["857", "IBM857"],
languages=["Turkish"],
),
Encoding(
name="cp858",
aliases=["858", "IBM858"],
languages=["Western Europe"],
),
Encoding(
name="cp860",
aliases=["860", "IBM860"],
languages=["Portuguese"],
),
Encoding(
name="cp861",
aliases=["861", "CP-IS", "IBM861"],
languages=["Icelandic"],
),
Encoding(
name="cp862",
aliases=["862", "IBM862"],
languages=["Hebrew"],
),
Encoding(
name="cp863",
aliases=["863", "IBM863"],
languages=["Canadian"],
),
Encoding(
name="cp864",
aliases=["IBM864"],
languages=["Arabic"],
),
Encoding(
name="cp865",
aliases=["865", "IBM865"],
languages=["Danish", "Norwegian"],
),
Encoding(
name="cp866",
aliases=["866", "IBM866"],
languages=["Russian"],
),
Encoding(
name="cp869",
aliases=["869", "CP-GR", "IBM869"],
languages=["Greek"],
),
Encoding(
name="cp874",
aliases=[""],
languages=["Thai"],
),
Encoding(
name="cp875",
aliases=[""],
languages=["Greek"],
),
Encoding(
name="cp932",
aliases=["932", "ms932", "mskanji", "ms-kanji"],
languages=["Japanese"],
),
Encoding(
name="cp949",
aliases=["949", "ms949", "uhc"],
languages=["Korean"],
),
Encoding(
name="cp950",
aliases=["950", "ms950"],
languages=["Traditional Chinese"],
),
Encoding(
name="cp1006",
aliases=[""],
languages=["Urdu"],
),
Encoding(
name="cp1026",
aliases=["ibm1026"],
languages=["Turkish"],
),
Encoding(
name="cp1125",
aliases=["1125", "ibm1125", "cp866u", "ruscii"],
languages=["Ukrainian"],
),
Encoding(
name="cp1140",
aliases=["ibm1140"],
languages=["Western Europe"],
),
Encoding(
name="cp1250",
aliases=["windows-1250"],
languages=["Central and Eastern Europe"],
),
Encoding(
name="cp1251",
aliases=["windows-1251"],
languages=["Bulgarian", "Byelorussian", "Macedonian", "Russian", "Serbian"],
),
Encoding(
name="cp1252",
aliases=["windows-1252"],
languages=["Western Europe"],
),
Encoding(
name="cp1253",
aliases=["windows-1253"],
languages=["Greek"],
),
Encoding(
name="cp1254",
aliases=["windows-1254"],
languages=["Turkish"],
),
Encoding(
name="cp1255",
aliases=["windows-1255"],
languages=["Hebrew"],
),
Encoding(
name="cp1256",
aliases=["windows-1256"],
languages=["Arabic"],
),
Encoding(
name="cp1257",
aliases=["windows-1257"],
languages=["Baltic languages"],
),
Encoding(
name="cp1258",
aliases=["windows-1258"],
languages=["Vietnamese"],
),
Encoding(
name="euc_jp",
aliases=["eucjp", "ujis", "u-jis"],
languages=["Japanese"],
),
Encoding(
name="euc_jis_2004",
aliases=["jisx0213", "eucjis2004"],
languages=["Japanese"],
),
Encoding(
name="euc_jisx0213",
aliases=["eucjisx0213"],
languages=["Japanese"],
),
Encoding(
name="euc_kr",
aliases=[
"euckr",
"korean",
"ksc5601",
"ks_c-5601",
"ks_c-5601-1987",
"ksx1001",
"ks_x-1001",
],
languages=["Korean"],
),
Encoding(
name="gb2312",
aliases=[
"chinese",
"csiso58gb231280",
"euc-cn",
"euccn",
"eucgb2312-cn",
"gb2312-1980",
"gb2312-80",
"iso-ir-58",
],
languages=["Simplified Chinese"],
),
Encoding(
name="gbk",
aliases=["936", "cp936", "ms936"],
languages=["Unified Chinese"],
),
Encoding(
name="gb18030",
aliases=["gb18030-2000"],
languages=["Unified Chinese"],
),
Encoding(
name="hz",
aliases=["hzgb", "hz-gb", "hz-gb-2312"],
languages=["Simplified Chinese"],
),
Encoding(
name="iso2022_jp",
aliases=["csiso2022jp", "iso2022jp", "iso-2022-jp"],
languages=["Japanese"],
),
Encoding(
name="iso2022_jp_1",
aliases=["iso2022jp-1", "iso-2022-jp-1"],
languages=["Japanese"],
),
Encoding(
name="iso2022_jp_2",
aliases=["iso2022jp-2", "iso-2022-jp-2"],
languages=[
"Japanese",
"Korean",
"Simplified Chinese",
"Western Europe",
"Greek",
],
),
Encoding(
name="iso2022_jp_2004",
aliases=["iso2022jp-2004", "iso-2022-jp-2004"],
languages=["Japanese"],
),
Encoding(
name="iso2022_jp_3",
aliases=["iso2022jp-3", "iso-2022-jp-3"],
languages=["Japanese"],
),
Encoding(
name="iso2022_jp_ext",
aliases=["iso2022jp-ext", "iso-2022-jp-ext"],
languages=["Japanese"],
),
Encoding(
name="iso2022_kr",
aliases=["csiso2022kr", "iso2022kr", "iso-2022-kr"],
languages=["Korean"],
),
Encoding(
name="latin_1",
aliases=[
"iso-8859-1",
"iso8859-1",
"8859",
"cp819",
"latin",
"latin1",
"L1",
],
languages=["Western Europe"],
),
Encoding(
name="iso8859_2",
aliases=["iso-8859-2", "latin2", "L2"],
languages=["Central and Eastern Europe"],
),
Encoding(
name="iso8859_3",
aliases=["iso-8859-3", "latin3", "L3"],
languages=["Esperanto", "Maltese"],
),
Encoding(
name="iso8859_4",
aliases=["iso-8859-4", "latin4", "L4"],
languages=["Baltic languages"],
),
Encoding(
name="iso8859_5",
aliases=["iso-8859-5", "cyrillic"],
languages=["Bulgarian", "Byelorussian", "Macedonian", "Russian", "Serbian"],
),
Encoding(
name="iso8859_6",
aliases=["iso-8859-6", "arabic"],
languages=["Arabic"],
),
Encoding(
name="iso8859_7",
aliases=["iso-8859-7", "greek", "greek8"],
languages=["Greek"],
),
Encoding(
name="iso8859_8",
aliases=["iso-8859-8", "hebrew"],
languages=["Hebrew"],
),
Encoding(
name="iso8859_9",
aliases=["iso-8859-9", "latin5", "L5"],
languages=["Turkish"],
),
Encoding(
name="iso8859_10",
aliases=["iso-8859-10", "latin6", "L6"],
languages=["Nordic languages"],
),
Encoding(
name="iso8859_11",
aliases=["iso-8859-11", "thai"],
languages=["Thai languages"],
),
Encoding(
name="iso8859_13",
aliases=["iso-8859-13", "latin7", "L7"],
languages=["Baltic languages"],
),
Encoding(
name="iso8859_14",
aliases=["iso-8859-14", "latin8", "L8"],
languages=["Celtic languages"],
),
Encoding(
name="iso8859_15",
aliases=["iso-8859-15", "latin9", "L9"],
languages=["Western Europe"],
),
Encoding(
name="iso8859_16",
aliases=["iso-8859-16", "latin10", "L10"],
languages=["South-Eastern Europe"],
),
Encoding(
name="johab",
aliases=["cp1361", "ms1361"],
languages=["Korean"],
),
Encoding(
name="koi8_r",
aliases=[""],
languages=["Russian"],
),
Encoding(
name="koi8_t",
aliases=[""],
languages=["Tajik"],
),
Encoding(
name="koi8_u",
aliases=[""],
languages=["Ukrainian"],
),
Encoding(
name="kz1048",
aliases=["kz_1048", "strk1048_2002", "rk1048"],
languages=["Kazakh"],
),
Encoding(
name="mac_cyrillic",
aliases=["maccyrillic"],
languages=["Bulgarian", "Byelorussian", "Macedonian", "Russian", "Serbian"],
),
Encoding(
name="mac_greek",
aliases=["macgreek"],
languages=["Greek"],
),
Encoding(
name="mac_iceland",
aliases=["maciceland"],
languages=["Icelandic"],
),
Encoding(
name="mac_latin2",
aliases=["maclatin2", "maccentraleurope", "mac_centeuro"],
languages=["Central and Eastern Europe"],
),
Encoding(
name="mac_roman",
aliases=["macroman", "macintosh"],
languages=["Western Europe"],
),
Encoding(
name="mac_turkish",
aliases=["macturkish"],
languages=["Turkish"],
),
Encoding(
name="ptcp154",
aliases=["csptcp154", "pt154", "cp154", "cyrillic-asian"],
languages=["Kazakh"],
),
Encoding(
name="shift_jis",
aliases=["csshiftjis", "shiftjis", "sjis", "s_jis"],
languages=["Japanese"],
),
Encoding(
name="shift_jis_2004",
aliases=["shiftjis2004", "sjis_2004", "sjis2004"],
languages=["Japanese"],
),
Encoding(
name="shift_jisx0213",
aliases=["shiftjisx0213", "sjisx0213", "s_jisx0213"],
languages=["Japanese"],
),
Encoding(
name="utf_32",
aliases=["U32", "utf32"],
languages=["all languages"],
),
Encoding(
name="utf_32_be",
aliases=["UTF-32BE"],
languages=["all languages"],
),
Encoding(
name="utf_32_le",
aliases=["UTF-32LE"],
languages=["all languages"],
),
Encoding(
name="utf_16",
aliases=["U16", "utf16"],
languages=["all languages"],
),
Encoding(
name="utf_16_be",
aliases=["UTF-16BE"],
languages=["all languages"],
),
Encoding(
name="utf_16_le",
aliases=["UTF-16LE"],
languages=["all languages"],
),
Encoding(
name="utf_7",
aliases=["U7", "unicode-1-1-utf-7"],
languages=["all languages"],
),
Encoding(
name="utf_8",
aliases=["U8", "UTF", "utf8", "cp65001"],
languages=["all languages"],
),
Encoding(
name="utf_8_sig",
aliases=[""],
languages=["all languages"],
),
]