path: root/dodai/tools/himo.py

# Copyright (C) 2010  Leonard Thomas
#
# This file is part of Dodai.
#
# Dodai is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Dodai is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Dodai.  If not, see <http://www.gnu.org/licenses/>.

import chardet
import re
import sys
import unicodedata
from htmlentitydefs import name2codepoint
from htmlentitydefs import codepoint2name
from decimal import Decimal as D

class Himo(object):
    """
    A unicode-string object with some added features to help with
    unicode decoding and output conversions.
    """

    MAP = {169:u'(C)', 174:u'(R)', 8471:u'(P)'}

    def __init__(self, data, encoding=None):
        """
        data:       Accepts any type of string object (int, float,
                    string, unicode)

        encoding:   Character encoding to help with converting the input
                    into unicode

        The input data will be converted into an unicode object, unless
        the input data is already an unicode object.  If the param
        'encoding' is set, the input data will be converted to unicode
        using that value.  If no 'encoding' is given this object will
        attempt to figure out the encoding.  First the encoding of the
        operating system will be used.  If there are any errors, the
        chardet module will be used.  This object makes no guarantees
        that the correct encoding will be detected.

        """

        self._encoding = encoding or self._system_encoding()
        self.data = self._decode(data)

    def ascii(self):
        """
        Returns an ascii representation of this object value.
        Throws HimoAsciiError if this method was unable to
        convert a unicode character down to it's root character.
        For example if in your string you have a character
        like the letter 'e' but it has an accent mark over it,
        this method will convert that character to it's root
        character.  Thus 'e' with an accent mark over it will
        replaced with the regular letter 'e'.

        """
        out = []
        for char in self.data:
            if ord(char) < 127:
                out.append(char)
            elif ord(char) in self.MAP:
                out.append(self.MAP[ord(char)])
            else:
                num = unicodedata.decomposition(char).split(' ')[0]
                if num:
                    out.append(unichr(int(num, 16)))
                else:
                    print char
                    raise HimoAsciiError("Unable to convert 'u{0}' "\
                                    "character to ascii".format(ord(char)))
        return str(''.join(out))

    def html(self):
        """
        Returns a unicode string containing this object's value
        html enetity encoded.
        """
        out = []
        for char in self.data:
            out.append(self._html_char_encode(char))
        return ''.join(out)

    def decimal(self):
        """
        Returns a decimal object with the value of this object

        """

        return D(self.data)

    def _decode(self, data):
        # Returns a unicode string.  If data contains any html encoded
        # characters, the characters will be converted to their unicode
        # equivalent

        data = self._as_unicode(data)
        expression = re.compile(r'&(#?)(x?)(\w+);')
        return expression.subn(self._html_decode, data)[0]

    def _as_unicode(self, data):
        # Returns string as a unicode string

        if not isinstance(data, unicode):
            if not isinstance(data, str):
                data = str(data)
            try:
                data = data.decode(self._encoding)
            except UnicodeDecodeError:
                info = chardet.detect(data)
                self.encoding = info['encoding']
                data = data.decode(info['encoding'])
        return unicodedata.normalize('NFC', data)

    def _html_char_encode(self, char):
        # Returns an html version of the char

        number = ord(char)
        try:
            char = "&{0};".format(codepoint2name[number])
        except KeyError:
            if number > 127:
                char = "&#{0};".format(number)
        return char

    def _html_decode(self, values):
        # Returns the unicode character from the re.subn

        value = values.group(3)
        if values.group(1):
            if values.group(2):
                return unichr(int('0x{0}'.format(value), 16))
            else:
                return unichr(int(value))
        else:
            try:
                char = name2codepoint[value]
            except KeyError:
                return values.group()
            else:
                return unichr(char)

    def _system_encoding(self):
        # Returns the character encoding of the system

        encoding = sys.getfilesystemencoding()
        if not encoding:
            encoding = sys.getdefaultencoding()
        return encoding

    #def __cmp__(self, other):
    #    if self.__eq__(other):
    #        return 1
    #    else:
    #        pool = [str(self.data), str(other)]
    #        pool.sort()
    #        if pool[0] == self.data:
    #            return -1
    #        else:
    #            return 1


    def _is_himo(self, other):
        if hasattr(other, '_is_himo'):
            return True
        return False

    def __len__(self):
        return len(self.data)

    def __repr__(self):
        return repr(self.data)

    def __str__(self):
        return self.data.encode(self._encoding)

    def __iter__(self):
        for char in self.data:
            yield char

    def __int__(self):
        return int(self.data)

    def __float__(self):
        return float(self.data)

    def __eq__(self, other):
        if self._is_himo(other):
            other = other.data
        return self.data.__eq__(other)

    def __ne__(self, other):
        if self._is_himo(other):
            other = other.data
        return self.data.__ne__(other)

    def __gt__(self, other):
        if self._is_himo(other):
            other = other.data
        lines = [self.data, other]
        lines.sort()
        if lines[0] == self.data:
            return True
        else:
            return False

    def __lt__(self, other):
        if self._is_himo(other):
            other = other.data
        lines = [self.data, other]
        lines.sort()
        if lines[0] != self.data:
            return True
        else:
            return False

    def __cmp__(self, other):
        if self.__eq__(other):
            return 0
        elif self.__lt__(other):
            return -1
        else:
            return 1

    def __unicode__(self):
        return self.data

    def capitalize(self, *args, **kargs):
        return self.data.capitalize(*args, **kargs)

    def center(self, *args, **kargs):
        return self.data.center(*args, **kargs)

    def count(self, *args, **kargs):
        return self.data.count(*args, **kargs)

    def decode(self, *args, **kargs):
        return self.data.decode(*args, **kargs)

    def encode(self, *args, **kargs):
        return self.data.encode(*args, **kargs)

    def encode(self, *args, **kargs):
        return self.data.encode(*args, **kargs)

    def endswith(self, *args, **kargs):
        return self.data.endswith(*args, **kargs)

    def expandtabs(self, *args, **kargs):
        return self.data.expandtabs(*args, **kargs)

    def find(self, *args, **kargs):
        return self.data.find(*args, **kargs)

    def format(self, *args, **kargs):
        return self.data.format(*args, **kargs)

    def index(self, *args, **kargs):
        return self.data.index(*args, **kargs)

    def isalnum(self, *args, **kargs):
        return self.data.isalnum(*args, **kargs)

    def isalpha(self, *args, **kargs):
        return self.data.isalpha(*args, **kargs)

    def isdecimal(self, *args, **kargs):
        return self.data.isdecimal(*args, **kargs)

    def isdigit(self, *args, **kargs):
        return self.data.isdigit(*args, **kargs)

    def islower(self, *args, **kargs):
        return self.data.islower(*args, **kargs)

    def isnumeric(self, *args, **kargs):
        return self.data.isnumeric(*args, **kargs)

    def isspace(self, *args, **kargs):
        return self.data.isspace(*args, **kargs)

    def istitle(self, *args, **kargs):
        return self.data.istitle(*args, **kargs)

    def isupper(self, *args, **kargs):
        return self.data.isupper(*args, **kargs)

    def join(self, *args, **kargs):
        return self.data.join(*args, **kargs)

    def ljust(self, *args, **kargs):
        return self.data.ljust(*args, **kargs)

    def lower(self, *args, **kargs):
        return self.data.lower(*args, **kargs)

    def lstrip(self, *args, **kargs):
        return self.data.lstrip(*args, **kargs)

    def partition(self, *args, **kargs):
        return self.data.partition(*args, **kargs)

    def replace(self, *args, **kargs):
        return self.data.replace(*args, **kargs)

    def rfind(self, *args, **kargs):
        return self.data.rfind(*args, **kargs)

    def rindex(self, *args, **kargs):
        return self.data.rindex(*args, **kargs)

    def rjust(self, *args, **kargs):
        return self.data.rjust(*args, **kargs)

    def rpartition(self, *args, **kargs):
        return self.data.rpartition(*args, **kargs)

    def rsplit(self, *args, **kargs):
        return self.data.rsplit(*args, **kargs)

    def rstrip(self, *args, **kargs):
        return self.data.rstrip(*args, **kargs)

    def split(self, *args, **kargs):
        return self.data.split(*args, **kargs)

    def splitlines(self, *args, **kargs):
        return self.data.splitlines(*args, **kargs)

    def startswith(self, *args, **kargs):
        return self.data.startswith(*args, **kargs)

    def strip(self, *args, **kargs):
        return self.data.strip(*args, **kargs)

    def swapcase(self, *args, **kargs):
        return self.data.swapcase(*args, **kargs)

    def title(self, *args, **kargs):
        return self.data.title(*args, **kargs)

    def translate(self, *args, **kargs):
        return self.data.translate(*args, **kargs)

    def upper(self, *args, **kargs):
        return self.data.upper(*args, **kargs)

    def zfill(self, *args, **kargs):
        return self.data.zfill(*args, **kargs)

class HimoAsciiError(Exception):
    pass