[WIP] Fix UTF-8 handling

src/modules/manifest.py

@@ -1077,7 +1077,7 @@ def set_content(self, content=None, excludes=EmptyI, pathname=None,
                 # together has to be solved somewhere else, though.)
                 if pathname:
                         try:
-                                with open(pathname, "r") as mfile:
+                                with open(pathname, "r", encoding='UTF-8') as mfile:
                                         content = mfile.read()
                         except EnvironmentError as e:
                                 raise apx._convert_error(e)

src/modules/misc.py

@@ -508,11 +508,6 @@ def setlocale(category, loc=None, printer=None):
                 printer("Unable to set locale{0}; locale package may be broken "
                     "or\nnot installed.  Reverting to C locale.".format(dl))
                 locale.setlocale(category, "C")
-# Correct preferred encoding so that we don't try to decode files with ascii codec
-        if locale.getpreferredencoding(False) != "UTF-8":
-                locale.setlocale(locale.LC_CTYPE, "en_US.UTF-8")
-                locale.setlocale(locale.LC_COLLATE, "en_US.UTF-8")
-
 def N_(message):
         """Return its argument; used to mark strings for localization when
         their use is delayed by the program."""

src/modules/search_storage.py

@@ -75,7 +75,7 @@ def consistent_open(data_list, directory, timeout = 1):
                         # in the function is greater than timeout.
                         try:
                                 f = os.path.join(directory, d.get_file_name())
-                                fh = open(f, 'r')
+                                fh = open(f, 'r', encoding='UTF-8')
                                 # If we get here, then the current index file
                                 # is present.
                                 if missing == None:

src/pkg/manifests/package:pkg.p5m

@@ -500,4 +500,3 @@ depend type=require fmri=library/python/pycurl-35
 depend type=require fmri=library/python/pyopenssl-35
 depend type=require fmri=library/python/rapidjson-35
 depend type=require fmri=library/python/six-35
-depend type=require fmri=locale/en