use UTF8 as default instead of ASCII for .java .gradle pom.xml

.java .gradle and XML files all can use any encoding.  Most code is ASCII,
but authors' names, etc. can easily be non-ASCII.  UTF-8 is by far the most
common file encoding.  While UTF-8 is the default encoding inside the code
in Python 3, it still has to deal with the real world, so the encoding
needs to be explicitly set when reading and writing files. So this switches
fdroidserver to expect UTF-8 instead of ASCII when parsing these files. For
now, this commit means that we only support UTF-8 encoded *.java, pom.xml
or *.gradle files.  Ideally, the code would detect the encoding and use the
actual one, but that's a lot more work, and its something that will not
happen often. We can cross that bridge when we come to it.

One approach, which is taken in the commit when possible, is to keep the
data as `bytes`, in which case the encoding doesn't matter.

This also fixes this crash when parsing gradle and maven files with
non-ASCII chars:

ERROR: test_adapt_gradle (__main__.BuildTest)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/var/lib/jenkins/workspace/fdroidserver-eighthave/tests/build.TestCase", line 59, in test_adapt_gradle
    fdroidserver.build.adapt_gradle(testsdir)
  File "/var/lib/jenkins/workspace/fdroidserver-eighthave/fdroidserver/build.py", line 445, in adapt_gradle
    path)
  File "/var/lib/jenkins/workspace/fdroidserver-eighthave/fdroidserver/common.py", line 188, in regsub_file
    text = f.read()
  File "/usr/lib/python3.4/encodings/ascii.py", line 26, in decode
    return codecs.ascii_decode(input, self.errors)[0]
UnicodeDecodeError: 'ascii' codec can't decode byte 0xe2 in position 9460: ordinal not in range(128)

diff --git a/fdroidserver/common.py b/fdroidserver/common.py
index 0a3c8e751acb04f6cf37dcb0d18908ef35a8d956..03a8743d2686efd73b807be1361b741e4ebcd237 100644 (file)
--- a/fdroidserver/common.py
+++ b/fdroidserver/common.py
@@ -184,10 +184,10 @@ def fill_config_defaults(thisconfig):
 
 
 def regsub_file(pattern, repl, path):
-    with open(path, 'r') as f:
+    with open(path, 'rb') as f:
         text = f.read()
-    text = re.sub(pattern, repl, text)
-    with open(path, 'w') as f:
+    text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
+    with open(path, 'wb') as f:
         f.write(text)
 
 
@@ -1724,14 +1724,14 @@ def remove_signing_keys(build_dir):
         if 'build.gradle' in files:
             path = os.path.join(root, 'build.gradle')
 
-            with open(path, "r") as o:
+            with open(path, "r", encoding='utf8') as o:
                 lines = o.readlines()
 
             changed = False
 
             opened = 0
             i = 0
-            with open(path, "w") as o:
+            with open(path, "w", encoding='utf8') as o:
                 while i < len(lines):
                     line = lines[i]
                     i += 1

diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py
index a9be8ef03d7c51e7d254b9ef9dfc7bbc4b7517ab..ded6cd1a4057c9e345f715ab89a7b5fc9b534339 100644 (file)
--- a/fdroidserver/metadata.py
+++ b/fdroidserver/metadata.py
@@ -836,7 +836,7 @@ def get_default_app_info(metadatapath=None):
             for root, dirs, files in os.walk(os.getcwd()):
                 if 'build.gradle' in files:
                     p = os.path.join(root, 'build.gradle')
-                    with open(p) as f:
+                    with open(p, 'rb') as f:
                         data = f.read()
                     m = pattern.search(data)
                     if m:

diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py
index 53c938fbe339e7b33560ca6a7aa220dac5a14560..41ecbb50c286018fced962fb348f882ca4cd4b41 100644 (file)
--- a/fdroidserver/scanner.py
+++ b/fdroidserver/scanner.py
@@ -199,7 +199,7 @@ def scan_source(build_dir, root_dir, build):
             elif ext == 'java':
                 if not os.path.isfile(fp):
                     continue
-                with open(fp, 'r') as f:
+                with open(fp, 'r', encoding='utf8') as f:
                     for line in f:
                         if 'DexClassLoader' in line:
                             count += handleproblem('DexClassLoader', fd, fp)
@@ -208,7 +208,7 @@ def scan_source(build_dir, root_dir, build):
             elif ext == 'gradle':
                 if not os.path.isfile(fp):
                     continue
-                with open(fp, 'r') as f:
+                with open(fp, 'r', encoding='utf8') as f:
                     lines = f.readlines()
                 for i, line in enumerate(lines):
                     if is_used_by_gradle(line):

diff --git a/tests/build.TestCase b/tests/build.TestCase
index b3b90fc3ded775290d8ea6f861931e5b2da8fff9..369c3836f250996e2ee0cd995e760aeb4b0f6ebb 100755 (executable)
--- a/tests/build.TestCase
+++ b/tests/build.TestCase
@@ -57,13 +57,13 @@ class BuildTest(unittest.TestCase):
         fdroidserver.build.config = {}
         fdroidserver.build.config['build_tools'] = teststring
         fdroidserver.build.adapt_gradle(testsdir)
-        pattern = re.compile("buildToolsVersion[\s=]+'%s'\s+" % teststring)
+        pattern = re.compile(bytes("buildToolsVersion[\s=]+'%s'\s+" % teststring, 'utf8'))
         for p in ('source-files/fdroid/fdroidclient/build.gradle',
                   'source-files/Zillode/syncthing-silk/build.gradle',
                   'source-files/open-keychain/open-keychain/build.gradle',
                   'source-files/osmandapp/osmand/build.gradle',
                   'source-files/open-keychain/open-keychain/OpenKeychain/build.gradle'):
-            with open(os.path.join(testsdir, p), 'r') as f:
+            with open(os.path.join(testsdir, p), 'rb') as f:
                 filedata = f.read()
             self.assertIsNotNone(pattern.search(filedata))