From e58ad330f4ab6900c12f84d0f242de1f0446cab2 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 3 Apr 2017 20:24:00 +0200 Subject: [PATCH] encode filenames as bytes to handle all locale setups This was failing on environments that did not have any LANG or LC_* locale variables set. This is a valid setup, and is common in headless setups, so it needs to be handled. This also adds a new pass of the test suite without the locale env vars set so that this situation is also tests on gitlab-ci, not only gpjenkins. The error this caused was: UnicodeEncodeError: 'ascii' codec can't encode characters in position 6-18: ordinal not in range(128) --- .gitlab-ci.yml | 2 +- fdroidserver/common.py | 23 +++++++++++++---------- fdroidserver/index.py | 8 ++++---- fdroidserver/update.py | 24 +++++++++++++----------- 4 files changed, 31 insertions(+), 26 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0d1713c0..cbda46b7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: fdroid/ci:server-20161223 +image: registry.gitlab.com/fdroid/ci-images:server-latest test: script: diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 42918ed7..0c12c3f9 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1700,7 +1700,8 @@ class KnownApks: def get_file_extension(filename): """get the normalized file extension, can be blank string but never None""" - + if isinstance(filename, bytes): + filename = filename.decode('utf-8') return os.path.splitext(filename)[1].lower()[1:] @@ -2333,15 +2334,17 @@ def get_per_app_repos(): def is_repo_file(filename): '''Whether the file in a repo is a build product to be delivered to users''' + if isinstance(filename, str): + filename = filename.encode('utf-8', errors="surrogateescape") return os.path.isfile(filename) \ - and not filename.endswith('.asc') \ - and not filename.endswith('.sig') \ + and not filename.endswith(b'.asc') \ + and not filename.endswith(b'.sig') \ and os.path.basename(filename) not in [ - 'index.jar', - 'index_unsigned.jar', - 'index.xml', - 'index.html', - 'index-v1.jar', - 'index-v1.json', - 'categories.txt', + b'index.jar', + b'index_unsigned.jar', + b'index.xml', + b'index.html', + b'index-v1.jar', + b'index-v1.json', + b'categories.txt', ] diff --git a/fdroidserver/index.py b/fdroidserver/index.py index c284ce50..b42c63d1 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -436,14 +436,14 @@ def make_v0(apps, apks, repodir, repodict, requestsdict): and common.config['make_current_version_link'] \ and repodir == 'repo': # only create these namefield = common.config['current_version_name_source'] - sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get(namefield)) - apklinkname = sanitized_name + '.apk' - current_version_path = os.path.join(repodir, current_version_file) + sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8')) + apklinkname = sanitized_name + b'.apk' + current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape') if os.path.islink(apklinkname): os.remove(apklinkname) os.symlink(current_version_path, apklinkname) # also symlink gpg signature, if it exists - for extension in ('.asc', '.sig'): + for extension in (b'.asc', b'.sig'): sigfile_path = current_version_path + extension if os.path.exists(sigfile_path): siglinkname = apklinkname + extension diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 6412029b..0e42d684 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -656,13 +656,15 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): cachechanged = False repo_files = [] + repodir = repodir.encode('utf-8') for name in os.listdir(repodir): file_extension = common.get_file_extension(name) if file_extension == 'apk' or file_extension == 'obb': continue filename = os.path.join(repodir, name) - if filename.endswith('_src.tar.gz'): - logging.debug('skipping source tarball: ' + filename) + name_utf8 = name.decode('utf-8') + if filename.endswith(b'_src.tar.gz'): + logging.debug('skipping source tarball: ' + filename.decode('utf-8')) continue if not common.is_repo_file(filename): continue @@ -683,34 +685,34 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): else: repo_file['added'] = datetime(*a[:6]) if repo_file.get('hash') == shasum: - logging.debug("Reading " + name + " from cache") + logging.debug("Reading " + name_utf8 + " from cache") usecache = True else: logging.debug("Ignoring stale cache data for " + name) if not usecache: - logging.debug("Processing " + name) + logging.debug("Processing " + name_utf8) repo_file = collections.OrderedDict() # TODO rename apkname globally to something more generic - repo_file['name'] = name - repo_file['apkName'] = name + repo_file['name'] = name_utf8 + repo_file['apkName'] = name_utf8 repo_file['hash'] = shasum repo_file['hashType'] = 'sha256' repo_file['versionCode'] = 0 repo_file['versionName'] = shasum # the static ID is the SHA256 unless it is set in the metadata repo_file['packageName'] = shasum - n = name.split('_') + n = name_utf8.split('_') if len(n) == 2: packageName = n[0] versionCode = n[1].split('.')[0] - if re.match(r'^-?[0-9]+$', versionCode) \ - and common.is_valid_package_name(name.split('_')[0]): + if re.match('^-?[0-9]+$', versionCode) \ + and common.is_valid_package_name(name_utf8.split('_')[0]): repo_file['packageName'] = packageName repo_file['versionCode'] = int(versionCode) - srcfilename = name + "_src.tar.gz" + srcfilename = name + b'_src.tar.gz' if os.path.exists(os.path.join(repodir, srcfilename)): - repo_file['srcname'] = srcfilename + repo_file['srcname'] = srcfilename.decode('utf-8') repo_file['size'] = stat.st_size apkcache[name] = repo_file -- 2.30.2