-#!/usr/bin/env python2
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
#
# update.py - part of the FDroid server tools
# Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
import zipfile
import hashlib
import pickle
+import urllib.parse
from datetime import datetime, timedelta
from xml.dom.minidom import Document
from argparse import ArgumentParser
from pyasn1.error import PyAsn1Error
from pyasn1.codec.der import decoder, encoder
from pyasn1_modules import rfc2315
-from hashlib import md5
from binascii import hexlify, unhexlify
from PIL import Image
import logging
-import common
-import metadata
-from common import FDroidPopen, SdkToolsPopen
-from metadata import MetaDataException
+from . import common
+from . import metadata
+from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
+from .metadata import MetaDataException
+
+METADATA_VERSION = 16
screen_densities = ['640', '480', '320', '240', '160', '120']
requiresroot = 'Yes'
else:
requiresroot = 'No'
- wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s}}\n' % (
+ wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
appid,
app.Name,
time.strftime('%Y-%m-%d', app.added) if app.added else '',
app.Bitcoin,
app.Litecoin,
app.License,
- requiresroot)
+ requiresroot,
+ app.AuthorName,
+ app.AuthorEmail)
if app.Provides:
wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
gotcurrentver = True
apklist.append(apk)
# Include ones we can't build, as a special case...
- for thisbuild in app.builds:
- if thisbuild['disable']:
- if thisbuild['vercode'] == app.CurrentVersionCode:
+ for build in app.builds:
+ if build.disable:
+ if build.vercode == app.CurrentVersionCode:
cantupdate = True
# TODO: Nasty: vercode is a string in the build, and an int elsewhere
- apklist.append({'versioncode': int(thisbuild['vercode']),
- 'version': thisbuild['version'],
- 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(thisbuild['disable']),
+ apklist.append({'versioncode': int(build.vercode),
+ 'version': build.version,
+ 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
})
else:
builtit = False
for apk in apklist:
- if apk['versioncode'] == int(thisbuild['vercode']):
+ if apk['versioncode'] == int(build.vercode):
builtit = True
break
if not builtit:
buildfails = True
- apklist.append({'versioncode': int(thisbuild['vercode']),
- 'version': thisbuild['version'],
- 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, thisbuild['vercode']),
+ apklist.append({'versioncode': int(build.vercode),
+ 'version': build.version,
+ 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
})
if app.CurrentVersionCode == '0':
cantupdate = True
if validapks == 0 and not app.Disabled:
wikidata += '\n[[Category:Apps with no packages]]\n'
if cantupdate and not app.Disabled:
- wikidata += "\n[[Category:Apps we can't update]]\n"
+ wikidata += "\n[[Category:Apps we cannot update]]\n"
if buildfails and not app.Disabled:
wikidata += "\n[[Category:Apps with failing builds]]\n"
elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
:param apkcache: current apk cache information
:param repodirs: the repo directories to process
"""
- for appid, app in apps.iteritems():
+ for appid, app in apps.items():
for build in app.builds:
- if not build['disable']:
+ if not build.disable:
continue
- apkfilename = appid + '_' + str(build['vercode']) + '.apk'
+ apkfilename = appid + '_' + str(build.vercode) + '.apk'
iconfilename = "%s.%s.png" % (
appid,
- build['vercode'])
+ build.vercode)
for repodir in repodirs:
files = [
os.path.join(repodir, apkfilename),
if not os.path.isfile(iconpath):
return
+ fp = None
try:
- im = Image.open(iconpath)
+ fp = open(iconpath, 'rb')
+ im = Image.open(fp)
size = dpi_to_px(density)
if any(length > size for length in im.size):
iconpath, oldsize, im.size))
im.save(iconpath, "PNG")
- except Exception, e:
+ except Exception as e:
logging.error("Failed resizing {0} - {1}".format(iconpath, e))
+ finally:
+ if fp:
+ fp.close()
+
def resize_all_icons(repodirs):
"""Resize all icons that exceed the max size
cert = None
# verify the jar signature is correct
- args = ['jarsigner', '-verify', apkpath]
+ args = [config['jarsigner'], '-verify', apkpath]
p = FDroidPopen(args)
if p.returncode != 0:
logging.critical(apkpath + " has a bad signature!")
cert_encoded = encoder.encode(certificates)[4:]
- return md5(cert_encoded.encode('hex')).hexdigest()
+ return hashlib.md5(hexlify(cert_encoded)).hexdigest()
-def scan_apks(apps, apkcache, repodir, knownapks):
+def get_icon_bytes(apkzip, iconsrc):
+ '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
+ try:
+ return apkzip.read(iconsrc)
+ except KeyError:
+ return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
+
+
+def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
"""Scan the apks in the given repo directory.
This also extracts the icons.
:param apkcache: current apk cache information
:param repodir: repo directory to scan
:param knownapks: known apks info
+ :param use_date_from_apk: use date from APK (instead of current date)
+ for newly added APKs
:returns: (apks, cachechanged) where apks is a list of apk information,
and cachechanged is True if the apkcache got changed.
"""
icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
sdkversion_pat = re.compile(".*'([0-9]*)'.*")
- string_pat = re.compile(".*'([^']*)'.*")
+ string_pat = re.compile(".* name='([^']*)'.*")
for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
apkfilename = apkfile[len(repodir) + 1:]
usecache = False
if apkfilename in apkcache:
- thisinfo = apkcache[apkfilename]
- if thisinfo['sha256'] == shasum:
+ apk = apkcache[apkfilename]
+ if apk['sha256'] == shasum:
logging.debug("Reading " + apkfilename + " from cache")
usecache = True
else:
if not usecache:
logging.debug("Processing " + apkfilename)
- thisinfo = {}
- thisinfo['apkname'] = apkfilename
- thisinfo['sha256'] = shasum
+ apk = {}
+ apk['apkname'] = apkfilename
+ apk['sha256'] = shasum
srcfilename = apkfilename[:-4] + "_src.tar.gz"
if os.path.exists(os.path.join(repodir, srcfilename)):
- thisinfo['srcname'] = srcfilename
- thisinfo['size'] = os.path.getsize(apkfile)
- thisinfo['permissions'] = set()
- thisinfo['features'] = set()
- thisinfo['icons_src'] = {}
- thisinfo['icons'] = {}
+ apk['srcname'] = srcfilename
+ apk['size'] = os.path.getsize(apkfile)
+ apk['permissions'] = set()
+ apk['features'] = set()
+ apk['icons_src'] = {}
+ apk['icons'] = {}
p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
if p.returncode != 0:
if options.delete_unknown:
for line in p.output.splitlines():
if line.startswith("package:"):
try:
- thisinfo['id'] = re.match(name_pat, line).group(1)
- thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
- thisinfo['version'] = re.match(vername_pat, line).group(1)
- except Exception, e:
+ apk['id'] = re.match(name_pat, line).group(1)
+ apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
+ apk['version'] = re.match(vername_pat, line).group(1)
+ except Exception as e:
logging.error("Package matching failed: " + str(e))
logging.info("Line was: " + line)
sys.exit(1)
elif line.startswith("application:"):
- thisinfo['name'] = re.match(label_pat, line).group(1)
+ apk['name'] = re.match(label_pat, line).group(1)
# Keep path to non-dpi icon in case we need it
match = re.match(icon_pat_nodpi, line)
if match:
- thisinfo['icons_src']['-1'] = match.group(1)
+ apk['icons_src']['-1'] = match.group(1)
elif line.startswith("launchable-activity:"):
# Only use launchable-activity as fallback to application
- if not thisinfo['name']:
- thisinfo['name'] = re.match(label_pat, line).group(1)
- if '-1' not in thisinfo['icons_src']:
+ if not apk['name']:
+ apk['name'] = re.match(label_pat, line).group(1)
+ if '-1' not in apk['icons_src']:
match = re.match(icon_pat_nodpi, line)
if match:
- thisinfo['icons_src']['-1'] = match.group(1)
+ apk['icons_src']['-1'] = match.group(1)
elif line.startswith("application-icon-"):
match = re.match(icon_pat, line)
if match:
density = match.group(1)
path = match.group(2)
- thisinfo['icons_src'][density] = path
+ apk['icons_src'][density] = path
elif line.startswith("sdkVersion:"):
m = re.match(sdkversion_pat, line)
if m is None:
logging.error(line.replace('sdkVersion:', '')
+ ' is not a valid minSdkVersion!')
else:
- thisinfo['sdkversion'] = m.group(1)
+ apk['minSdkVersion'] = m.group(1)
+ # if target not set, default to min
+ if 'targetSdkVersion' not in apk:
+ apk['targetSdkVersion'] = m.group(1)
+ elif line.startswith("targetSdkVersion:"):
+ m = re.match(sdkversion_pat, line)
+ if m is None:
+ logging.error(line.replace('targetSdkVersion:', '')
+ + ' is not a valid targetSdkVersion!')
+ else:
+ apk['targetSdkVersion'] = m.group(1)
elif line.startswith("maxSdkVersion:"):
- thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
+ apk['maxSdkVersion'] = re.match(sdkversion_pat, line).group(1)
elif line.startswith("native-code:"):
- thisinfo['nativecode'] = []
+ apk['nativecode'] = []
for arch in line[13:].split(' '):
- thisinfo['nativecode'].append(arch[1:-1])
+ apk['nativecode'].append(arch[1:-1])
elif line.startswith("uses-permission:"):
perm = re.match(string_pat, line).group(1)
if perm.startswith("android.permission."):
perm = perm[19:]
- thisinfo['permissions'].add(perm)
+ apk['permissions'].add(perm)
elif line.startswith("uses-feature:"):
perm = re.match(string_pat, line).group(1)
# Filter out this, it's only added with the latest SDK tools and
and perm != "android.hardware.screen.landscape":
if perm.startswith("android.feature."):
perm = perm[16:]
- thisinfo['features'].add(perm)
+ apk['features'].add(perm)
- if 'sdkversion' not in thisinfo:
+ if 'minSdkVersion' not in apk:
logging.warn("No SDK version information found in {0}".format(apkfile))
- thisinfo['sdkversion'] = 0
+ apk['minSdkVersion'] = 1
# Check for debuggable apks...
if common.isApkDebuggable(apkfile, config):
- logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
+ logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
# Get the signature (or md5 of, to be precise)...
logging.debug('Getting signature of {0}'.format(apkfile))
- thisinfo['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
- if not thisinfo['sig']:
+ apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
+ if not apk['sig']:
logging.critical("Failed to get apk signature")
sys.exit(1)
- apk = zipfile.ZipFile(apkfile, 'r')
+ apkzip = zipfile.ZipFile(apkfile, 'r')
# if an APK has files newer than the system time, suggest updating
# the system clock. This is useful for offline systems, used for
# signing, which do not have another source of clock sync info. It
# has to be more than 24 hours newer because ZIP/APK files do not
# store timezone info
- info = apk.getinfo('AndroidManifest.xml')
- dt_obj = datetime(*info.date_time)
- checkdt = dt_obj - timedelta(1)
- if datetime.today() < checkdt:
- logging.warn('System clock is older than manifest in: '
- + apkfilename + '\nSet clock to that time using:\n'
- + 'sudo date -s "' + str(dt_obj) + '"')
+ manifest = apkzip.getinfo('AndroidManifest.xml')
+ if manifest.date_time[1] == 0: # month can't be zero
+ logging.debug('AndroidManifest.xml has no date')
+ else:
+ dt_obj = datetime(*manifest.date_time)
+ checkdt = dt_obj - timedelta(1)
+ if datetime.today() < checkdt:
+ logging.warn('System clock is older than manifest in: '
+ + apkfilename
+ + '\nSet clock to that time using:\n'
+ + 'sudo date -s "' + str(dt_obj) + '"')
iconfilename = "%s.%s.png" % (
- thisinfo['id'],
- thisinfo['versioncode'])
+ apk['id'],
+ apk['versioncode'])
# Extract the icon file...
empty_densities = []
for density in screen_densities:
- if density not in thisinfo['icons_src']:
+ if density not in apk['icons_src']:
empty_densities.append(density)
continue
- iconsrc = thisinfo['icons_src'][density]
+ iconsrc = apk['icons_src'][density]
icon_dir = get_icon_dir(repodir, density)
icondest = os.path.join(icon_dir, iconfilename)
try:
with open(icondest, 'wb') as f:
- f.write(apk.read(iconsrc))
- thisinfo['icons'][density] = iconfilename
+ f.write(get_icon_bytes(apkzip, iconsrc))
+ apk['icons'][density] = iconfilename
except:
logging.warn("Error retrieving icon file")
- del thisinfo['icons'][density]
- del thisinfo['icons_src'][density]
+ del apk['icons'][density]
+ del apk['icons_src'][density]
empty_densities.append(density)
- if '-1' in thisinfo['icons_src']:
- iconsrc = thisinfo['icons_src']['-1']
+ if '-1' in apk['icons_src']:
+ iconsrc = apk['icons_src']['-1']
iconpath = os.path.join(
get_icon_dir(repodir, '0'), iconfilename)
with open(iconpath, 'wb') as f:
- f.write(apk.read(iconsrc))
+ f.write(get_icon_bytes(apkzip, iconsrc))
try:
im = Image.open(iconpath)
dpi = px_to_dpi(im.size[0])
for density in screen_densities:
- if density in thisinfo['icons']:
+ if density in apk['icons']:
break
if density == screen_densities[-1] or dpi >= int(density):
- thisinfo['icons'][density] = iconfilename
+ apk['icons'][density] = iconfilename
shutil.move(iconpath,
os.path.join(get_icon_dir(repodir, density), iconfilename))
empty_densities.remove(density)
break
- except Exception, e:
+ except Exception as e:
logging.warn("Failed reading {0} - {1}".format(iconpath, e))
- if thisinfo['icons']:
- thisinfo['icon'] = iconfilename
+ if apk['icons']:
+ apk['icon'] = iconfilename
- apk.close()
+ apkzip.close()
# First try resizing down to not lose quality
last_density = None
get_icon_dir(repodir, last_density), iconfilename)
iconpath = os.path.join(
get_icon_dir(repodir, density), iconfilename)
+ fp = None
try:
- im = Image.open(last_iconpath)
- except:
- logging.warn("Invalid image file at %s" % last_iconpath)
- continue
+ fp = open(last_iconpath, 'rb')
+ im = Image.open(fp)
- size = dpi_to_px(density)
+ size = dpi_to_px(density)
- im.thumbnail((size, size), Image.ANTIALIAS)
- im.save(iconpath, "PNG")
- empty_densities.remove(density)
+ im.thumbnail((size, size), Image.ANTIALIAS)
+ im.save(iconpath, "PNG")
+ empty_densities.remove(density)
+ except:
+ logging.warning("Invalid image file at %s" % last_iconpath)
+ finally:
+ if fp:
+ fp.close()
# Then just copy from the highest resolution available
last_density = None
# Copy from icons-mdpi to icons since mdpi is the baseline density
baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
if os.path.isfile(baseline):
- thisinfo['icons']['0'] = iconfilename
+ apk['icons']['0'] = iconfilename
shutil.copyfile(baseline,
os.path.join(get_icon_dir(repodir, '0'), iconfilename))
# Record in known apks, getting the added date at the same time..
- added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
+ added = knownapks.recordapk(apk['apkname'], apk['id'])
if added:
- thisinfo['added'] = added
+ if use_date_from_apk and manifest.date_time[1] != 0:
+ added = datetime(*manifest.date_time).timetuple()
+ logging.debug("Using date from APK")
- apkcache[apkfilename] = thisinfo
+ apk['added'] = added
+
+ apkcache[apkfilename] = apk
cachechanged = True
- apks.append(thisinfo)
+ apks.append(apk)
return apks, cachechanged
def cert_fingerprint(data):
digest = hashlib.sha256(data).digest()
ret = []
- ret.append(' '.join("%02X" % ord(b) for b in digest))
+ ret.append(' '.join("%02X" % b for b in bytearray(digest)))
return " ".join(ret)
if 'repo_pubkey' in config:
pubkey = unhexlify(config['repo_pubkey'])
else:
- p = FDroidPopen(['keytool', '-exportcert',
- '-alias', config['repo_keyalias'],
- '-keystore', config['keystore'],
- '-storepass:file', config['keystorepassfile']]
- + config['smartcardoptions'], output=False)
+ p = FDroidPopenBytes([config['keytool'], '-exportcert',
+ '-alias', config['repo_keyalias'],
+ '-keystore', config['keystore'],
+ '-storepass:file', config['keystorepassfile']]
+ + config['smartcardoptions'],
+ output=False, stderr_to_stdout=False)
if p.returncode != 0 or len(p.output) < 20:
msg = "Failed to get repo pubkey!"
if config['keystore'] == 'NONE':
repoel = doc.createElement("repo")
+ mirrorcheckfailed = False
+ for mirror in config.get('mirrors', []):
+ base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
+ if config.get('nonstandardwebroot') is not True and base != 'fdroid':
+ logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
+ mirrorcheckfailed = True
+ if mirrorcheckfailed:
+ sys.exit(1)
+
if archive:
repoel.setAttribute("name", config['archive_name'])
if config['repo_maxage'] != 0:
repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
repoel.setAttribute("url", config['archive_url'])
addElement('description', config['archive_description'], doc, repoel)
+ urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
+ for mirror in config.get('mirrors', []):
+ addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
else:
repoel.setAttribute("name", config['repo_name'])
repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
repoel.setAttribute("url", config['repo_url'])
addElement('description', config['repo_description'], doc, repoel)
+ urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
+ for mirror in config.get('mirrors', []):
+ addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
- repoel.setAttribute("version", "14")
+ repoel.setAttribute("version", str(METADATA_VERSION))
repoel.setAttribute("timestamp", str(int(time.time())))
nosigningkey = False
logging.warning("\tfdroid update --create-key")
sys.exit(1)
- repoel.setAttribute("pubkey", extract_pubkey())
+ repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
root.appendChild(repoel)
for appid in sortedids:
addElement('source', app.SourceCode, doc, apel)
addElement('tracker', app.IssueTracker, doc, apel)
addElementNonEmpty('changelog', app.Changelog, doc, apel)
+ addElementNonEmpty('author', app.AuthorName, doc, apel)
+ addElementNonEmpty('email', app.AuthorEmail, doc, apel)
addElementNonEmpty('donate', app.Donate, doc, apel)
addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
apkel.appendChild(hashel)
addElement('sig', apk['sig'], doc, apkel)
addElement('size', str(apk['size']), doc, apkel)
- addElement('sdkver', str(apk['sdkversion']), doc, apkel)
- if 'maxsdkversion' in apk:
- addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
+ addElement('sdkver', str(apk['minSdkVersion']), doc, apkel)
+ if 'targetSdkVersion' in apk:
+ addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
+ if 'maxSdkVersion' in apk:
+ addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
if 'added' in apk:
addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
os.symlink(sigfile_path, siglinkname)
if options.pretty:
- output = doc.toprettyxml()
+ output = doc.toprettyxml(encoding='utf-8')
else:
- output = doc.toxml()
+ output = doc.toxml(encoding='utf-8')
with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
f.write(output)
if os.path.exists(signed):
os.remove(signed)
else:
- args = ['jarsigner', '-keystore', config['keystore'],
+ args = [config['jarsigner'], '-keystore', config['keystore'],
'-storepass:file', config['keystorepassfile'],
'-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
signed, config['repo_keyalias']]
catdata = ''
for cat in categories:
catdata += cat + '\n'
- with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
+ with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
f.write(catdata)
def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
- for appid, app in apps.iteritems():
+ for appid, app in apps.items():
if app.ArchivePolicy:
keepversions = int(app.ArchivePolicy[:-9])
to_path = os.path.join(to_dir, filename)
shutil.move(from_path, to_path)
+ logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
+ .format(appid, len(apks), keepversions, len(archapks)))
+
if len(apks) > keepversions:
apklist = filter_apk_list_sorted(apks)
# Move back the ones we don't want.
help="Clean update - don't uses caches, reprocess all apks")
parser.add_argument("--nosign", action="store_true", default=False,
help="When configured for signed indexes, create only unsigned indexes at this stage")
+ parser.add_argument("--use-date-from-apk", action="store_true", default=False,
+ help="Use date from apk instead of current time for newly added apks")
options = parser.parse_args()
config = common.read_config(options)
+ if not ('jarsigner' in config and 'keytool' in config):
+ logging.critical('Java JDK not found! Install in standard location or set java_paths!')
+ sys.exit(1)
+
repodirs = ['repo']
if config['archive_older'] != 0:
repodirs.append('archive')
# Generate a list of categories...
categories = set()
- for app in apps.itervalues():
+ for app in apps.values():
categories.update(app.Categories)
# Read known apks data (will be updated and written back when we've finished)
apkcachefile = os.path.join('tmp', 'apkcache')
if not options.clean and os.path.exists(apkcachefile):
with open(apkcachefile, 'rb') as cf:
- apkcache = pickle.load(cf)
+ apkcache = pickle.load(cf, encoding='utf-8')
+ if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
+ apkcache = {}
else:
apkcache = {}
delete_disabled_builds(apps, apkcache, repodirs)
# Scan all apks in the main repo
- apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks)
+ apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
# Generate warnings for apk's with no metadata (or create skeleton
# metadata files, if requested on the command line)
if 'name' not in apk:
logging.error(apk['id'] + ' does not have a name! Skipping...')
continue
- f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
+ f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
f.write("License:Unknown\n")
f.write("Web Site:\n")
f.write("Source Code:\n")
# Scan the archive repo for apks as well
if len(repodirs) > 1:
- archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
+ archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
if cc:
cachechanged = True
else:
# level. When doing this, we use the info from the most recent version's apk.
# We deal with figuring out when the app was added and last updated at the
# same time.
- for appid, app in apps.iteritems():
+ for appid, app in apps.items():
bestver = 0
for apk in apks + archapks:
if apk['id'] == appid:
if app.Name is None:
app.Name = bestapk['name']
app.icon = bestapk['icon'] if 'icon' in bestapk else None
+ if app.CurrentVersionCode is None:
+ app.CurrentVersionCode = str(bestver)
# Sort the app list by name, then the web site doesn't have to by default.
# (we had to wait until we'd scanned the apks to do this, because mostly the
# name comes from there!)
- sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid].Name.upper())
+ sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
# APKs are placed into multiple repos based on the app package, providing
# per-app subscription feeds for nightly builds and things like it
if config['per_app_repos']:
add_apks_to_per_app_repos(repodirs[0], apks)
- for appid, app in apps.iteritems():
+ for appid, app in apps.items():
repodir = os.path.join(appid, 'fdroid', 'repo')
appdict = dict()
appdict[appid] = app
# Generate latest apps data for widget
if os.path.exists(os.path.join('stats', 'latestapps.txt')):
data = ''
- for line in file(os.path.join('stats', 'latestapps.txt')):
- appid = line.rstrip()
- data += appid + "\t"
- app = apps[appid]
- data += app.Name + "\t"
- if app.icon is not None:
- data += app.icon + "\t"
- data += app.License + "\n"
- with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
+ with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
+ for line in f:
+ appid = line.rstrip()
+ data += appid + "\t"
+ app = apps[appid]
+ data += app.Name + "\t"
+ if app.icon is not None:
+ data += app.icon + "\t"
+ data += app.License + "\n"
+ with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
f.write(data)
if cachechanged:
+ apkcache["METADATA_VERSION"] = METADATA_VERSION
with open(apkcachefile, 'wb') as cf:
pickle.dump(apkcache, cf)
~ianmdlvl / fdroidserver.git / blobdiff