chiark / gitweb /
Merge branch 'repo-update-break-down' into 'master'
[fdroidserver.git] / fdroidserver / update.py
1 #!/usr/bin/env python3
2 #
3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17 # GNU Affero General Public License for more details.
18 #
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
21
22 import sys
23 import os
24 import shutil
25 import glob
26 import re
27 import socket
28 import zipfile
29 import hashlib
30 import pickle
31 from datetime import datetime, timedelta
32 from argparse import ArgumentParser
33
34 import collections
35 from binascii import hexlify
36
37 from PIL import Image
38 import logging
39
40 from . import common
41 from . import index
42 from . import metadata
43 from .common import SdkToolsPopen
44 from .exception import BuildException, FDroidException
45
46 METADATA_VERSION = 18
47
48 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
49 UNSET_VERSION_CODE = -0x100000000
50
51 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
52 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
53 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
54 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
55 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
56 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
57 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
58 APK_PERMISSION_PAT = \
59     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
60 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
61
62 screen_densities = ['640', '480', '320', '240', '160', '120']
63 screen_resolutions = {
64     "xxxhdpi": '640',
65     "xxhdpi": '480',
66     "xhdpi": '320',
67     "hdpi": '240',
68     "mdpi": '160',
69     "ldpi": '120',
70     "undefined": '-1',
71     "anydpi": '65534',
72     "nodpi": '65535'
73 }
74
75 all_screen_densities = ['0'] + screen_densities
76
77 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
78 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
79
80 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
81 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
82 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
83                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
84
85
86 def dpi_to_px(density):
87     return (int(density) * 48) / 160
88
89
90 def px_to_dpi(px):
91     return (int(px) * 160) / 48
92
93
94 def get_icon_dir(repodir, density):
95     if density == '0':
96         return os.path.join(repodir, "icons")
97     return os.path.join(repodir, "icons-%s" % density)
98
99
100 def get_icon_dirs(repodir):
101     for density in screen_densities:
102         yield get_icon_dir(repodir, density)
103
104
105 def get_all_icon_dirs(repodir):
106     for density in all_screen_densities:
107         yield get_icon_dir(repodir, density)
108
109
110 def update_wiki(apps, sortedids, apks):
111     """Update the wiki
112
113     :param apps: fully populated list of all applications
114     :param apks: all apks, except...
115     """
116     logging.info("Updating wiki")
117     wikicat = 'Apps'
118     wikiredircat = 'App Redirects'
119     import mwclient
120     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
121                          path=config['wiki_path'])
122     site.login(config['wiki_user'], config['wiki_password'])
123     generated_pages = {}
124     generated_redirects = {}
125
126     for appid in sortedids:
127         app = metadata.App(apps[appid])
128
129         wikidata = ''
130         if app.Disabled:
131             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
132         if app.AntiFeatures:
133             for af in app.AntiFeatures:
134                 wikidata += '{{AntiFeature|' + af + '}}\n'
135         if app.RequiresRoot:
136             requiresroot = 'Yes'
137         else:
138             requiresroot = 'No'
139         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
140             appid,
141             app.Name,
142             app.added.strftime('%Y-%m-%d') if app.added else '',
143             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
144             app.SourceCode,
145             app.IssueTracker,
146             app.WebSite,
147             app.Changelog,
148             app.Donate,
149             app.FlattrID,
150             app.Bitcoin,
151             app.Litecoin,
152             app.License,
153             requiresroot,
154             app.AuthorName,
155             app.AuthorEmail)
156
157         if app.Provides:
158             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
159
160         wikidata += app.Summary
161         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
162
163         wikidata += "=Description=\n"
164         wikidata += metadata.description_wiki(app.Description) + "\n"
165
166         wikidata += "=Maintainer Notes=\n"
167         if app.MaintainerNotes:
168             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
169         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
170
171         # Get a list of all packages for this application...
172         apklist = []
173         gotcurrentver = False
174         cantupdate = False
175         buildfails = False
176         for apk in apks:
177             if apk['packageName'] == appid:
178                 if str(apk['versionCode']) == app.CurrentVersionCode:
179                     gotcurrentver = True
180                 apklist.append(apk)
181         # Include ones we can't build, as a special case...
182         for build in app.builds:
183             if build.disable:
184                 if build.versionCode == app.CurrentVersionCode:
185                     cantupdate = True
186                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
187                 apklist.append({'versionCode': int(build.versionCode),
188                                 'versionName': build.versionName,
189                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
190                                 })
191             else:
192                 builtit = False
193                 for apk in apklist:
194                     if apk['versionCode'] == int(build.versionCode):
195                         builtit = True
196                         break
197                 if not builtit:
198                     buildfails = True
199                     apklist.append({'versionCode': int(build.versionCode),
200                                     'versionName': build.versionName,
201                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
202                                     })
203         if app.CurrentVersionCode == '0':
204             cantupdate = True
205         # Sort with most recent first...
206         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
207
208         wikidata += "=Versions=\n"
209         if len(apklist) == 0:
210             wikidata += "We currently have no versions of this app available."
211         elif not gotcurrentver:
212             wikidata += "We don't have the current version of this app."
213         else:
214             wikidata += "We have the current version of this app."
215         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
216         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
217         if len(app.NoSourceSince) > 0:
218             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
219         if len(app.CurrentVersion) > 0:
220             wikidata += "The current (recommended) version is " + app.CurrentVersion
221             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
222         validapks = 0
223         for apk in apklist:
224             wikidata += "==" + apk['versionName'] + "==\n"
225
226             if 'buildproblem' in apk:
227                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
228             else:
229                 validapks += 1
230                 wikidata += "This version is built and signed by "
231                 if 'srcname' in apk:
232                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
233                 else:
234                     wikidata += "the original developer.\n\n"
235             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
236
237         wikidata += '\n[[Category:' + wikicat + ']]\n'
238         if len(app.NoSourceSince) > 0:
239             wikidata += '\n[[Category:Apps missing source code]]\n'
240         if validapks == 0 and not app.Disabled:
241             wikidata += '\n[[Category:Apps with no packages]]\n'
242         if cantupdate and not app.Disabled:
243             wikidata += "\n[[Category:Apps we cannot update]]\n"
244         if buildfails and not app.Disabled:
245             wikidata += "\n[[Category:Apps with failing builds]]\n"
246         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
247             wikidata += '\n[[Category:Apps to Update]]\n'
248         if app.Disabled:
249             wikidata += '\n[[Category:Apps that are disabled]]\n'
250         if app.UpdateCheckMode == 'None' and not app.Disabled:
251             wikidata += '\n[[Category:Apps with no update check]]\n'
252         for appcat in app.Categories:
253             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
254
255         # We can't have underscores in the page name, even if they're in
256         # the package ID, because MediaWiki messes with them...
257         pagename = appid.replace('_', ' ')
258
259         # Drop a trailing newline, because mediawiki is going to drop it anyway
260         # and it we don't we'll think the page has changed when it hasn't...
261         if wikidata.endswith('\n'):
262             wikidata = wikidata[:-1]
263
264         generated_pages[pagename] = wikidata
265
266         # Make a redirect from the name to the ID too, unless there's
267         # already an existing page with the name and it isn't a redirect.
268         noclobber = False
269         apppagename = app.Name.replace('_', ' ')
270         apppagename = apppagename.replace('{', '')
271         apppagename = apppagename.replace('}', ' ')
272         apppagename = apppagename.replace(':', ' ')
273         apppagename = apppagename.replace('[', ' ')
274         apppagename = apppagename.replace(']', ' ')
275         # Drop double spaces caused mostly by replacing ':' above
276         apppagename = apppagename.replace('  ', ' ')
277         for expagename in site.allpages(prefix=apppagename,
278                                         filterredir='nonredirects',
279                                         generator=False):
280             if expagename == apppagename:
281                 noclobber = True
282         # Another reason not to make the redirect page is if the app name
283         # is the same as it's ID, because that will overwrite the real page
284         # with an redirect to itself! (Although it seems like an odd
285         # scenario this happens a lot, e.g. where there is metadata but no
286         # builds or binaries to extract a name from.
287         if apppagename == pagename:
288             noclobber = True
289         if not noclobber:
290             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
291
292     for tcat, genp in [(wikicat, generated_pages),
293                        (wikiredircat, generated_redirects)]:
294         catpages = site.Pages['Category:' + tcat]
295         existingpages = []
296         for page in catpages:
297             existingpages.append(page.name)
298             if page.name in genp:
299                 pagetxt = page.edit()
300                 if pagetxt != genp[page.name]:
301                     logging.debug("Updating modified page " + page.name)
302                     page.save(genp[page.name], summary='Auto-updated')
303                 else:
304                     logging.debug("Page " + page.name + " is unchanged")
305             else:
306                 logging.warn("Deleting page " + page.name)
307                 page.delete('No longer published')
308         for pagename, text in genp.items():
309             logging.debug("Checking " + pagename)
310             if pagename not in existingpages:
311                 logging.debug("Creating page " + pagename)
312                 try:
313                     newpage = site.Pages[pagename]
314                     newpage.save(text, summary='Auto-created')
315                 except Exception as e:
316                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
317
318     # Purge server cache to ensure counts are up to date
319     site.pages['Repository Maintenance'].purge()
320
321
322 def delete_disabled_builds(apps, apkcache, repodirs):
323     """Delete disabled build outputs.
324
325     :param apps: list of all applications, as per metadata.read_metadata
326     :param apkcache: current apk cache information
327     :param repodirs: the repo directories to process
328     """
329     for appid, app in apps.items():
330         for build in app['builds']:
331             if not build.disable:
332                 continue
333             apkfilename = common.get_release_filename(app, build)
334             iconfilename = "%s.%s.png" % (
335                 appid,
336                 build.versionCode)
337             for repodir in repodirs:
338                 files = [
339                     os.path.join(repodir, apkfilename),
340                     os.path.join(repodir, apkfilename + '.asc'),
341                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
342                 ]
343                 for density in all_screen_densities:
344                     repo_dir = get_icon_dir(repodir, density)
345                     files.append(os.path.join(repo_dir, iconfilename))
346
347                 for f in files:
348                     if os.path.exists(f):
349                         logging.info("Deleting disabled build output " + f)
350                         os.remove(f)
351             if apkfilename in apkcache:
352                 del apkcache[apkfilename]
353
354
355 def resize_icon(iconpath, density):
356
357     if not os.path.isfile(iconpath):
358         return
359
360     fp = None
361     try:
362         fp = open(iconpath, 'rb')
363         im = Image.open(fp)
364         size = dpi_to_px(density)
365
366         if any(length > size for length in im.size):
367             oldsize = im.size
368             im.thumbnail((size, size), Image.ANTIALIAS)
369             logging.debug("%s was too large at %s - new size is %s" % (
370                 iconpath, oldsize, im.size))
371             im.save(iconpath, "PNG")
372
373     except Exception as e:
374         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
375
376     finally:
377         if fp:
378             fp.close()
379
380
381 def resize_all_icons(repodirs):
382     """Resize all icons that exceed the max size
383
384     :param repodirs: the repo directories to process
385     """
386     for repodir in repodirs:
387         for density in screen_densities:
388             icon_dir = get_icon_dir(repodir, density)
389             icon_glob = os.path.join(icon_dir, '*.png')
390             for iconpath in glob.glob(icon_glob):
391                 resize_icon(iconpath, density)
392
393
394 def getsig(apkpath):
395     """ Get the signing certificate of an apk. To get the same md5 has that
396     Android gets, we encode the .RSA certificate in a specific format and pass
397     it hex-encoded to the md5 digest algorithm.
398
399     :param apkpath: path to the apk
400     :returns: A string containing the md5 of the signature of the apk or None
401               if an error occurred.
402     """
403
404     with zipfile.ZipFile(apkpath, 'r') as apk:
405         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
406
407         if len(certs) < 1:
408             logging.error("Found no signing certificates on %s" % apkpath)
409             return None
410         if len(certs) > 1:
411             logging.error("Found multiple signing certificates on %s" % apkpath)
412             return None
413
414         cert = apk.read(certs[0])
415
416     cert_encoded = common.get_certificate(cert)
417
418     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
419
420
421 def get_cache_file():
422     return os.path.join('tmp', 'apkcache')
423
424
425 def get_cache():
426     """Get the cached dict of the APK index
427
428     Gather information about all the apk files in the repo directory,
429     using cached data if possible. Some of the index operations take a
430     long time, like calculating the SHA-256 and verifying the APK
431     signature.
432
433     The cache is invalidated if the metadata version is different, or
434     the 'allow_disabled_algorithms' config/option is different.  In
435     those cases, there is no easy way to know what has changed from
436     the cache, so just rerun the whole thing.
437
438     :return: apkcache
439
440     """
441     apkcachefile = get_cache_file()
442     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
443     if not options.clean and os.path.exists(apkcachefile):
444         with open(apkcachefile, 'rb') as cf:
445             apkcache = pickle.load(cf, encoding='utf-8')
446         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
447            or apkcache.get('allow_disabled_algorithms') != ada:
448             apkcache = {}
449     else:
450         apkcache = {}
451
452     apkcache["METADATA_VERSION"] = METADATA_VERSION
453     apkcache['allow_disabled_algorithms'] = ada
454
455     return apkcache
456
457
458 def write_cache(apkcache):
459     apkcachefile = get_cache_file()
460     cache_path = os.path.dirname(apkcachefile)
461     if not os.path.exists(cache_path):
462         os.makedirs(cache_path)
463     with open(apkcachefile, 'wb') as cf:
464         pickle.dump(apkcache, cf)
465
466
467 def get_icon_bytes(apkzip, iconsrc):
468     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
469     try:
470         return apkzip.read(iconsrc)
471     except KeyError:
472         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
473
474
475 def sha256sum(filename):
476     '''Calculate the sha256 of the given file'''
477     sha = hashlib.sha256()
478     with open(filename, 'rb') as f:
479         while True:
480             t = f.read(16384)
481             if len(t) == 0:
482                 break
483             sha.update(t)
484     return sha.hexdigest()
485
486
487 def has_known_vulnerability(filename):
488     """checks for known vulnerabilities in the APK
489
490     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
491     version.  Google also enforces this:
492     https://support.google.com/faqs/answer/6376725?hl=en
493
494     Checks whether there are more than one classes.dex or AndroidManifest.xml
495     files, which is invalid and an essential part of the "Master Key" attack.
496
497     http://www.saurik.com/id/17
498     """
499
500     # statically load this pattern
501     if not hasattr(has_known_vulnerability, "pattern"):
502         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
503
504     files_in_apk = set()
505     with zipfile.ZipFile(filename) as zf:
506         for name in zf.namelist():
507             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
508                 lib = zf.open(name)
509                 while True:
510                     chunk = lib.read(4096)
511                     if chunk == b'':
512                         break
513                     m = has_known_vulnerability.pattern.search(chunk)
514                     if m:
515                         version = m.group(1).decode('ascii')
516                         if version.startswith('1.0.1') and version[5] >= 'r' \
517                            or version.startswith('1.0.2') and version[5] >= 'f':
518                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
519                         else:
520                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
521                             return True
522                         break
523             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
524                 if name in files_in_apk:
525                     return True
526                 files_in_apk.add(name)
527
528     return False
529
530
531 def insert_obbs(repodir, apps, apks):
532     """Scans the .obb files in a given repo directory and adds them to the
533     relevant APK instances.  OBB files have versionCodes like APK
534     files, and they are loosely associated.  If there is an OBB file
535     present, then any APK with the same or higher versionCode will use
536     that OBB file.  There are two OBB types: main and patch, each APK
537     can only have only have one of each.
538
539     https://developer.android.com/google/play/expansion-files.html
540
541     :param repodir: repo directory to scan
542     :param apps: list of current, valid apps
543     :param apks: current information on all APKs
544
545     """
546
547     def obbWarnDelete(f, msg):
548         logging.warning(msg + f)
549         if options.delete_unknown:
550             logging.error("Deleting unknown file: " + f)
551             os.remove(f)
552
553     obbs = []
554     java_Integer_MIN_VALUE = -pow(2, 31)
555     currentPackageNames = apps.keys()
556     for f in glob.glob(os.path.join(repodir, '*.obb')):
557         obbfile = os.path.basename(f)
558         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
559         chunks = obbfile.split('.')
560         if chunks[0] != 'main' and chunks[0] != 'patch':
561             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
562             continue
563         if not re.match(r'^-?[0-9]+$', chunks[1]):
564             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
565             continue
566         versionCode = int(chunks[1])
567         packagename = ".".join(chunks[2:-1])
568
569         highestVersionCode = java_Integer_MIN_VALUE
570         if packagename not in currentPackageNames:
571             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
572             continue
573         for apk in apks:
574             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
575                 highestVersionCode = apk['versionCode']
576         if versionCode > highestVersionCode:
577             obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
578                           + ') than any APK: ')
579             continue
580         obbsha256 = sha256sum(f)
581         obbs.append((packagename, versionCode, obbfile, obbsha256))
582
583     for apk in apks:
584         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
585             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
586                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
587                     apk['obbMainFile'] = obbfile
588                     apk['obbMainFileSha256'] = obbsha256
589                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
590                     apk['obbPatchFile'] = obbfile
591                     apk['obbPatchFileSha256'] = obbsha256
592             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
593                 break
594
595
596 def _get_localized_dict(app, locale):
597     '''get the dict to add localized store metadata to'''
598     if 'localized' not in app:
599         app['localized'] = collections.OrderedDict()
600     if locale not in app['localized']:
601         app['localized'][locale] = collections.OrderedDict()
602     return app['localized'][locale]
603
604
605 def _set_localized_text_entry(app, locale, key, f):
606     limit = config['char_limits'][key]
607     localized = _get_localized_dict(app, locale)
608     with open(f) as fp:
609         text = fp.read()[:limit]
610         if len(text) > 0:
611             localized[key] = text
612
613
614 def _set_author_entry(app, key, f):
615     limit = config['char_limits']['author']
616     with open(f) as fp:
617         text = fp.read()[:limit]
618         if len(text) > 0:
619             app[key] = text
620
621
622 def copy_triple_t_store_metadata(apps):
623     """Include store metadata from the app's source repo
624
625     The Triple-T Gradle Play Publisher is a plugin that has a standard
626     file layout for all of the metadata and graphics that the Google
627     Play Store accepts.  Since F-Droid has the git repo, it can just
628     pluck those files directly.  This method reads any text files into
629     the app dict, then copies any graphics into the fdroid repo
630     directory structure.
631
632     This needs to be run before insert_localized_app_metadata() so that
633     the graphics files that are copied into the fdroid repo get
634     properly indexed.
635
636     https://github.com/Triple-T/gradle-play-publisher#upload-images
637     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
638
639     """
640
641     if not os.path.isdir('build'):
642         return  # nothing to do
643
644     for packageName, app in apps.items():
645         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
646             logging.debug('Triple-T Gradle Play Publisher: ' + d)
647             for root, dirs, files in os.walk(d):
648                 segments = root.split('/')
649                 locale = segments[-2]
650                 for f in files:
651                     if f == 'fulldescription':
652                         _set_localized_text_entry(app, locale, 'description',
653                                                   os.path.join(root, f))
654                         continue
655                     elif f == 'shortdescription':
656                         _set_localized_text_entry(app, locale, 'summary',
657                                                   os.path.join(root, f))
658                         continue
659                     elif f == 'title':
660                         _set_localized_text_entry(app, locale, 'name',
661                                                   os.path.join(root, f))
662                         continue
663                     elif f == 'video':
664                         _set_localized_text_entry(app, locale, 'video',
665                                                   os.path.join(root, f))
666                         continue
667                     elif f == 'whatsnew':
668                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
669                                                   os.path.join(root, f))
670                         continue
671                     elif f == 'contactEmail':
672                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
673                         continue
674                     elif f == 'contactPhone':
675                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
676                         continue
677                     elif f == 'contactWebsite':
678                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
679                         continue
680
681                     base, extension = common.get_extension(f)
682                     dirname = os.path.basename(root)
683                     if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
684                         if segments[-2] == 'listing':
685                             locale = segments[-3]
686                         else:
687                             locale = segments[-2]
688                         destdir = os.path.join('repo', packageName, locale)
689                         os.makedirs(destdir, mode=0o755, exist_ok=True)
690                         sourcefile = os.path.join(root, f)
691                         destfile = os.path.join(destdir, dirname + '.' + extension)
692                         logging.debug('copying ' + sourcefile + ' ' + destfile)
693                         shutil.copy(sourcefile, destfile)
694
695
696 def insert_localized_app_metadata(apps):
697     """scans standard locations for graphics and localized text
698
699     Scans for localized description files, store graphics, and
700     screenshot PNG files in statically defined screenshots directory
701     and adds them to the app metadata.  The screenshots and graphic
702     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
703     and must be in the following layout:
704     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
705
706     repo/packageName/locale/featureGraphic.png
707     repo/packageName/locale/phoneScreenshots/1.png
708     repo/packageName/locale/phoneScreenshots/2.png
709
710     The changelog files must be text files named with the versionCode
711     ending with ".txt" and must be in the following layout:
712     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
713
714     repo/packageName/locale/changelogs/12345.txt
715
716     This will scan the each app's source repo then the metadata/ dir
717     for these standard locations of changelog files.  If it finds
718     them, they will be added to the dict of all packages, with the
719     versions in the metadata/ folder taking precendence over the what
720     is in the app's source repo.
721
722     Where "packageName" is the app's packageName and "locale" is the locale
723     of the graphics, e.g. what language they are in, using the IETF RFC5646
724     format (en-US, fr-CA, es-MX, etc).
725
726     This will also scan the app's git for a fastlane folder, and the
727     metadata/ folder and the apps' source repos for standard locations
728     of graphic and screenshot files.  If it finds them, it will copy
729     them into the repo.  The fastlane files follow this pattern:
730     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
731
732     """
733
734     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
735     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
736     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
737
738     for srcd in sorted(sourcedirs):
739         if not os.path.isdir(srcd):
740             continue
741         for root, dirs, files in os.walk(srcd):
742             segments = root.split('/')
743             packageName = segments[1]
744             if packageName not in apps:
745                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
746                 continue
747             locale = segments[-1]
748             destdir = os.path.join('repo', packageName, locale)
749             for f in files:
750                 if f in ('description.txt', 'full_description.txt'):
751                     _set_localized_text_entry(apps[packageName], locale, 'description',
752                                               os.path.join(root, f))
753                     continue
754                 elif f in ('summary.txt', 'short_description.txt'):
755                     _set_localized_text_entry(apps[packageName], locale, 'summary',
756                                               os.path.join(root, f))
757                     continue
758                 elif f in ('name.txt', 'title.txt'):
759                     _set_localized_text_entry(apps[packageName], locale, 'name',
760                                               os.path.join(root, f))
761                     continue
762                 elif f == 'video.txt':
763                     _set_localized_text_entry(apps[packageName], locale, 'video',
764                                               os.path.join(root, f))
765                     continue
766                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
767                     locale = segments[-2]
768                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
769                                               os.path.join(root, f))
770                     continue
771
772                 base, extension = common.get_extension(f)
773                 if locale == 'images':
774                     locale = segments[-2]
775                     destdir = os.path.join('repo', packageName, locale)
776                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
777                     os.makedirs(destdir, mode=0o755, exist_ok=True)
778                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
779                     shutil.copy(os.path.join(root, f), destdir)
780             for d in dirs:
781                 if d in SCREENSHOT_DIRS:
782                     for f in glob.glob(os.path.join(root, d, '*.*')):
783                         _, extension = common.get_extension(f)
784                         if extension in ALLOWED_EXTENSIONS:
785                             screenshotdestdir = os.path.join(destdir, d)
786                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
787                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
788                             shutil.copy(f, screenshotdestdir)
789
790     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
791     for d in repofiles:
792         if not os.path.isdir(d):
793             continue
794         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
795             if not os.path.isfile(f):
796                 continue
797             segments = f.split('/')
798             packageName = segments[1]
799             locale = segments[2]
800             screenshotdir = segments[3]
801             filename = os.path.basename(f)
802             base, extension = common.get_extension(filename)
803
804             if packageName not in apps:
805                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
806                                 % (filename, packageName))
807                 continue
808             graphics = _get_localized_dict(apps[packageName], locale)
809
810             if extension not in ALLOWED_EXTENSIONS:
811                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
812             elif base in GRAPHIC_NAMES:
813                 # there can only be zero or one of these per locale
814                 graphics[base] = filename
815             elif screenshotdir in SCREENSHOT_DIRS:
816                 # there can any number of these per locale
817                 logging.debug('adding to ' + screenshotdir + ': ' + f)
818                 if screenshotdir not in graphics:
819                     graphics[screenshotdir] = []
820                 graphics[screenshotdir].append(filename)
821             else:
822                 logging.warning('Unsupported graphics file found: ' + f)
823
824
825 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
826     """Scan a repo for all files with an extension except APK/OBB
827
828     :param apkcache: current cached info about all repo files
829     :param repodir: repo directory to scan
830     :param knownapks: list of all known files, as per metadata.read_metadata
831     :param use_date_from_file: use date from file (instead of current date)
832                                for newly added files
833     """
834
835     cachechanged = False
836     repo_files = []
837     repodir = repodir.encode('utf-8')
838     for name in os.listdir(repodir):
839         file_extension = common.get_file_extension(name)
840         if file_extension == 'apk' or file_extension == 'obb':
841             continue
842         filename = os.path.join(repodir, name)
843         name_utf8 = name.decode('utf-8')
844         if filename.endswith(b'_src.tar.gz'):
845             logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
846             continue
847         if not common.is_repo_file(filename):
848             continue
849         stat = os.stat(filename)
850         if stat.st_size == 0:
851             raise FDroidException(filename + ' is zero size!')
852
853         shasum = sha256sum(filename)
854         usecache = False
855         if name in apkcache:
856             repo_file = apkcache[name]
857             # added time is cached as tuple but used here as datetime instance
858             if 'added' in repo_file:
859                 a = repo_file['added']
860                 if isinstance(a, datetime):
861                     repo_file['added'] = a
862                 else:
863                     repo_file['added'] = datetime(*a[:6])
864             if repo_file.get('hash') == shasum:
865                 logging.debug("Reading " + name_utf8 + " from cache")
866                 usecache = True
867             else:
868                 logging.debug("Ignoring stale cache data for " + name)
869
870         if not usecache:
871             logging.debug("Processing " + name_utf8)
872             repo_file = collections.OrderedDict()
873             repo_file['name'] = os.path.splitext(name_utf8)[0]
874             # TODO rename apkname globally to something more generic
875             repo_file['apkName'] = name_utf8
876             repo_file['hash'] = shasum
877             repo_file['hashType'] = 'sha256'
878             repo_file['versionCode'] = 0
879             repo_file['versionName'] = shasum
880             # the static ID is the SHA256 unless it is set in the metadata
881             repo_file['packageName'] = shasum
882
883             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
884             if m:
885                 repo_file['packageName'] = m.group(1)
886                 repo_file['versionCode'] = int(m.group(2))
887             srcfilename = name + b'_src.tar.gz'
888             if os.path.exists(os.path.join(repodir, srcfilename)):
889                 repo_file['srcname'] = srcfilename.decode('utf-8')
890             repo_file['size'] = stat.st_size
891
892             apkcache[name] = repo_file
893             cachechanged = True
894
895         if use_date_from_file:
896             timestamp = stat.st_ctime
897             default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
898         else:
899             default_date_param = None
900
901         # Record in knownapks, getting the added date at the same time..
902         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
903                                     default_date=default_date_param)
904         if added:
905             repo_file['added'] = added
906
907         repo_files.append(repo_file)
908
909     return repo_files, cachechanged
910
911
912 def scan_apk(apk_file):
913     """
914     Scans an APK file and returns dictionary with metadata of the APK.
915
916     Attention: This does *not* verify that the APK signature is correct.
917
918     :param apk_file: The (ideally absolute) path to the APK file
919     :raises BuildException
920     :return A dict containing APK metadata
921     """
922     apk = {
923         'hash': sha256sum(apk_file),
924         'hashType': 'sha256',
925         'uses-permission': [],
926         'uses-permission-sdk-23': [],
927         'features': [],
928         'icons_src': {},
929         'icons': {},
930         'antiFeatures': set(),
931     }
932
933     if SdkToolsPopen(['aapt', 'version'], output=False):
934         scan_apk_aapt(apk, apk_file)
935     else:
936         scan_apk_androguard(apk, apk_file)
937
938     # Get the signature
939     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
940     apk['sig'] = getsig(apk_file)
941     if not apk['sig']:
942         raise BuildException("Failed to get apk signature")
943
944     # Get size of the APK
945     apk['size'] = os.path.getsize(apk_file)
946
947     if 'minSdkVersion' not in apk:
948         logging.warning("No SDK version information found in {0}".format(apk_file))
949         apk['minSdkVersion'] = 1
950
951     # Check for known vulnerabilities
952     if has_known_vulnerability(apk_file):
953         apk['antiFeatures'].add('KnownVuln')
954
955     return apk
956
957
958 def scan_apk_aapt(apk, apkfile):
959     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
960     if p.returncode != 0:
961         if options.delete_unknown:
962             if os.path.exists(apkfile):
963                 logging.error("Failed to get apk information, deleting " + apkfile)
964                 os.remove(apkfile)
965             else:
966                 logging.error("Could not find {0} to remove it".format(apkfile))
967         else:
968             logging.error("Failed to get apk information, skipping " + apkfile)
969         raise BuildException("Invalid APK")
970     for line in p.output.splitlines():
971         if line.startswith("package:"):
972             try:
973                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
974                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
975                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
976             except Exception as e:
977                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
978         elif line.startswith("application:"):
979             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
980             # Keep path to non-dpi icon in case we need it
981             match = re.match(APK_ICON_PAT_NODPI, line)
982             if match:
983                 apk['icons_src']['-1'] = match.group(1)
984         elif line.startswith("launchable-activity:"):
985             # Only use launchable-activity as fallback to application
986             if not apk['name']:
987                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
988             if '-1' not in apk['icons_src']:
989                 match = re.match(APK_ICON_PAT_NODPI, line)
990                 if match:
991                     apk['icons_src']['-1'] = match.group(1)
992         elif line.startswith("application-icon-"):
993             match = re.match(APK_ICON_PAT, line)
994             if match:
995                 density = match.group(1)
996                 path = match.group(2)
997                 apk['icons_src'][density] = path
998         elif line.startswith("sdkVersion:"):
999             m = re.match(APK_SDK_VERSION_PAT, line)
1000             if m is None:
1001                 logging.error(line.replace('sdkVersion:', '')
1002                               + ' is not a valid minSdkVersion!')
1003             else:
1004                 apk['minSdkVersion'] = m.group(1)
1005                 # if target not set, default to min
1006                 if 'targetSdkVersion' not in apk:
1007                     apk['targetSdkVersion'] = m.group(1)
1008         elif line.startswith("targetSdkVersion:"):
1009             m = re.match(APK_SDK_VERSION_PAT, line)
1010             if m is None:
1011                 logging.error(line.replace('targetSdkVersion:', '')
1012                               + ' is not a valid targetSdkVersion!')
1013             else:
1014                 apk['targetSdkVersion'] = m.group(1)
1015         elif line.startswith("maxSdkVersion:"):
1016             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1017         elif line.startswith("native-code:"):
1018             apk['nativecode'] = []
1019             for arch in line[13:].split(' '):
1020                 apk['nativecode'].append(arch[1:-1])
1021         elif line.startswith('uses-permission:'):
1022             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1023             if perm_match['maxSdkVersion']:
1024                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1025             permission = UsesPermission(
1026                 perm_match['name'],
1027                 perm_match['maxSdkVersion']
1028             )
1029
1030             apk['uses-permission'].append(permission)
1031         elif line.startswith('uses-permission-sdk-23:'):
1032             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1033             if perm_match['maxSdkVersion']:
1034                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1035             permission_sdk_23 = UsesPermissionSdk23(
1036                 perm_match['name'],
1037                 perm_match['maxSdkVersion']
1038             )
1039
1040             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1041
1042         elif line.startswith('uses-feature:'):
1043             feature = re.match(APK_FEATURE_PAT, line).group(1)
1044             # Filter out this, it's only added with the latest SDK tools and
1045             # causes problems for lots of apps.
1046             if feature != "android.hardware.screen.portrait" \
1047                     and feature != "android.hardware.screen.landscape":
1048                 if feature.startswith("android.feature."):
1049                     feature = feature[16:]
1050                 apk['features'].add(feature)
1051
1052
1053 def scan_apk_androguard(apk, apkfile):
1054     try:
1055         from androguard.core.bytecodes.apk import APK
1056         apkobject = APK(apkfile)
1057         if apkobject.is_valid_APK():
1058             arsc = apkobject.get_android_resources()
1059         else:
1060             if options.delete_unknown:
1061                 if os.path.exists(apkfile):
1062                     logging.error("Failed to get apk information, deleting " + apkfile)
1063                     os.remove(apkfile)
1064                 else:
1065                     logging.error("Could not find {0} to remove it".format(apkfile))
1066             else:
1067                 logging.error("Failed to get apk information, skipping " + apkfile)
1068             raise BuildException("Invaild APK")
1069     except ImportError:
1070         raise FDroidException("androguard library is not installed and aapt not present")
1071     except FileNotFoundError:
1072         logging.error("Could not open apk file for analysis")
1073         raise BuildException("Invalid APK")
1074
1075     apk['packageName'] = apkobject.get_package()
1076     apk['versionCode'] = int(apkobject.get_androidversion_code())
1077     apk['versionName'] = apkobject.get_androidversion_name()
1078     if apk['versionName'][0] == "@":
1079         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1080         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1081         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1082     apk['name'] = apkobject.get_app_name()
1083
1084     if apkobject.get_max_sdk_version() is not None:
1085         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1086     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1087     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1088
1089     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1090     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1091
1092     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1093
1094     for file in apkobject.get_files():
1095         d_re = density_re.match(file)
1096         if d_re:
1097             folder = d_re.group(1).split('-')
1098             if len(folder) > 1:
1099                 resolution = folder[1]
1100             else:
1101                 resolution = 'mdpi'
1102             density = screen_resolutions[resolution]
1103             apk['icons_src'][density] = d_re.group(0)
1104
1105     if apk['icons_src'].get('-1') is None:
1106         apk['icons_src']['-1'] = apk['icons_src']['160']
1107
1108     arch_re = re.compile("^lib/(.*)/.*$")
1109     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1110     if len(arch) >= 1:
1111         apk['nativecode'] = []
1112         apk['nativecode'].extend(sorted(list(arch)))
1113
1114     xml = apkobject.get_android_manifest_xml()
1115
1116     for item in xml.getElementsByTagName('uses-permission'):
1117         name = str(item.getAttribute("android:name"))
1118         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1119         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1120         permission = UsesPermission(
1121             name,
1122             maxSdkVersion
1123         )
1124         apk['uses-permission'].append(permission)
1125
1126     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1127         name = str(item.getAttribute("android:name"))
1128         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1129         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1130         permission_sdk_23 = UsesPermissionSdk23(
1131             name,
1132             maxSdkVersion
1133         )
1134         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1135
1136     for item in xml.getElementsByTagName('uses-feature'):
1137         feature = str(item.getAttribute("android:name"))
1138         if feature != "android.hardware.screen.portrait" \
1139                 and feature != "android.hardware.screen.landscape":
1140             if feature.startswith("android.feature."):
1141                 feature = feature[16:]
1142         apk['features'].append(feature)
1143
1144
1145 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1146                 allow_disabled_algorithms=False, archive_bad_sig=False):
1147     """Processes the apk with the given filename in the given repo directory.
1148
1149     This also extracts the icons.
1150
1151     :param apkcache: current apk cache information
1152     :param apkfilename: the filename of the apk to scan
1153     :param repodir: repo directory to scan
1154     :param knownapks: known apks info
1155     :param use_date_from_apk: use date from APK (instead of current date)
1156                               for newly added APKs
1157     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1158                                       disabled algorithms in the signature (e.g. MD5)
1159     :param archive_bad_sig: move APKs with a bad signature to the archive
1160     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1161      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1162     """
1163
1164     if ' ' in apkfilename:
1165         if options.rename_apks:
1166             newfilename = apkfilename.replace(' ', '_')
1167             os.rename(os.path.join(repodir, apkfilename),
1168                       os.path.join(repodir, newfilename))
1169             apkfilename = newfilename
1170         else:
1171             logging.critical("Spaces in filenames are not allowed.")
1172             return True, None, False
1173
1174     apk = {}
1175     apkfile = os.path.join(repodir, apkfilename)
1176
1177     cachechanged = False
1178     usecache = False
1179     if apkfilename in apkcache:
1180         apk = apkcache[apkfilename]
1181         if apk.get('hash') == sha256sum(apkfile):
1182             logging.debug("Reading " + apkfilename + " from cache")
1183             usecache = True
1184         else:
1185             logging.debug("Ignoring stale cache data for " + apkfilename)
1186
1187     if not usecache:
1188         logging.debug("Processing " + apkfilename)
1189
1190         try:
1191             apk = scan_apk(apkfile)
1192         except BuildException:
1193             logging.warning('Skipping "%s" with invalid signature!', apkfilename)
1194             return True, None, False
1195
1196         # Check for debuggable apks...
1197         if common.isApkAndDebuggable(apkfile):
1198             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1199
1200         if options.rename_apks:
1201             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1202             std_short_name = os.path.join(repodir, n)
1203             if apkfile != std_short_name:
1204                 if os.path.exists(std_short_name):
1205                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1206                     if apkfile != std_long_name:
1207                         if os.path.exists(std_long_name):
1208                             dupdir = os.path.join('duplicates', repodir)
1209                             if not os.path.isdir(dupdir):
1210                                 os.makedirs(dupdir, exist_ok=True)
1211                             dupfile = os.path.join('duplicates', std_long_name)
1212                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1213                             os.rename(apkfile, dupfile)
1214                             return True, None, False
1215                         else:
1216                             os.rename(apkfile, std_long_name)
1217                     apkfile = std_long_name
1218                 else:
1219                     os.rename(apkfile, std_short_name)
1220                     apkfile = std_short_name
1221                 apkfilename = apkfile[len(repodir) + 1:]
1222
1223         apk['apkName'] = apkfilename
1224         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1225         if os.path.exists(os.path.join(repodir, srcfilename)):
1226             apk['srcname'] = srcfilename
1227
1228         # verify the jar signature is correct, allow deprecated
1229         # algorithms only if the APK is in the archive.
1230         skipapk = False
1231         if not common.verify_apk_signature(apkfile):
1232             if repodir == 'archive' or allow_disabled_algorithms:
1233                 if common.verify_old_apk_signature(apkfile):
1234                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1235                 else:
1236                     skipapk = True
1237             else:
1238                 skipapk = True
1239
1240         if skipapk:
1241             if archive_bad_sig:
1242                 logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
1243                 move_apk_between_sections(repodir, 'archive', apk)
1244             else:
1245                 logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
1246             return True, None, False
1247
1248         apkzip = zipfile.ZipFile(apkfile, 'r')
1249
1250         # if an APK has files newer than the system time, suggest updating
1251         # the system clock.  This is useful for offline systems, used for
1252         # signing, which do not have another source of clock sync info. It
1253         # has to be more than 24 hours newer because ZIP/APK files do not
1254         # store timezone info
1255         manifest = apkzip.getinfo('AndroidManifest.xml')
1256         if manifest.date_time[1] == 0:  # month can't be zero
1257             logging.debug('AndroidManifest.xml has no date')
1258         else:
1259             dt_obj = datetime(*manifest.date_time)
1260             checkdt = dt_obj - timedelta(1)
1261             if datetime.today() < checkdt:
1262                 logging.warning('System clock is older than manifest in: '
1263                                 + apkfilename
1264                                 + '\nSet clock to that time using:\n'
1265                                 + 'sudo date -s "' + str(dt_obj) + '"')
1266
1267         # extract icons from APK zip file
1268         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1269         try:
1270             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1271         finally:
1272             apkzip.close()  # ensure that APK zip file gets closed
1273
1274         # resize existing icons for densities missing in the APK
1275         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1276
1277         if use_date_from_apk and manifest.date_time[1] != 0:
1278             default_date_param = datetime(*manifest.date_time)
1279         else:
1280             default_date_param = None
1281
1282         # Record in known apks, getting the added date at the same time..
1283         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1284                                     default_date=default_date_param)
1285         if added:
1286             apk['added'] = added
1287
1288         apkcache[apkfilename] = apk
1289         cachechanged = True
1290
1291     return False, apk, cachechanged
1292
1293
1294 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1295     """Processes the apks in the given repo directory.
1296
1297     This also extracts the icons.
1298
1299     :param apkcache: current apk cache information
1300     :param repodir: repo directory to scan
1301     :param knownapks: known apks info
1302     :param use_date_from_apk: use date from APK (instead of current date)
1303                               for newly added APKs
1304     :returns: (apks, cachechanged) where apks is a list of apk information,
1305               and cachechanged is True if the apkcache got changed.
1306     """
1307
1308     cachechanged = False
1309
1310     for icon_dir in get_all_icon_dirs(repodir):
1311         if os.path.exists(icon_dir):
1312             if options.clean:
1313                 shutil.rmtree(icon_dir)
1314                 os.makedirs(icon_dir)
1315         else:
1316             os.makedirs(icon_dir)
1317
1318     apks = []
1319     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1320         apkfilename = apkfile[len(repodir) + 1:]
1321         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1322         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1323                                              use_date_from_apk, ada, True)
1324         if skip:
1325             continue
1326         apks.append(apk)
1327         cachechanged = cachechanged or cachethis
1328
1329     return apks, cachechanged
1330
1331
1332 def extract_apk_icons(icon_filename, apk, apk_zip, repo_dir):
1333     """
1334     Extracts icons from the given APK zip in various densities,
1335     saves them into given repo directory
1336     and stores their names in the APK metadata dictionary.
1337
1338     :param icon_filename: A string representing the icon's file name
1339     :param apk: A populated dictionary containing APK metadata.
1340                 Needs to have 'icons_src' key
1341     :param apk_zip: An opened zipfile.ZipFile of the APK file
1342     :param repo_dir: The directory of the APK's repository
1343     :return: A list of icon densities that are missing
1344     """
1345     empty_densities = []
1346     for density in screen_densities:
1347         if density not in apk['icons_src']:
1348             empty_densities.append(density)
1349             continue
1350         icon_src = apk['icons_src'][density]
1351         icon_dir = get_icon_dir(repo_dir, density)
1352         icon_dest = os.path.join(icon_dir, icon_filename)
1353
1354         # Extract the icon files per density
1355         try:
1356             with open(icon_dest, 'wb') as f:
1357                 f.write(get_icon_bytes(apk_zip, icon_src))
1358             apk['icons'][density] = icon_filename
1359         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1360             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1361             del apk['icons_src'][density]
1362             empty_densities.append(density)
1363
1364     if '-1' in apk['icons_src']:
1365         icon_src = apk['icons_src']['-1']
1366         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1367         with open(icon_path, 'wb') as f:
1368             f.write(get_icon_bytes(apk_zip, icon_src))
1369         try:
1370             im = Image.open(icon_path)
1371             dpi = px_to_dpi(im.size[0])
1372             for density in screen_densities:
1373                 if density in apk['icons']:
1374                     break
1375                 if density == screen_densities[-1] or dpi >= int(density):
1376                     apk['icons'][density] = icon_filename
1377                     shutil.move(icon_path,
1378                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1379                     empty_densities.remove(density)
1380                     break
1381         except Exception as e:
1382             logging.warning("Failed reading {0} - {1}".format(icon_path, e))
1383
1384     if apk['icons']:
1385         apk['icon'] = icon_filename
1386
1387     return empty_densities
1388
1389
1390 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1391     """
1392     Resize existing icons for densities missing in the APK to ensure all densities are available
1393
1394     :param empty_densities: A list of icon densities that are missing
1395     :param icon_filename: A string representing the icon's file name
1396     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1397     :param repo_dir: The directory of the APK's repository
1398     """
1399     # First try resizing down to not lose quality
1400     last_density = None
1401     for density in screen_densities:
1402         if density not in empty_densities:
1403             last_density = density
1404             continue
1405         if last_density is None:
1406             continue
1407         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1408
1409         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1410         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1411         fp = None
1412         try:
1413             fp = open(last_icon_path, 'rb')
1414             im = Image.open(fp)
1415
1416             size = dpi_to_px(density)
1417
1418             im.thumbnail((size, size), Image.ANTIALIAS)
1419             im.save(icon_path, "PNG")
1420             empty_densities.remove(density)
1421         except Exception as e:
1422             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1423         finally:
1424             if fp:
1425                 fp.close()
1426
1427     # Then just copy from the highest resolution available
1428     last_density = None
1429     for density in reversed(screen_densities):
1430         if density not in empty_densities:
1431             last_density = density
1432             continue
1433
1434         if last_density is None:
1435             continue
1436
1437         shutil.copyfile(
1438             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1439             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1440         )
1441         empty_densities.remove(density)
1442
1443     for density in screen_densities:
1444         icon_dir = get_icon_dir(repo_dir, density)
1445         icon_dest = os.path.join(icon_dir, icon_filename)
1446         resize_icon(icon_dest, density)
1447
1448     # Copy from icons-mdpi to icons since mdpi is the baseline density
1449     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1450     if os.path.isfile(baseline):
1451         apk['icons']['0'] = icon_filename
1452         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1453
1454
1455 def apply_info_from_latest_apk(apps, apks):
1456     """
1457     Some information from the apks needs to be applied up to the application level.
1458     When doing this, we use the info from the most recent version's apk.
1459     We deal with figuring out when the app was added and last updated at the same time.
1460     """
1461     for appid, app in apps.items():
1462         bestver = UNSET_VERSION_CODE
1463         for apk in apks:
1464             if apk['packageName'] == appid:
1465                 if apk['versionCode'] > bestver:
1466                     bestver = apk['versionCode']
1467                     bestapk = apk
1468
1469                 if 'added' in apk:
1470                     if not app.added or apk['added'] < app.added:
1471                         app.added = apk['added']
1472                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1473                         app.lastUpdated = apk['added']
1474
1475         if not app.added:
1476             logging.debug("Don't know when " + appid + " was added")
1477         if not app.lastUpdated:
1478             logging.debug("Don't know when " + appid + " was last updated")
1479
1480         if bestver == UNSET_VERSION_CODE:
1481
1482             if app.Name is None:
1483                 app.Name = app.AutoName or appid
1484             app.icon = None
1485             logging.debug("Application " + appid + " has no packages")
1486         else:
1487             if app.Name is None:
1488                 app.Name = bestapk['name']
1489             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1490             if app.CurrentVersionCode is None:
1491                 app.CurrentVersionCode = str(bestver)
1492
1493
1494 def make_categories_txt(repodir, categories):
1495     '''Write a category list in the repo to allow quick access'''
1496     catdata = ''
1497     for cat in sorted(categories):
1498         catdata += cat + '\n'
1499     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1500         f.write(catdata)
1501
1502
1503 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1504
1505     def filter_apk_list_sorted(apk_list):
1506         res = []
1507         for apk in apk_list:
1508             if apk['packageName'] == appid:
1509                 res.append(apk)
1510
1511         # Sort the apk list by version code. First is highest/newest.
1512         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1513
1514     for appid, app in apps.items():
1515
1516         if app.ArchivePolicy:
1517             keepversions = int(app.ArchivePolicy[:-9])
1518         else:
1519             keepversions = defaultkeepversions
1520
1521         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1522                       .format(appid, len(apks), keepversions, len(archapks)))
1523
1524         current_app_apks = filter_apk_list_sorted(apks)
1525         if len(current_app_apks) > keepversions:
1526             # Move back the ones we don't want.
1527             for apk in current_app_apks[keepversions:]:
1528                 move_apk_between_sections(repodir, archivedir, apk)
1529                 archapks.append(apk)
1530                 apks.remove(apk)
1531
1532         current_app_archapks = filter_apk_list_sorted(archapks)
1533         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1534             kept = 0
1535             # Move forward the ones we want again, except DisableAlgorithm
1536             for apk in current_app_archapks:
1537                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1538                     move_apk_between_sections(archivedir, repodir, apk)
1539                     archapks.remove(apk)
1540                     apks.append(apk)
1541                     kept += 1
1542                 if kept == keepversions:
1543                     break
1544
1545
1546 def move_apk_between_sections(from_dir, to_dir, apk):
1547     """move an APK from repo to archive or vice versa"""
1548
1549     def _move_file(from_dir, to_dir, filename, ignore_missing):
1550         from_path = os.path.join(from_dir, filename)
1551         if ignore_missing and not os.path.exists(from_path):
1552             return
1553         to_path = os.path.join(to_dir, filename)
1554         if not os.path.exists(to_dir):
1555             os.mkdir(to_dir)
1556         shutil.move(from_path, to_path)
1557
1558     if from_dir == to_dir:
1559         return
1560
1561     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1562     _move_file(from_dir, to_dir, apk['apkName'], False)
1563     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1564     for density in all_screen_densities:
1565         from_icon_dir = get_icon_dir(from_dir, density)
1566         to_icon_dir = get_icon_dir(to_dir, density)
1567         if density not in apk['icons']:
1568             continue
1569         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1570     if 'srcname' in apk:
1571         _move_file(from_dir, to_dir, apk['srcname'], False)
1572
1573
1574 def add_apks_to_per_app_repos(repodir, apks):
1575     apks_per_app = dict()
1576     for apk in apks:
1577         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1578         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1579         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1580         apks_per_app[apk['packageName']] = apk
1581
1582         if not os.path.exists(apk['per_app_icons']):
1583             logging.info('Adding new repo for only ' + apk['packageName'])
1584             os.makedirs(apk['per_app_icons'])
1585
1586         apkpath = os.path.join(repodir, apk['apkName'])
1587         shutil.copy(apkpath, apk['per_app_repo'])
1588         apksigpath = apkpath + '.sig'
1589         if os.path.exists(apksigpath):
1590             shutil.copy(apksigpath, apk['per_app_repo'])
1591         apkascpath = apkpath + '.asc'
1592         if os.path.exists(apkascpath):
1593             shutil.copy(apkascpath, apk['per_app_repo'])
1594
1595
1596 config = None
1597 options = None
1598
1599
1600 def main():
1601
1602     global config, options
1603
1604     # Parse command line...
1605     parser = ArgumentParser()
1606     common.setup_global_opts(parser)
1607     parser.add_argument("--create-key", action="store_true", default=False,
1608                         help="Create a repo signing key in a keystore")
1609     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1610                         help="Create skeleton metadata files that are missing")
1611     parser.add_argument("--delete-unknown", action="store_true", default=False,
1612                         help="Delete APKs and/or OBBs without metadata from the repo")
1613     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1614                         help="Report on build data status")
1615     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1616                         help="Interactively ask about things that need updating.")
1617     parser.add_argument("-I", "--icons", action="store_true", default=False,
1618                         help="Resize all the icons exceeding the max pixel size and exit")
1619     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1620                         help="Specify editor to use in interactive mode. Default " +
1621                         "is /etc/alternatives/editor")
1622     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1623                         help="Update the wiki")
1624     parser.add_argument("--pretty", action="store_true", default=False,
1625                         help="Produce human-readable index.xml")
1626     parser.add_argument("--clean", action="store_true", default=False,
1627                         help="Clean update - don't uses caches, reprocess all apks")
1628     parser.add_argument("--nosign", action="store_true", default=False,
1629                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1630     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1631                         help="Use date from apk instead of current time for newly added apks")
1632     parser.add_argument("--rename-apks", action="store_true", default=False,
1633                         help="Rename APK files that do not match package.name_123.apk")
1634     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1635                         help="Include APKs that are signed with disabled algorithms like MD5")
1636     metadata.add_metadata_arguments(parser)
1637     options = parser.parse_args()
1638     metadata.warnings_action = options.W
1639
1640     config = common.read_config(options)
1641
1642     if not ('jarsigner' in config and 'keytool' in config):
1643         raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1644
1645     repodirs = ['repo']
1646     if config['archive_older'] != 0:
1647         repodirs.append('archive')
1648         if not os.path.exists('archive'):
1649             os.mkdir('archive')
1650
1651     if options.icons:
1652         resize_all_icons(repodirs)
1653         sys.exit(0)
1654
1655     if options.rename_apks:
1656         options.clean = True
1657
1658     # check that icons exist now, rather than fail at the end of `fdroid update`
1659     for k in ['repo_icon', 'archive_icon']:
1660         if k in config:
1661             if not os.path.exists(config[k]):
1662                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1663                 sys.exit(1)
1664
1665     # if the user asks to create a keystore, do it now, reusing whatever it can
1666     if options.create_key:
1667         if os.path.exists(config['keystore']):
1668             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1669             logging.critical("\t'" + config['keystore'] + "'")
1670             sys.exit(1)
1671
1672         if 'repo_keyalias' not in config:
1673             config['repo_keyalias'] = socket.getfqdn()
1674             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1675         if 'keydname' not in config:
1676             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1677             common.write_to_config(config, 'keydname', config['keydname'])
1678         if 'keystore' not in config:
1679             config['keystore'] = common.default_config['keystore']
1680             common.write_to_config(config, 'keystore', config['keystore'])
1681
1682         password = common.genpassword()
1683         if 'keystorepass' not in config:
1684             config['keystorepass'] = password
1685             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1686         if 'keypass' not in config:
1687             config['keypass'] = password
1688             common.write_to_config(config, 'keypass', config['keypass'])
1689         common.genkeystore(config)
1690
1691     # Get all apps...
1692     apps = metadata.read_metadata()
1693
1694     # Generate a list of categories...
1695     categories = set()
1696     for app in apps.values():
1697         categories.update(app.Categories)
1698
1699     # Read known apks data (will be updated and written back when we've finished)
1700     knownapks = common.KnownApks()
1701
1702     # Get APK cache
1703     apkcache = get_cache()
1704
1705     # Delete builds for disabled apps
1706     delete_disabled_builds(apps, apkcache, repodirs)
1707
1708     # Scan all apks in the main repo
1709     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1710
1711     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1712                                            options.use_date_from_apk)
1713     cachechanged = cachechanged or fcachechanged
1714     apks += files
1715     # Generate warnings for apk's with no metadata (or create skeleton
1716     # metadata files, if requested on the command line)
1717     newmetadata = False
1718     for apk in apks:
1719         if apk['packageName'] not in apps:
1720             if options.create_metadata:
1721                 if 'name' not in apk:
1722                     logging.error(apk['packageName'] + ' does not have a name! Skipping...')
1723                     continue
1724                 f = open(os.path.join('metadata', apk['packageName'] + '.txt'), 'w', encoding='utf8')
1725                 f.write("License:Unknown\n")
1726                 f.write("Web Site:\n")
1727                 f.write("Source Code:\n")
1728                 f.write("Issue Tracker:\n")
1729                 f.write("Changelog:\n")
1730                 f.write("Summary:" + apk['name'] + "\n")
1731                 f.write("Description:\n")
1732                 f.write(apk['name'] + "\n")
1733                 f.write(".\n")
1734                 f.write("Name:" + apk['name'] + "\n")
1735                 f.close()
1736                 logging.info("Generated skeleton metadata for " + apk['packageName'])
1737                 newmetadata = True
1738             else:
1739                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1740                 if options.delete_unknown:
1741                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1742                     rmf = os.path.join(repodirs[0], apk['apkName'])
1743                     if not os.path.exists(rmf):
1744                         logging.error("Could not find {0} to remove it".format(rmf))
1745                     else:
1746                         os.remove(rmf)
1747                 else:
1748                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1749
1750     # update the metadata with the newly created ones included
1751     if newmetadata:
1752         apps = metadata.read_metadata()
1753
1754     copy_triple_t_store_metadata(apps)
1755     insert_obbs(repodirs[0], apps, apks)
1756     insert_localized_app_metadata(apps)
1757
1758     # Scan the archive repo for apks as well
1759     if len(repodirs) > 1:
1760         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1761         if cc:
1762             cachechanged = True
1763     else:
1764         archapks = []
1765
1766     # Apply information from latest apks to the application and update dates
1767     apply_info_from_latest_apk(apps, apks + archapks)
1768
1769     # Sort the app list by name, then the web site doesn't have to by default.
1770     # (we had to wait until we'd scanned the apks to do this, because mostly the
1771     # name comes from there!)
1772     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1773
1774     # APKs are placed into multiple repos based on the app package, providing
1775     # per-app subscription feeds for nightly builds and things like it
1776     if config['per_app_repos']:
1777         add_apks_to_per_app_repos(repodirs[0], apks)
1778         for appid, app in apps.items():
1779             repodir = os.path.join(appid, 'fdroid', 'repo')
1780             appdict = dict()
1781             appdict[appid] = app
1782             if os.path.isdir(repodir):
1783                 index.make(appdict, [appid], apks, repodir, False)
1784             else:
1785                 logging.info('Skipping index generation for ' + appid)
1786         return
1787
1788     if len(repodirs) > 1:
1789         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1790
1791     # Make the index for the main repo...
1792     index.make(apps, sortedids, apks, repodirs[0], False)
1793     make_categories_txt(repodirs[0], categories)
1794
1795     # If there's an archive repo,  make the index for it. We already scanned it
1796     # earlier on.
1797     if len(repodirs) > 1:
1798         index.make(apps, sortedids, archapks, repodirs[1], True)
1799
1800     git_remote = config.get('binary_transparency_remote')
1801     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1802         from . import btlog
1803         btlog.make_binary_transparency_log(repodirs)
1804
1805     if config['update_stats']:
1806         # Update known apks info...
1807         knownapks.writeifchanged()
1808
1809         # Generate latest apps data for widget
1810         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1811             data = ''
1812             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1813                 for line in f:
1814                     appid = line.rstrip()
1815                     data += appid + "\t"
1816                     app = apps[appid]
1817                     data += app.Name + "\t"
1818                     if app.icon is not None:
1819                         data += app.icon + "\t"
1820                     data += app.License + "\n"
1821             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1822                 f.write(data)
1823
1824     if cachechanged:
1825         write_cache(apkcache)
1826
1827     # Update the wiki...
1828     if options.wiki:
1829         update_wiki(apps, sortedids, apks + archapks)
1830
1831     logging.info("Finished.")
1832
1833
1834 if __name__ == "__main__":
1835     main()