chiark / gitweb /
Merge branch 'plural' into 'master'
[fdroidserver.git] / fdroidserver / update.py
1 #!/usr/bin/env python3
2 #
3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17 # GNU Affero General Public License for more details.
18 #
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
21
22 import sys
23 import os
24 import shutil
25 import glob
26 import re
27 import socket
28 import zipfile
29 import hashlib
30 import pickle
31 from datetime import datetime, timedelta
32 from argparse import ArgumentParser
33
34 import collections
35 from binascii import hexlify
36
37 from PIL import Image
38 import logging
39
40 from . import _
41 from . import common
42 from . import index
43 from . import metadata
44 from .common import SdkToolsPopen
45 from .exception import BuildException, FDroidException
46
47 METADATA_VERSION = 18
48
49 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
50 UNSET_VERSION_CODE = -0x100000000
51
52 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
53 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
54 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
55 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
56 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
57 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
58 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
59 APK_PERMISSION_PAT = \
60     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
61 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
62
63 screen_densities = ['640', '480', '320', '240', '160', '120']
64 screen_resolutions = {
65     "xxxhdpi": '640',
66     "xxhdpi": '480',
67     "xhdpi": '320',
68     "hdpi": '240',
69     "mdpi": '160',
70     "ldpi": '120',
71     "undefined": '-1',
72     "anydpi": '65534',
73     "nodpi": '65535'
74 }
75
76 all_screen_densities = ['0'] + screen_densities
77
78 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
79 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
80
81 ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
82 GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
83 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
84                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
85
86
87 def dpi_to_px(density):
88     return (int(density) * 48) / 160
89
90
91 def px_to_dpi(px):
92     return (int(px) * 160) / 48
93
94
95 def get_icon_dir(repodir, density):
96     if density == '0':
97         return os.path.join(repodir, "icons")
98     return os.path.join(repodir, "icons-%s" % density)
99
100
101 def get_icon_dirs(repodir):
102     for density in screen_densities:
103         yield get_icon_dir(repodir, density)
104
105
106 def get_all_icon_dirs(repodir):
107     for density in all_screen_densities:
108         yield get_icon_dir(repodir, density)
109
110
111 def update_wiki(apps, sortedids, apks):
112     """Update the wiki
113
114     :param apps: fully populated list of all applications
115     :param apks: all apks, except...
116     """
117     logging.info("Updating wiki")
118     wikicat = 'Apps'
119     wikiredircat = 'App Redirects'
120     import mwclient
121     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
122                          path=config['wiki_path'])
123     site.login(config['wiki_user'], config['wiki_password'])
124     generated_pages = {}
125     generated_redirects = {}
126
127     for appid in sortedids:
128         app = metadata.App(apps[appid])
129
130         wikidata = ''
131         if app.Disabled:
132             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
133         if app.AntiFeatures:
134             for af in app.AntiFeatures:
135                 wikidata += '{{AntiFeature|' + af + '}}\n'
136         if app.RequiresRoot:
137             requiresroot = 'Yes'
138         else:
139             requiresroot = 'No'
140         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
141             appid,
142             app.Name,
143             app.added.strftime('%Y-%m-%d') if app.added else '',
144             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
145             app.SourceCode,
146             app.IssueTracker,
147             app.WebSite,
148             app.Changelog,
149             app.Donate,
150             app.FlattrID,
151             app.Bitcoin,
152             app.Litecoin,
153             app.License,
154             requiresroot,
155             app.AuthorName,
156             app.AuthorEmail)
157
158         if app.Provides:
159             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
160
161         wikidata += app.Summary
162         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
163
164         wikidata += "=Description=\n"
165         wikidata += metadata.description_wiki(app.Description) + "\n"
166
167         wikidata += "=Maintainer Notes=\n"
168         if app.MaintainerNotes:
169             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
170         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
171
172         # Get a list of all packages for this application...
173         apklist = []
174         gotcurrentver = False
175         cantupdate = False
176         buildfails = False
177         for apk in apks:
178             if apk['packageName'] == appid:
179                 if str(apk['versionCode']) == app.CurrentVersionCode:
180                     gotcurrentver = True
181                 apklist.append(apk)
182         # Include ones we can't build, as a special case...
183         for build in app.builds:
184             if build.disable:
185                 if build.versionCode == app.CurrentVersionCode:
186                     cantupdate = True
187                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
188                 apklist.append({'versionCode': int(build.versionCode),
189                                 'versionName': build.versionName,
190                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
191                                 })
192             else:
193                 builtit = False
194                 for apk in apklist:
195                     if apk['versionCode'] == int(build.versionCode):
196                         builtit = True
197                         break
198                 if not builtit:
199                     buildfails = True
200                     apklist.append({'versionCode': int(build.versionCode),
201                                     'versionName': build.versionName,
202                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
203                                     })
204         if app.CurrentVersionCode == '0':
205             cantupdate = True
206         # Sort with most recent first...
207         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
208
209         wikidata += "=Versions=\n"
210         if len(apklist) == 0:
211             wikidata += "We currently have no versions of this app available."
212         elif not gotcurrentver:
213             wikidata += "We don't have the current version of this app."
214         else:
215             wikidata += "We have the current version of this app."
216         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
217         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
218         if len(app.NoSourceSince) > 0:
219             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
220         if len(app.CurrentVersion) > 0:
221             wikidata += "The current (recommended) version is " + app.CurrentVersion
222             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
223         validapks = 0
224         for apk in apklist:
225             wikidata += "==" + apk['versionName'] + "==\n"
226
227             if 'buildproblem' in apk:
228                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
229             else:
230                 validapks += 1
231                 wikidata += "This version is built and signed by "
232                 if 'srcname' in apk:
233                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
234                 else:
235                     wikidata += "the original developer.\n\n"
236             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
237
238         wikidata += '\n[[Category:' + wikicat + ']]\n'
239         if len(app.NoSourceSince) > 0:
240             wikidata += '\n[[Category:Apps missing source code]]\n'
241         if validapks == 0 and not app.Disabled:
242             wikidata += '\n[[Category:Apps with no packages]]\n'
243         if cantupdate and not app.Disabled:
244             wikidata += "\n[[Category:Apps we cannot update]]\n"
245         if buildfails and not app.Disabled:
246             wikidata += "\n[[Category:Apps with failing builds]]\n"
247         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
248             wikidata += '\n[[Category:Apps to Update]]\n'
249         if app.Disabled:
250             wikidata += '\n[[Category:Apps that are disabled]]\n'
251         if app.UpdateCheckMode == 'None' and not app.Disabled:
252             wikidata += '\n[[Category:Apps with no update check]]\n'
253         for appcat in app.Categories:
254             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
255
256         # We can't have underscores in the page name, even if they're in
257         # the package ID, because MediaWiki messes with them...
258         pagename = appid.replace('_', ' ')
259
260         # Drop a trailing newline, because mediawiki is going to drop it anyway
261         # and it we don't we'll think the page has changed when it hasn't...
262         if wikidata.endswith('\n'):
263             wikidata = wikidata[:-1]
264
265         generated_pages[pagename] = wikidata
266
267         # Make a redirect from the name to the ID too, unless there's
268         # already an existing page with the name and it isn't a redirect.
269         noclobber = False
270         apppagename = app.Name.replace('_', ' ')
271         apppagename = apppagename.replace('{', '')
272         apppagename = apppagename.replace('}', ' ')
273         apppagename = apppagename.replace(':', ' ')
274         apppagename = apppagename.replace('[', ' ')
275         apppagename = apppagename.replace(']', ' ')
276         # Drop double spaces caused mostly by replacing ':' above
277         apppagename = apppagename.replace('  ', ' ')
278         for expagename in site.allpages(prefix=apppagename,
279                                         filterredir='nonredirects',
280                                         generator=False):
281             if expagename == apppagename:
282                 noclobber = True
283         # Another reason not to make the redirect page is if the app name
284         # is the same as it's ID, because that will overwrite the real page
285         # with an redirect to itself! (Although it seems like an odd
286         # scenario this happens a lot, e.g. where there is metadata but no
287         # builds or binaries to extract a name from.
288         if apppagename == pagename:
289             noclobber = True
290         if not noclobber:
291             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
292
293     for tcat, genp in [(wikicat, generated_pages),
294                        (wikiredircat, generated_redirects)]:
295         catpages = site.Pages['Category:' + tcat]
296         existingpages = []
297         for page in catpages:
298             existingpages.append(page.name)
299             if page.name in genp:
300                 pagetxt = page.edit()
301                 if pagetxt != genp[page.name]:
302                     logging.debug("Updating modified page " + page.name)
303                     page.save(genp[page.name], summary='Auto-updated')
304                 else:
305                     logging.debug("Page " + page.name + " is unchanged")
306             else:
307                 logging.warn("Deleting page " + page.name)
308                 page.delete('No longer published')
309         for pagename, text in genp.items():
310             logging.debug("Checking " + pagename)
311             if pagename not in existingpages:
312                 logging.debug("Creating page " + pagename)
313                 try:
314                     newpage = site.Pages[pagename]
315                     newpage.save(text, summary='Auto-created')
316                 except Exception as e:
317                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
318
319     # Purge server cache to ensure counts are up to date
320     site.pages['Repository Maintenance'].purge()
321
322
323 def delete_disabled_builds(apps, apkcache, repodirs):
324     """Delete disabled build outputs.
325
326     :param apps: list of all applications, as per metadata.read_metadata
327     :param apkcache: current apk cache information
328     :param repodirs: the repo directories to process
329     """
330     for appid, app in apps.items():
331         for build in app['builds']:
332             if not build.disable:
333                 continue
334             apkfilename = common.get_release_filename(app, build)
335             iconfilename = "%s.%s.png" % (
336                 appid,
337                 build.versionCode)
338             for repodir in repodirs:
339                 files = [
340                     os.path.join(repodir, apkfilename),
341                     os.path.join(repodir, apkfilename + '.asc'),
342                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
343                 ]
344                 for density in all_screen_densities:
345                     repo_dir = get_icon_dir(repodir, density)
346                     files.append(os.path.join(repo_dir, iconfilename))
347
348                 for f in files:
349                     if os.path.exists(f):
350                         logging.info("Deleting disabled build output " + f)
351                         os.remove(f)
352             if apkfilename in apkcache:
353                 del apkcache[apkfilename]
354
355
356 def resize_icon(iconpath, density):
357
358     if not os.path.isfile(iconpath):
359         return
360
361     fp = None
362     try:
363         fp = open(iconpath, 'rb')
364         im = Image.open(fp)
365         size = dpi_to_px(density)
366
367         if any(length > size for length in im.size):
368             oldsize = im.size
369             im.thumbnail((size, size), Image.ANTIALIAS)
370             logging.debug("%s was too large at %s - new size is %s" % (
371                 iconpath, oldsize, im.size))
372             im.save(iconpath, "PNG")
373
374     except Exception as e:
375         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
376
377     finally:
378         if fp:
379             fp.close()
380
381
382 def resize_all_icons(repodirs):
383     """Resize all icons that exceed the max size
384
385     :param repodirs: the repo directories to process
386     """
387     for repodir in repodirs:
388         for density in screen_densities:
389             icon_dir = get_icon_dir(repodir, density)
390             icon_glob = os.path.join(icon_dir, '*.png')
391             for iconpath in glob.glob(icon_glob):
392                 resize_icon(iconpath, density)
393
394
395 def getsig(apkpath):
396     """ Get the signing certificate of an apk. To get the same md5 has that
397     Android gets, we encode the .RSA certificate in a specific format and pass
398     it hex-encoded to the md5 digest algorithm.
399
400     :param apkpath: path to the apk
401     :returns: A string containing the md5 of the signature of the apk or None
402               if an error occurred.
403     """
404
405     with zipfile.ZipFile(apkpath, 'r') as apk:
406         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
407
408         if len(certs) < 1:
409             logging.error("Found no signing certificates on %s" % apkpath)
410             return None
411         if len(certs) > 1:
412             logging.error("Found multiple signing certificates on %s" % apkpath)
413             return None
414
415         cert = apk.read(certs[0])
416
417     cert_encoded = common.get_certificate(cert)
418
419     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
420
421
422 def get_cache_file():
423     return os.path.join('tmp', 'apkcache')
424
425
426 def get_cache():
427     """Get the cached dict of the APK index
428
429     Gather information about all the apk files in the repo directory,
430     using cached data if possible. Some of the index operations take a
431     long time, like calculating the SHA-256 and verifying the APK
432     signature.
433
434     The cache is invalidated if the metadata version is different, or
435     the 'allow_disabled_algorithms' config/option is different.  In
436     those cases, there is no easy way to know what has changed from
437     the cache, so just rerun the whole thing.
438
439     :return: apkcache
440
441     """
442     apkcachefile = get_cache_file()
443     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
444     if not options.clean and os.path.exists(apkcachefile):
445         with open(apkcachefile, 'rb') as cf:
446             apkcache = pickle.load(cf, encoding='utf-8')
447         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
448            or apkcache.get('allow_disabled_algorithms') != ada:
449             apkcache = {}
450     else:
451         apkcache = {}
452
453     apkcache["METADATA_VERSION"] = METADATA_VERSION
454     apkcache['allow_disabled_algorithms'] = ada
455
456     return apkcache
457
458
459 def write_cache(apkcache):
460     apkcachefile = get_cache_file()
461     cache_path = os.path.dirname(apkcachefile)
462     if not os.path.exists(cache_path):
463         os.makedirs(cache_path)
464     with open(apkcachefile, 'wb') as cf:
465         pickle.dump(apkcache, cf)
466
467
468 def get_icon_bytes(apkzip, iconsrc):
469     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
470     try:
471         return apkzip.read(iconsrc)
472     except KeyError:
473         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
474
475
476 def sha256sum(filename):
477     '''Calculate the sha256 of the given file'''
478     sha = hashlib.sha256()
479     with open(filename, 'rb') as f:
480         while True:
481             t = f.read(16384)
482             if len(t) == 0:
483                 break
484             sha.update(t)
485     return sha.hexdigest()
486
487
488 def has_known_vulnerability(filename):
489     """checks for known vulnerabilities in the APK
490
491     Checks OpenSSL .so files in the APK to see if they are a known vulnerable
492     version.  Google also enforces this:
493     https://support.google.com/faqs/answer/6376725?hl=en
494
495     Checks whether there are more than one classes.dex or AndroidManifest.xml
496     files, which is invalid and an essential part of the "Master Key" attack.
497
498     http://www.saurik.com/id/17
499     """
500
501     # statically load this pattern
502     if not hasattr(has_known_vulnerability, "pattern"):
503         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
504
505     files_in_apk = set()
506     with zipfile.ZipFile(filename) as zf:
507         for name in zf.namelist():
508             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
509                 lib = zf.open(name)
510                 while True:
511                     chunk = lib.read(4096)
512                     if chunk == b'':
513                         break
514                     m = has_known_vulnerability.pattern.search(chunk)
515                     if m:
516                         version = m.group(1).decode('ascii')
517                         if (version.startswith('1.0.1') and len(version) > 5 and version[5] >= 'r') \
518                            or (version.startswith('1.0.2') and len(version) > 5 and version[5] >= 'f') \
519                            or re.match(r'[1-9]\.[1-9]\.[0-9].*', version):
520                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
521                         else:
522                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
523                             return True
524                         break
525             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
526                 if name in files_in_apk:
527                     return True
528                 files_in_apk.add(name)
529
530     return False
531
532
533 def insert_obbs(repodir, apps, apks):
534     """Scans the .obb files in a given repo directory and adds them to the
535     relevant APK instances.  OBB files have versionCodes like APK
536     files, and they are loosely associated.  If there is an OBB file
537     present, then any APK with the same or higher versionCode will use
538     that OBB file.  There are two OBB types: main and patch, each APK
539     can only have only have one of each.
540
541     https://developer.android.com/google/play/expansion-files.html
542
543     :param repodir: repo directory to scan
544     :param apps: list of current, valid apps
545     :param apks: current information on all APKs
546
547     """
548
549     def obbWarnDelete(f, msg):
550         logging.warning(msg + f)
551         if options.delete_unknown:
552             logging.error("Deleting unknown file: " + f)
553             os.remove(f)
554
555     obbs = []
556     java_Integer_MIN_VALUE = -pow(2, 31)
557     currentPackageNames = apps.keys()
558     for f in glob.glob(os.path.join(repodir, '*.obb')):
559         obbfile = os.path.basename(f)
560         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
561         chunks = obbfile.split('.')
562         if chunks[0] != 'main' and chunks[0] != 'patch':
563             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
564             continue
565         if not re.match(r'^-?[0-9]+$', chunks[1]):
566             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
567             continue
568         versionCode = int(chunks[1])
569         packagename = ".".join(chunks[2:-1])
570
571         highestVersionCode = java_Integer_MIN_VALUE
572         if packagename not in currentPackageNames:
573             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
574             continue
575         for apk in apks:
576             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
577                 highestVersionCode = apk['versionCode']
578         if versionCode > highestVersionCode:
579             obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
580                           + ') than any APK: ')
581             continue
582         obbsha256 = sha256sum(f)
583         obbs.append((packagename, versionCode, obbfile, obbsha256))
584
585     for apk in apks:
586         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
587             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
588                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
589                     apk['obbMainFile'] = obbfile
590                     apk['obbMainFileSha256'] = obbsha256
591                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
592                     apk['obbPatchFile'] = obbfile
593                     apk['obbPatchFileSha256'] = obbsha256
594             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
595                 break
596
597
598 def translate_per_build_anti_features(apps, apks):
599     """Grab the anti-features list from the build metadata
600
601     For most Anti-Features, they are really most applicable per-APK,
602     not for an app.  An app can fix a vulnerability, add/remove
603     tracking, etc.  This reads the 'antifeatures' list from the Build
604     entries in the fdroiddata metadata file, then transforms it into
605     the 'antiFeatures' list of unique items for the index.
606
607     The field key is all lower case in the metadata file to match the
608     rest of the Build fields.  It is 'antiFeatures' camel case in the
609     implementation, index, and fdroidclient since it is translated
610     from the build 'antifeatures' field, not directly included.
611
612     """
613
614     antiFeatures = dict()
615     for packageName, app in apps.items():
616         d = dict()
617         for build in app['builds']:
618             afl = build.get('antifeatures')
619             if afl:
620                 d[int(build.versionCode)] = afl
621         if len(d) > 0:
622             antiFeatures[packageName] = d
623
624     for apk in apks:
625         d = antiFeatures.get(apk['packageName'])
626         if d:
627             afl = d.get(apk['versionCode'])
628             if afl:
629                 apk['antiFeatures'].update(afl)
630
631
632 def _get_localized_dict(app, locale):
633     '''get the dict to add localized store metadata to'''
634     if 'localized' not in app:
635         app['localized'] = collections.OrderedDict()
636     if locale not in app['localized']:
637         app['localized'][locale] = collections.OrderedDict()
638     return app['localized'][locale]
639
640
641 def _set_localized_text_entry(app, locale, key, f):
642     limit = config['char_limits'][key]
643     localized = _get_localized_dict(app, locale)
644     with open(f) as fp:
645         text = fp.read()[:limit]
646         if len(text) > 0:
647             localized[key] = text
648
649
650 def _set_author_entry(app, key, f):
651     limit = config['char_limits']['author']
652     with open(f) as fp:
653         text = fp.read()[:limit]
654         if len(text) > 0:
655             app[key] = text
656
657
658 def copy_triple_t_store_metadata(apps):
659     """Include store metadata from the app's source repo
660
661     The Triple-T Gradle Play Publisher is a plugin that has a standard
662     file layout for all of the metadata and graphics that the Google
663     Play Store accepts.  Since F-Droid has the git repo, it can just
664     pluck those files directly.  This method reads any text files into
665     the app dict, then copies any graphics into the fdroid repo
666     directory structure.
667
668     This needs to be run before insert_localized_app_metadata() so that
669     the graphics files that are copied into the fdroid repo get
670     properly indexed.
671
672     https://github.com/Triple-T/gradle-play-publisher#upload-images
673     https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
674
675     """
676
677     if not os.path.isdir('build'):
678         return  # nothing to do
679
680     for packageName, app in apps.items():
681         for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
682             logging.debug('Triple-T Gradle Play Publisher: ' + d)
683             for root, dirs, files in os.walk(d):
684                 segments = root.split('/')
685                 locale = segments[-2]
686                 for f in files:
687                     if f == 'fulldescription':
688                         _set_localized_text_entry(app, locale, 'description',
689                                                   os.path.join(root, f))
690                         continue
691                     elif f == 'shortdescription':
692                         _set_localized_text_entry(app, locale, 'summary',
693                                                   os.path.join(root, f))
694                         continue
695                     elif f == 'title':
696                         _set_localized_text_entry(app, locale, 'name',
697                                                   os.path.join(root, f))
698                         continue
699                     elif f == 'video':
700                         _set_localized_text_entry(app, locale, 'video',
701                                                   os.path.join(root, f))
702                         continue
703                     elif f == 'whatsnew':
704                         _set_localized_text_entry(app, segments[-1], 'whatsNew',
705                                                   os.path.join(root, f))
706                         continue
707                     elif f == 'contactEmail':
708                         _set_author_entry(app, 'authorEmail', os.path.join(root, f))
709                         continue
710                     elif f == 'contactPhone':
711                         _set_author_entry(app, 'authorPhone', os.path.join(root, f))
712                         continue
713                     elif f == 'contactWebsite':
714                         _set_author_entry(app, 'authorWebSite', os.path.join(root, f))
715                         continue
716
717                     base, extension = common.get_extension(f)
718                     dirname = os.path.basename(root)
719                     if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
720                         if segments[-2] == 'listing':
721                             locale = segments[-3]
722                         else:
723                             locale = segments[-2]
724                         destdir = os.path.join('repo', packageName, locale)
725                         os.makedirs(destdir, mode=0o755, exist_ok=True)
726                         sourcefile = os.path.join(root, f)
727                         destfile = os.path.join(destdir, dirname + '.' + extension)
728                         logging.debug('copying ' + sourcefile + ' ' + destfile)
729                         shutil.copy(sourcefile, destfile)
730
731
732 def insert_localized_app_metadata(apps):
733     """scans standard locations for graphics and localized text
734
735     Scans for localized description files, store graphics, and
736     screenshot PNG files in statically defined screenshots directory
737     and adds them to the app metadata.  The screenshots and graphic
738     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
739     and must be in the following layout:
740     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
741
742     repo/packageName/locale/featureGraphic.png
743     repo/packageName/locale/phoneScreenshots/1.png
744     repo/packageName/locale/phoneScreenshots/2.png
745
746     The changelog files must be text files named with the versionCode
747     ending with ".txt" and must be in the following layout:
748     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
749
750     repo/packageName/locale/changelogs/12345.txt
751
752     This will scan the each app's source repo then the metadata/ dir
753     for these standard locations of changelog files.  If it finds
754     them, they will be added to the dict of all packages, with the
755     versions in the metadata/ folder taking precendence over the what
756     is in the app's source repo.
757
758     Where "packageName" is the app's packageName and "locale" is the locale
759     of the graphics, e.g. what language they are in, using the IETF RFC5646
760     format (en-US, fr-CA, es-MX, etc).
761
762     This will also scan the app's git for a fastlane folder, and the
763     metadata/ folder and the apps' source repos for standard locations
764     of graphic and screenshot files.  If it finds them, it will copy
765     them into the repo.  The fastlane files follow this pattern:
766     https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
767
768     """
769
770     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
771     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
772     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
773
774     for srcd in sorted(sourcedirs):
775         if not os.path.isdir(srcd):
776             continue
777         for root, dirs, files in os.walk(srcd):
778             segments = root.split('/')
779             packageName = segments[1]
780             if packageName not in apps:
781                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
782                 continue
783             locale = segments[-1]
784             destdir = os.path.join('repo', packageName, locale)
785             for f in files:
786                 if f in ('description.txt', 'full_description.txt'):
787                     _set_localized_text_entry(apps[packageName], locale, 'description',
788                                               os.path.join(root, f))
789                     continue
790                 elif f in ('summary.txt', 'short_description.txt'):
791                     _set_localized_text_entry(apps[packageName], locale, 'summary',
792                                               os.path.join(root, f))
793                     continue
794                 elif f in ('name.txt', 'title.txt'):
795                     _set_localized_text_entry(apps[packageName], locale, 'name',
796                                               os.path.join(root, f))
797                     continue
798                 elif f == 'video.txt':
799                     _set_localized_text_entry(apps[packageName], locale, 'video',
800                                               os.path.join(root, f))
801                     continue
802                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
803                     locale = segments[-2]
804                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
805                                               os.path.join(root, f))
806                     continue
807
808                 base, extension = common.get_extension(f)
809                 if locale == 'images':
810                     locale = segments[-2]
811                     destdir = os.path.join('repo', packageName, locale)
812                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
813                     os.makedirs(destdir, mode=0o755, exist_ok=True)
814                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
815                     shutil.copy(os.path.join(root, f), destdir)
816             for d in dirs:
817                 if d in SCREENSHOT_DIRS:
818                     for f in glob.glob(os.path.join(root, d, '*.*')):
819                         _, extension = common.get_extension(f)
820                         if extension in ALLOWED_EXTENSIONS:
821                             screenshotdestdir = os.path.join(destdir, d)
822                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
823                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
824                             shutil.copy(f, screenshotdestdir)
825
826     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
827     for d in repofiles:
828         if not os.path.isdir(d):
829             continue
830         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
831             if not os.path.isfile(f):
832                 continue
833             segments = f.split('/')
834             packageName = segments[1]
835             locale = segments[2]
836             screenshotdir = segments[3]
837             filename = os.path.basename(f)
838             base, extension = common.get_extension(filename)
839
840             if packageName not in apps:
841                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
842                                 % (filename, packageName))
843                 continue
844             graphics = _get_localized_dict(apps[packageName], locale)
845
846             if extension not in ALLOWED_EXTENSIONS:
847                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
848             elif base in GRAPHIC_NAMES:
849                 # there can only be zero or one of these per locale
850                 graphics[base] = filename
851             elif screenshotdir in SCREENSHOT_DIRS:
852                 # there can any number of these per locale
853                 logging.debug('adding to ' + screenshotdir + ': ' + f)
854                 if screenshotdir not in graphics:
855                     graphics[screenshotdir] = []
856                 graphics[screenshotdir].append(filename)
857             else:
858                 logging.warning('Unsupported graphics file found: ' + f)
859
860
861 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
862     """Scan a repo for all files with an extension except APK/OBB
863
864     :param apkcache: current cached info about all repo files
865     :param repodir: repo directory to scan
866     :param knownapks: list of all known files, as per metadata.read_metadata
867     :param use_date_from_file: use date from file (instead of current date)
868                                for newly added files
869     """
870
871     cachechanged = False
872     repo_files = []
873     repodir = repodir.encode('utf-8')
874     for name in os.listdir(repodir):
875         file_extension = common.get_file_extension(name)
876         if file_extension == 'apk' or file_extension == 'obb':
877             continue
878         filename = os.path.join(repodir, name)
879         name_utf8 = name.decode('utf-8')
880         if filename.endswith(b'_src.tar.gz'):
881             logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
882             continue
883         if not common.is_repo_file(filename):
884             continue
885         stat = os.stat(filename)
886         if stat.st_size == 0:
887             raise FDroidException(filename + ' is zero size!')
888
889         shasum = sha256sum(filename)
890         usecache = False
891         if name in apkcache:
892             repo_file = apkcache[name]
893             # added time is cached as tuple but used here as datetime instance
894             if 'added' in repo_file:
895                 a = repo_file['added']
896                 if isinstance(a, datetime):
897                     repo_file['added'] = a
898                 else:
899                     repo_file['added'] = datetime(*a[:6])
900             if repo_file.get('hash') == shasum:
901                 logging.debug("Reading " + name_utf8 + " from cache")
902                 usecache = True
903             else:
904                 logging.debug("Ignoring stale cache data for " + name)
905
906         if not usecache:
907             logging.debug("Processing " + name_utf8)
908             repo_file = collections.OrderedDict()
909             repo_file['name'] = os.path.splitext(name_utf8)[0]
910             # TODO rename apkname globally to something more generic
911             repo_file['apkName'] = name_utf8
912             repo_file['hash'] = shasum
913             repo_file['hashType'] = 'sha256'
914             repo_file['versionCode'] = 0
915             repo_file['versionName'] = shasum
916             # the static ID is the SHA256 unless it is set in the metadata
917             repo_file['packageName'] = shasum
918
919             m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8)
920             if m:
921                 repo_file['packageName'] = m.group(1)
922                 repo_file['versionCode'] = int(m.group(2))
923             srcfilename = name + b'_src.tar.gz'
924             if os.path.exists(os.path.join(repodir, srcfilename)):
925                 repo_file['srcname'] = srcfilename.decode('utf-8')
926             repo_file['size'] = stat.st_size
927
928             apkcache[name] = repo_file
929             cachechanged = True
930
931         if use_date_from_file:
932             timestamp = stat.st_ctime
933             default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
934         else:
935             default_date_param = None
936
937         # Record in knownapks, getting the added date at the same time..
938         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
939                                     default_date=default_date_param)
940         if added:
941             repo_file['added'] = added
942
943         repo_files.append(repo_file)
944
945     return repo_files, cachechanged
946
947
948 def scan_apk(apk_file):
949     """
950     Scans an APK file and returns dictionary with metadata of the APK.
951
952     Attention: This does *not* verify that the APK signature is correct.
953
954     :param apk_file: The (ideally absolute) path to the APK file
955     :raises BuildException
956     :return A dict containing APK metadata
957     """
958     apk = {
959         'hash': sha256sum(apk_file),
960         'hashType': 'sha256',
961         'uses-permission': [],
962         'uses-permission-sdk-23': [],
963         'features': [],
964         'icons_src': {},
965         'icons': {},
966         'antiFeatures': set(),
967     }
968
969     if SdkToolsPopen(['aapt', 'version'], output=False):
970         scan_apk_aapt(apk, apk_file)
971     else:
972         scan_apk_androguard(apk, apk_file)
973
974     # Get the signature
975     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
976     apk['sig'] = getsig(apk_file)
977     if not apk['sig']:
978         raise BuildException("Failed to get apk signature")
979
980     # Get size of the APK
981     apk['size'] = os.path.getsize(apk_file)
982
983     if 'minSdkVersion' not in apk:
984         logging.warning("No SDK version information found in {0}".format(apk_file))
985         apk['minSdkVersion'] = 1
986
987     # Check for known vulnerabilities
988     if has_known_vulnerability(apk_file):
989         apk['antiFeatures'].add('KnownVuln')
990
991     return apk
992
993
994 def scan_apk_aapt(apk, apkfile):
995     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
996     if p.returncode != 0:
997         if options.delete_unknown:
998             if os.path.exists(apkfile):
999                 logging.error("Failed to get apk information, deleting " + apkfile)
1000                 os.remove(apkfile)
1001             else:
1002                 logging.error("Could not find {0} to remove it".format(apkfile))
1003         else:
1004             logging.error("Failed to get apk information, skipping " + apkfile)
1005         raise BuildException("Invalid APK")
1006     for line in p.output.splitlines():
1007         if line.startswith("package:"):
1008             try:
1009                 apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
1010                 apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
1011                 apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
1012             except Exception as e:
1013                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
1014         elif line.startswith("application:"):
1015             apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1016             # Keep path to non-dpi icon in case we need it
1017             match = re.match(APK_ICON_PAT_NODPI, line)
1018             if match:
1019                 apk['icons_src']['-1'] = match.group(1)
1020         elif line.startswith("launchable-activity:"):
1021             # Only use launchable-activity as fallback to application
1022             if not apk['name']:
1023                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
1024             if '-1' not in apk['icons_src']:
1025                 match = re.match(APK_ICON_PAT_NODPI, line)
1026                 if match:
1027                     apk['icons_src']['-1'] = match.group(1)
1028         elif line.startswith("application-icon-"):
1029             match = re.match(APK_ICON_PAT, line)
1030             if match:
1031                 density = match.group(1)
1032                 path = match.group(2)
1033                 apk['icons_src'][density] = path
1034         elif line.startswith("sdkVersion:"):
1035             m = re.match(APK_SDK_VERSION_PAT, line)
1036             if m is None:
1037                 logging.error(line.replace('sdkVersion:', '')
1038                               + ' is not a valid minSdkVersion!')
1039             else:
1040                 apk['minSdkVersion'] = m.group(1)
1041                 # if target not set, default to min
1042                 if 'targetSdkVersion' not in apk:
1043                     apk['targetSdkVersion'] = m.group(1)
1044         elif line.startswith("targetSdkVersion:"):
1045             m = re.match(APK_SDK_VERSION_PAT, line)
1046             if m is None:
1047                 logging.error(line.replace('targetSdkVersion:', '')
1048                               + ' is not a valid targetSdkVersion!')
1049             else:
1050                 apk['targetSdkVersion'] = m.group(1)
1051         elif line.startswith("maxSdkVersion:"):
1052             apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
1053         elif line.startswith("native-code:"):
1054             apk['nativecode'] = []
1055             for arch in line[13:].split(' '):
1056                 apk['nativecode'].append(arch[1:-1])
1057         elif line.startswith('uses-permission:'):
1058             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1059             if perm_match['maxSdkVersion']:
1060                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1061             permission = UsesPermission(
1062                 perm_match['name'],
1063                 perm_match['maxSdkVersion']
1064             )
1065
1066             apk['uses-permission'].append(permission)
1067         elif line.startswith('uses-permission-sdk-23:'):
1068             perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
1069             if perm_match['maxSdkVersion']:
1070                 perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
1071             permission_sdk_23 = UsesPermissionSdk23(
1072                 perm_match['name'],
1073                 perm_match['maxSdkVersion']
1074             )
1075
1076             apk['uses-permission-sdk-23'].append(permission_sdk_23)
1077
1078         elif line.startswith('uses-feature:'):
1079             feature = re.match(APK_FEATURE_PAT, line).group(1)
1080             # Filter out this, it's only added with the latest SDK tools and
1081             # causes problems for lots of apps.
1082             if feature != "android.hardware.screen.portrait" \
1083                     and feature != "android.hardware.screen.landscape":
1084                 if feature.startswith("android.feature."):
1085                     feature = feature[16:]
1086                 apk['features'].add(feature)
1087
1088
1089 def scan_apk_androguard(apk, apkfile):
1090     try:
1091         from androguard.core.bytecodes.apk import APK
1092         apkobject = APK(apkfile)
1093         if apkobject.is_valid_APK():
1094             arsc = apkobject.get_android_resources()
1095         else:
1096             if options.delete_unknown:
1097                 if os.path.exists(apkfile):
1098                     logging.error("Failed to get apk information, deleting " + apkfile)
1099                     os.remove(apkfile)
1100                 else:
1101                     logging.error("Could not find {0} to remove it".format(apkfile))
1102             else:
1103                 logging.error("Failed to get apk information, skipping " + apkfile)
1104             raise BuildException("Invaild APK")
1105     except ImportError:
1106         raise FDroidException("androguard library is not installed and aapt not present")
1107     except FileNotFoundError:
1108         logging.error("Could not open apk file for analysis")
1109         raise BuildException("Invalid APK")
1110
1111     apk['packageName'] = apkobject.get_package()
1112     apk['versionCode'] = int(apkobject.get_androidversion_code())
1113     apk['versionName'] = apkobject.get_androidversion_name()
1114     if apk['versionName'][0] == "@":
1115         version_id = int(apk['versionName'].replace("@", "0x"), 16)
1116         version_id = arsc.get_id(apk['packageName'], version_id)[1]
1117         apk['versionName'] = arsc.get_string(apk['packageName'], version_id)[1]
1118     apk['name'] = apkobject.get_app_name()
1119
1120     if apkobject.get_max_sdk_version() is not None:
1121         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
1122     apk['minSdkVersion'] = apkobject.get_min_sdk_version()
1123     apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
1124
1125     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
1126     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
1127
1128     density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
1129
1130     for file in apkobject.get_files():
1131         d_re = density_re.match(file)
1132         if d_re:
1133             folder = d_re.group(1).split('-')
1134             if len(folder) > 1:
1135                 resolution = folder[1]
1136             else:
1137                 resolution = 'mdpi'
1138             density = screen_resolutions[resolution]
1139             apk['icons_src'][density] = d_re.group(0)
1140
1141     if apk['icons_src'].get('-1') is None:
1142         apk['icons_src']['-1'] = apk['icons_src']['160']
1143
1144     arch_re = re.compile("^lib/(.*)/.*$")
1145     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
1146     if len(arch) >= 1:
1147         apk['nativecode'] = []
1148         apk['nativecode'].extend(sorted(list(arch)))
1149
1150     xml = apkobject.get_android_manifest_xml()
1151
1152     for item in xml.getElementsByTagName('uses-permission'):
1153         name = str(item.getAttribute("android:name"))
1154         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1155         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1156         permission = UsesPermission(
1157             name,
1158             maxSdkVersion
1159         )
1160         apk['uses-permission'].append(permission)
1161
1162     for item in xml.getElementsByTagName('uses-permission-sdk-23'):
1163         name = str(item.getAttribute("android:name"))
1164         maxSdkVersion = item.getAttribute("android:maxSdkVersion")
1165         maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
1166         permission_sdk_23 = UsesPermissionSdk23(
1167             name,
1168             maxSdkVersion
1169         )
1170         apk['uses-permission-sdk-23'].append(permission_sdk_23)
1171
1172     for item in xml.getElementsByTagName('uses-feature'):
1173         feature = str(item.getAttribute("android:name"))
1174         if feature != "android.hardware.screen.portrait" \
1175                 and feature != "android.hardware.screen.landscape":
1176             if feature.startswith("android.feature."):
1177                 feature = feature[16:]
1178         apk['features'].append(feature)
1179
1180
1181 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
1182                 allow_disabled_algorithms=False, archive_bad_sig=False):
1183     """Processes the apk with the given filename in the given repo directory.
1184
1185     This also extracts the icons.
1186
1187     :param apkcache: current apk cache information
1188     :param apkfilename: the filename of the apk to scan
1189     :param repodir: repo directory to scan
1190     :param knownapks: known apks info
1191     :param use_date_from_apk: use date from APK (instead of current date)
1192                               for newly added APKs
1193     :param allow_disabled_algorithms: allow APKs with valid signatures that include
1194                                       disabled algorithms in the signature (e.g. MD5)
1195     :param archive_bad_sig: move APKs with a bad signature to the archive
1196     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
1197      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
1198     """
1199
1200     if ' ' in apkfilename:
1201         if options.rename_apks:
1202             newfilename = apkfilename.replace(' ', '_')
1203             os.rename(os.path.join(repodir, apkfilename),
1204                       os.path.join(repodir, newfilename))
1205             apkfilename = newfilename
1206         else:
1207             logging.critical("Spaces in filenames are not allowed.")
1208             return True, None, False
1209
1210     apk = {}
1211     apkfile = os.path.join(repodir, apkfilename)
1212
1213     cachechanged = False
1214     usecache = False
1215     if apkfilename in apkcache:
1216         apk = apkcache[apkfilename]
1217         if apk.get('hash') == sha256sum(apkfile):
1218             logging.debug("Reading " + apkfilename + " from cache")
1219             usecache = True
1220         else:
1221             logging.debug("Ignoring stale cache data for " + apkfilename)
1222
1223     if not usecache:
1224         logging.debug("Processing " + apkfilename)
1225
1226         try:
1227             apk = scan_apk(apkfile)
1228         except BuildException:
1229             logging.warning('Skipping "%s" with invalid signature!', apkfilename)
1230             return True, None, False
1231
1232         # Check for debuggable apks...
1233         if common.isApkAndDebuggable(apkfile):
1234             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
1235
1236         if options.rename_apks:
1237             n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk'
1238             std_short_name = os.path.join(repodir, n)
1239             if apkfile != std_short_name:
1240                 if os.path.exists(std_short_name):
1241                     std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk')
1242                     if apkfile != std_long_name:
1243                         if os.path.exists(std_long_name):
1244                             dupdir = os.path.join('duplicates', repodir)
1245                             if not os.path.isdir(dupdir):
1246                                 os.makedirs(dupdir, exist_ok=True)
1247                             dupfile = os.path.join('duplicates', std_long_name)
1248                             logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile)
1249                             os.rename(apkfile, dupfile)
1250                             return True, None, False
1251                         else:
1252                             os.rename(apkfile, std_long_name)
1253                     apkfile = std_long_name
1254                 else:
1255                     os.rename(apkfile, std_short_name)
1256                     apkfile = std_short_name
1257                 apkfilename = apkfile[len(repodir) + 1:]
1258
1259         apk['apkName'] = apkfilename
1260         srcfilename = apkfilename[:-4] + "_src.tar.gz"
1261         if os.path.exists(os.path.join(repodir, srcfilename)):
1262             apk['srcname'] = srcfilename
1263
1264         # verify the jar signature is correct, allow deprecated
1265         # algorithms only if the APK is in the archive.
1266         skipapk = False
1267         if not common.verify_apk_signature(apkfile):
1268             if repodir == 'archive' or allow_disabled_algorithms:
1269                 if common.verify_old_apk_signature(apkfile):
1270                     apk['antiFeatures'].update(['KnownVuln', 'DisabledAlgorithm'])
1271                 else:
1272                     skipapk = True
1273             else:
1274                 skipapk = True
1275
1276         if skipapk:
1277             if archive_bad_sig:
1278                 logging.warning('Archiving "' + apkfilename + '" with invalid signature!')
1279                 move_apk_between_sections(repodir, 'archive', apk)
1280             else:
1281                 logging.warning('Skipping "' + apkfilename + '" with invalid signature!')
1282             return True, None, False
1283
1284         apkzip = zipfile.ZipFile(apkfile, 'r')
1285
1286         # if an APK has files newer than the system time, suggest updating
1287         # the system clock.  This is useful for offline systems, used for
1288         # signing, which do not have another source of clock sync info. It
1289         # has to be more than 24 hours newer because ZIP/APK files do not
1290         # store timezone info
1291         manifest = apkzip.getinfo('AndroidManifest.xml')
1292         if manifest.date_time[1] == 0:  # month can't be zero
1293             logging.debug('AndroidManifest.xml has no date')
1294         else:
1295             dt_obj = datetime(*manifest.date_time)
1296             checkdt = dt_obj - timedelta(1)
1297             if datetime.today() < checkdt:
1298                 logging.warning('System clock is older than manifest in: '
1299                                 + apkfilename
1300                                 + '\nSet clock to that time using:\n'
1301                                 + 'sudo date -s "' + str(dt_obj) + '"')
1302
1303         # extract icons from APK zip file
1304         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
1305         try:
1306             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
1307         finally:
1308             apkzip.close()  # ensure that APK zip file gets closed
1309
1310         # resize existing icons for densities missing in the APK
1311         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
1312
1313         if use_date_from_apk and manifest.date_time[1] != 0:
1314             default_date_param = datetime(*manifest.date_time)
1315         else:
1316             default_date_param = None
1317
1318         # Record in known apks, getting the added date at the same time..
1319         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1320                                     default_date=default_date_param)
1321         if added:
1322             apk['added'] = added
1323
1324         apkcache[apkfilename] = apk
1325         cachechanged = True
1326
1327     return False, apk, cachechanged
1328
1329
1330 def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1331     """Processes the apks in the given repo directory.
1332
1333     This also extracts the icons.
1334
1335     :param apkcache: current apk cache information
1336     :param repodir: repo directory to scan
1337     :param knownapks: known apks info
1338     :param use_date_from_apk: use date from APK (instead of current date)
1339                               for newly added APKs
1340     :returns: (apks, cachechanged) where apks is a list of apk information,
1341               and cachechanged is True if the apkcache got changed.
1342     """
1343
1344     cachechanged = False
1345
1346     for icon_dir in get_all_icon_dirs(repodir):
1347         if os.path.exists(icon_dir):
1348             if options.clean:
1349                 shutil.rmtree(icon_dir)
1350                 os.makedirs(icon_dir)
1351         else:
1352             os.makedirs(icon_dir)
1353
1354     apks = []
1355     for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
1356         apkfilename = apkfile[len(repodir) + 1:]
1357         ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
1358         (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks,
1359                                              use_date_from_apk, ada, True)
1360         if skip:
1361             continue
1362         apks.append(apk)
1363         cachechanged = cachechanged or cachethis
1364
1365     return apks, cachechanged
1366
1367
1368 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
1369     """
1370     Extracts icons from the given APK zip in various densities,
1371     saves them into given repo directory
1372     and stores their names in the APK metadata dictionary.
1373
1374     :param icon_filename: A string representing the icon's file name
1375     :param apk: A populated dictionary containing APK metadata.
1376                 Needs to have 'icons_src' key
1377     :param apkzip: An opened zipfile.ZipFile of the APK file
1378     :param repo_dir: The directory of the APK's repository
1379     :return: A list of icon densities that are missing
1380     """
1381     empty_densities = []
1382     for density in screen_densities:
1383         if density not in apk['icons_src']:
1384             empty_densities.append(density)
1385             continue
1386         icon_src = apk['icons_src'][density]
1387         icon_dir = get_icon_dir(repo_dir, density)
1388         icon_dest = os.path.join(icon_dir, icon_filename)
1389
1390         # Extract the icon files per density
1391         if icon_src.endswith('.xml'):
1392             png = os.path.basename(icon_src)[:-4] + '.png'
1393             for f in apkzip.namelist():
1394                 if f.endswith(png):
1395                     m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
1396                     if m and screen_resolutions[m.group(2)] == density:
1397                         icon_src = f
1398             if icon_src.endswith('.xml'):
1399                 empty_densities.append(density)
1400                 continue
1401         try:
1402             with open(icon_dest, 'wb') as f:
1403                 f.write(get_icon_bytes(apkzip, icon_src))
1404             apk['icons'][density] = icon_filename
1405         except (zipfile.BadZipFile, ValueError, KeyError) as e:
1406             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
1407             del apk['icons_src'][density]
1408             empty_densities.append(density)
1409
1410     if '-1' in apk['icons_src']:
1411         icon_src = apk['icons_src']['-1']
1412         icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
1413         with open(icon_path, 'wb') as f:
1414             f.write(get_icon_bytes(apkzip, icon_src))
1415         try:
1416             im = Image.open(icon_path)
1417             dpi = px_to_dpi(im.size[0])
1418             for density in screen_densities:
1419                 if density in apk['icons']:
1420                     break
1421                 if density == screen_densities[-1] or dpi >= int(density):
1422                     apk['icons'][density] = icon_filename
1423                     shutil.move(icon_path,
1424                                 os.path.join(get_icon_dir(repo_dir, density), icon_filename))
1425                     empty_densities.remove(density)
1426                     break
1427         except Exception as e:
1428             logging.warning("Failed reading {0} - {1}".format(icon_path, e))
1429
1430     if apk['icons']:
1431         apk['icon'] = icon_filename
1432
1433     return empty_densities
1434
1435
1436 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
1437     """
1438     Resize existing icons for densities missing in the APK to ensure all densities are available
1439
1440     :param empty_densities: A list of icon densities that are missing
1441     :param icon_filename: A string representing the icon's file name
1442     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
1443     :param repo_dir: The directory of the APK's repository
1444     """
1445     # First try resizing down to not lose quality
1446     last_density = None
1447     for density in screen_densities:
1448         if density not in empty_densities:
1449             last_density = density
1450             continue
1451         if last_density is None:
1452             continue
1453         logging.debug("Density %s not available, resizing down from %s", density, last_density)
1454
1455         last_icon_path = os.path.join(get_icon_dir(repo_dir, last_density), icon_filename)
1456         icon_path = os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1457         fp = None
1458         try:
1459             fp = open(last_icon_path, 'rb')
1460             im = Image.open(fp)
1461
1462             size = dpi_to_px(density)
1463
1464             im.thumbnail((size, size), Image.ANTIALIAS)
1465             im.save(icon_path, "PNG")
1466             empty_densities.remove(density)
1467         except Exception as e:
1468             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
1469         finally:
1470             if fp:
1471                 fp.close()
1472
1473     # Then just copy from the highest resolution available
1474     last_density = None
1475     for density in reversed(screen_densities):
1476         if density not in empty_densities:
1477             last_density = density
1478             continue
1479
1480         if last_density is None:
1481             continue
1482
1483         shutil.copyfile(
1484             os.path.join(get_icon_dir(repo_dir, last_density), icon_filename),
1485             os.path.join(get_icon_dir(repo_dir, density), icon_filename)
1486         )
1487         empty_densities.remove(density)
1488
1489     for density in screen_densities:
1490         icon_dir = get_icon_dir(repo_dir, density)
1491         icon_dest = os.path.join(icon_dir, icon_filename)
1492         resize_icon(icon_dest, density)
1493
1494     # Copy from icons-mdpi to icons since mdpi is the baseline density
1495     baseline = os.path.join(get_icon_dir(repo_dir, '160'), icon_filename)
1496     if os.path.isfile(baseline):
1497         apk['icons']['0'] = icon_filename
1498         shutil.copyfile(baseline, os.path.join(get_icon_dir(repo_dir, '0'), icon_filename))
1499
1500
1501 def apply_info_from_latest_apk(apps, apks):
1502     """
1503     Some information from the apks needs to be applied up to the application level.
1504     When doing this, we use the info from the most recent version's apk.
1505     We deal with figuring out when the app was added and last updated at the same time.
1506     """
1507     for appid, app in apps.items():
1508         bestver = UNSET_VERSION_CODE
1509         for apk in apks:
1510             if apk['packageName'] == appid:
1511                 if apk['versionCode'] > bestver:
1512                     bestver = apk['versionCode']
1513                     bestapk = apk
1514
1515                 if 'added' in apk:
1516                     if not app.added or apk['added'] < app.added:
1517                         app.added = apk['added']
1518                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1519                         app.lastUpdated = apk['added']
1520
1521         if not app.added:
1522             logging.debug("Don't know when " + appid + " was added")
1523         if not app.lastUpdated:
1524             logging.debug("Don't know when " + appid + " was last updated")
1525
1526         if bestver == UNSET_VERSION_CODE:
1527
1528             if app.Name is None:
1529                 app.Name = app.AutoName or appid
1530             app.icon = None
1531             logging.debug("Application " + appid + " has no packages")
1532         else:
1533             if app.Name is None:
1534                 app.Name = bestapk['name']
1535             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1536             if app.CurrentVersionCode is None:
1537                 app.CurrentVersionCode = str(bestver)
1538
1539
1540 def make_categories_txt(repodir, categories):
1541     '''Write a category list in the repo to allow quick access'''
1542     catdata = ''
1543     for cat in sorted(categories):
1544         catdata += cat + '\n'
1545     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1546         f.write(catdata)
1547
1548
1549 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1550
1551     def filter_apk_list_sorted(apk_list):
1552         res = []
1553         for apk in apk_list:
1554             if apk['packageName'] == appid:
1555                 res.append(apk)
1556
1557         # Sort the apk list by version code. First is highest/newest.
1558         return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1559
1560     for appid, app in apps.items():
1561
1562         if app.ArchivePolicy:
1563             keepversions = int(app.ArchivePolicy[:-9])
1564         else:
1565             keepversions = defaultkeepversions
1566
1567         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1568                       .format(appid, len(apks), keepversions, len(archapks)))
1569
1570         current_app_apks = filter_apk_list_sorted(apks)
1571         if len(current_app_apks) > keepversions:
1572             # Move back the ones we don't want.
1573             for apk in current_app_apks[keepversions:]:
1574                 move_apk_between_sections(repodir, archivedir, apk)
1575                 archapks.append(apk)
1576                 apks.remove(apk)
1577
1578         current_app_archapks = filter_apk_list_sorted(archapks)
1579         if len(current_app_apks) < keepversions and len(current_app_archapks) > 0:
1580             kept = 0
1581             # Move forward the ones we want again, except DisableAlgorithm
1582             for apk in current_app_archapks:
1583                 if 'DisabledAlgorithm' not in apk['antiFeatures']:
1584                     move_apk_between_sections(archivedir, repodir, apk)
1585                     archapks.remove(apk)
1586                     apks.append(apk)
1587                     kept += 1
1588                 if kept == keepversions:
1589                     break
1590
1591
1592 def move_apk_between_sections(from_dir, to_dir, apk):
1593     """move an APK from repo to archive or vice versa"""
1594
1595     def _move_file(from_dir, to_dir, filename, ignore_missing):
1596         from_path = os.path.join(from_dir, filename)
1597         if ignore_missing and not os.path.exists(from_path):
1598             return
1599         to_path = os.path.join(to_dir, filename)
1600         if not os.path.exists(to_dir):
1601             os.mkdir(to_dir)
1602         shutil.move(from_path, to_path)
1603
1604     if from_dir == to_dir:
1605         return
1606
1607     logging.info("Moving %s from %s to %s" % (apk['apkName'], from_dir, to_dir))
1608     _move_file(from_dir, to_dir, apk['apkName'], False)
1609     _move_file(from_dir, to_dir, apk['apkName'] + '.asc', True)
1610     for density in all_screen_densities:
1611         from_icon_dir = get_icon_dir(from_dir, density)
1612         to_icon_dir = get_icon_dir(to_dir, density)
1613         if density not in apk['icons']:
1614             continue
1615         _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True)
1616     if 'srcname' in apk:
1617         _move_file(from_dir, to_dir, apk['srcname'], False)
1618
1619
1620 def add_apks_to_per_app_repos(repodir, apks):
1621     apks_per_app = dict()
1622     for apk in apks:
1623         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1624         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1625         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1626         apks_per_app[apk['packageName']] = apk
1627
1628         if not os.path.exists(apk['per_app_icons']):
1629             logging.info('Adding new repo for only ' + apk['packageName'])
1630             os.makedirs(apk['per_app_icons'])
1631
1632         apkpath = os.path.join(repodir, apk['apkName'])
1633         shutil.copy(apkpath, apk['per_app_repo'])
1634         apksigpath = apkpath + '.sig'
1635         if os.path.exists(apksigpath):
1636             shutil.copy(apksigpath, apk['per_app_repo'])
1637         apkascpath = apkpath + '.asc'
1638         if os.path.exists(apkascpath):
1639             shutil.copy(apkascpath, apk['per_app_repo'])
1640
1641
1642 def create_metadata_from_template(apk):
1643     '''create a new metadata file using internal or external template
1644
1645     Generate warnings for apk's with no metadata (or create skeleton
1646     metadata files, if requested on the command line).  Though the
1647     template file is YAML, this uses neither pyyaml nor ruamel.yaml
1648     since those impose things on the metadata file made from the
1649     template: field sort order, empty field value, formatting, etc.
1650     '''
1651
1652     import yaml
1653     if os.path.exists('template.yml'):
1654         with open('template.yml') as f:
1655             metatxt = f.read()
1656         if 'name' in apk and apk['name'] != '':
1657             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1658                              r'\1 ' + apk['name'],
1659                              metatxt,
1660                              flags=re.IGNORECASE | re.MULTILINE)
1661         else:
1662             logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1663             metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
1664                              r'\1 ' + apk['packageName'],
1665                              metatxt,
1666                              flags=re.IGNORECASE | re.MULTILINE)
1667         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1668             f.write(metatxt)
1669     else:
1670         app = dict()
1671         app['Categories'] = [os.path.basename(os.getcwd())]
1672         # include some blanks as part of the template
1673         app['AuthorName'] = ''
1674         app['Summary'] = ''
1675         app['WebSite'] = ''
1676         app['IssueTracker'] = ''
1677         app['SourceCode'] = ''
1678         app['CurrentVersionCode'] = 2147483647  # Java's Integer.MAX_VALUE
1679         if 'name' in apk and apk['name'] != '':
1680             app['Name'] = apk['name']
1681         else:
1682             logging.warning(apk['packageName'] + ' does not have a name! Using package name instead.')
1683             app['Name'] = apk['packageName']
1684         with open(os.path.join('metadata', apk['packageName'] + '.yml'), 'w') as f:
1685             yaml.dump(app, f, default_flow_style=False)
1686     logging.info("Generated skeleton metadata for " + apk['packageName'])
1687
1688
1689 config = None
1690 options = None
1691
1692
1693 def main():
1694
1695     global config, options
1696
1697     # Parse command line...
1698     parser = ArgumentParser()
1699     common.setup_global_opts(parser)
1700     parser.add_argument("--create-key", action="store_true", default=False,
1701                         help=_("Create a repo signing key in a keystore"))
1702     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1703                         help=_("Create skeleton metadata files that are missing"))
1704     parser.add_argument("--delete-unknown", action="store_true", default=False,
1705                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
1706     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1707                         help=_("Report on build data status"))
1708     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1709                         help=_("Interactively ask about things that need updating."))
1710     parser.add_argument("-I", "--icons", action="store_true", default=False,
1711                         help=_("Resize all the icons exceeding the max pixel size and exit"))
1712     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1713                         help=_("Specify editor to use in interactive mode. Default ") +
1714                         "is /etc/alternatives/editor")
1715     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1716                         help=_("Update the wiki"))
1717     parser.add_argument("--pretty", action="store_true", default=False,
1718                         help=_("Produce human-readable index.xml"))
1719     parser.add_argument("--clean", action="store_true", default=False,
1720                         help=_("Clean update - don't uses caches, reprocess all apks"))
1721     parser.add_argument("--nosign", action="store_true", default=False,
1722                         help=_("When configured for signed indexes, create only unsigned indexes at this stage"))
1723     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1724                         help=_("Use date from apk instead of current time for newly added apks"))
1725     parser.add_argument("--rename-apks", action="store_true", default=False,
1726                         help=_("Rename APK files that do not match package.name_123.apk"))
1727     parser.add_argument("--allow-disabled-algorithms", action="store_true", default=False,
1728                         help=_("Include APKs that are signed with disabled algorithms like MD5"))
1729     metadata.add_metadata_arguments(parser)
1730     options = parser.parse_args()
1731     metadata.warnings_action = options.W
1732
1733     config = common.read_config(options)
1734
1735     if not ('jarsigner' in config and 'keytool' in config):
1736         raise FDroidException('Java JDK not found! Install in standard location or set java_paths!')
1737
1738     repodirs = ['repo']
1739     if config['archive_older'] != 0:
1740         repodirs.append('archive')
1741         if not os.path.exists('archive'):
1742             os.mkdir('archive')
1743
1744     if options.icons:
1745         resize_all_icons(repodirs)
1746         sys.exit(0)
1747
1748     if options.rename_apks:
1749         options.clean = True
1750
1751     # check that icons exist now, rather than fail at the end of `fdroid update`
1752     for k in ['repo_icon', 'archive_icon']:
1753         if k in config:
1754             if not os.path.exists(config[k]):
1755                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1756                 sys.exit(1)
1757
1758     # if the user asks to create a keystore, do it now, reusing whatever it can
1759     if options.create_key:
1760         if os.path.exists(config['keystore']):
1761             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1762             logging.critical("\t'" + config['keystore'] + "'")
1763             sys.exit(1)
1764
1765         if 'repo_keyalias' not in config:
1766             config['repo_keyalias'] = socket.getfqdn()
1767             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1768         if 'keydname' not in config:
1769             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1770             common.write_to_config(config, 'keydname', config['keydname'])
1771         if 'keystore' not in config:
1772             config['keystore'] = common.default_config['keystore']
1773             common.write_to_config(config, 'keystore', config['keystore'])
1774
1775         password = common.genpassword()
1776         if 'keystorepass' not in config:
1777             config['keystorepass'] = password
1778             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1779         if 'keypass' not in config:
1780             config['keypass'] = password
1781             common.write_to_config(config, 'keypass', config['keypass'])
1782         common.genkeystore(config)
1783
1784     # Get all apps...
1785     apps = metadata.read_metadata()
1786
1787     # Generate a list of categories...
1788     categories = set()
1789     for app in apps.values():
1790         categories.update(app.Categories)
1791
1792     # Read known apks data (will be updated and written back when we've finished)
1793     knownapks = common.KnownApks()
1794
1795     # Get APK cache
1796     apkcache = get_cache()
1797
1798     # Delete builds for disabled apps
1799     delete_disabled_builds(apps, apkcache, repodirs)
1800
1801     # Scan all apks in the main repo
1802     apks, cachechanged = process_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1803
1804     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1805                                            options.use_date_from_apk)
1806     cachechanged = cachechanged or fcachechanged
1807     apks += files
1808     for apk in apks:
1809         if apk['packageName'] not in apps:
1810             if options.create_metadata:
1811                 create_metadata_from_template(apk)
1812                 apps = metadata.read_metadata()
1813             else:
1814                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1815                 if options.delete_unknown:
1816                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1817                     rmf = os.path.join(repodirs[0], apk['apkName'])
1818                     if not os.path.exists(rmf):
1819                         logging.error("Could not find {0} to remove it".format(rmf))
1820                     else:
1821                         os.remove(rmf)
1822                 else:
1823                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1824
1825     copy_triple_t_store_metadata(apps)
1826     insert_obbs(repodirs[0], apps, apks)
1827     insert_localized_app_metadata(apps)
1828     translate_per_build_anti_features(apps, apks)
1829
1830     # Scan the archive repo for apks as well
1831     if len(repodirs) > 1:
1832         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1833         if cc:
1834             cachechanged = True
1835     else:
1836         archapks = []
1837
1838     # Apply information from latest apks to the application and update dates
1839     apply_info_from_latest_apk(apps, apks + archapks)
1840
1841     # Sort the app list by name, then the web site doesn't have to by default.
1842     # (we had to wait until we'd scanned the apks to do this, because mostly the
1843     # name comes from there!)
1844     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1845
1846     # APKs are placed into multiple repos based on the app package, providing
1847     # per-app subscription feeds for nightly builds and things like it
1848     if config['per_app_repos']:
1849         add_apks_to_per_app_repos(repodirs[0], apks)
1850         for appid, app in apps.items():
1851             repodir = os.path.join(appid, 'fdroid', 'repo')
1852             appdict = dict()
1853             appdict[appid] = app
1854             if os.path.isdir(repodir):
1855                 index.make(appdict, [appid], apks, repodir, False)
1856             else:
1857                 logging.info('Skipping index generation for ' + appid)
1858         return
1859
1860     if len(repodirs) > 1:
1861         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1862
1863     # Make the index for the main repo...
1864     index.make(apps, sortedids, apks, repodirs[0], False)
1865     make_categories_txt(repodirs[0], categories)
1866
1867     # If there's an archive repo,  make the index for it. We already scanned it
1868     # earlier on.
1869     if len(repodirs) > 1:
1870         index.make(apps, sortedids, archapks, repodirs[1], True)
1871
1872     git_remote = config.get('binary_transparency_remote')
1873     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
1874         from . import btlog
1875         btlog.make_binary_transparency_log(repodirs)
1876
1877     if config['update_stats']:
1878         # Update known apks info...
1879         knownapks.writeifchanged()
1880
1881         # Generate latest apps data for widget
1882         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1883             data = ''
1884             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1885                 for line in f:
1886                     appid = line.rstrip()
1887                     data += appid + "\t"
1888                     app = apps[appid]
1889                     data += app.Name + "\t"
1890                     if app.icon is not None:
1891                         data += app.icon + "\t"
1892                     data += app.License + "\n"
1893             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1894                 f.write(data)
1895
1896     if cachechanged:
1897         write_cache(apkcache)
1898
1899     # Update the wiki...
1900     if options.wiki:
1901         update_wiki(apps, sortedids, apks + archapks)
1902
1903     logging.info(_("Finished"))
1904
1905
1906 if __name__ == "__main__":
1907     main()