fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import json
  27 import re
  28 import socket
  29 import zipfile
  30 import hashlib
  31 import pickle
  32 import platform
  33 from datetime import datetime, timedelta
  34 from argparse import ArgumentParser
  35
  36 import collections
  37 from binascii import hexlify
  38
  39 from PIL import Image
  40 import logging
  41
  42 from . import common
  43 from . import index
  44 from . import metadata
  45 from .common import SdkToolsPopen
  46
  47 METADATA_VERSION = 18
  48
  49 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
  50 UNSET_VERSION_CODE = -0x100000000
  51
  52 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
  53 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
  54 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
  55 APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
  56 APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
  57 APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
  58 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
  59 APK_PERMISSION_PAT = \
  60     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
  61 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
  62
  63 screen_densities = ['640', '480', '320', '240', '160', '120']
  64
  65 all_screen_densities = ['0'] + screen_densities
  66
  67 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  68 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  69
  70
  71 def dpi_to_px(density):
  72     return (int(density) * 48) / 160
  73
  74
  75 def px_to_dpi(px):
  76     return (int(px) * 160) / 48
  77
  78
  79 def get_icon_dir(repodir, density):
  80     if density == '0':
  81         return os.path.join(repodir, "icons")
  82     return os.path.join(repodir, "icons-%s" % density)
  83
  84
  85 def get_icon_dirs(repodir):
  86     for density in screen_densities:
  87         yield get_icon_dir(repodir, density)
  88
  89
  90 def get_all_icon_dirs(repodir):
  91     for density in all_screen_densities:
  92         yield get_icon_dir(repodir, density)
  93
  94
  95 def update_wiki(apps, sortedids, apks):
  96     """Update the wiki
  97
  98     :param apps: fully populated list of all applications
  99     :param apks: all apks, except...
 100     """
 101     logging.info("Updating wiki")
 102     wikicat = 'Apps'
 103     wikiredircat = 'App Redirects'
 104     import mwclient
 105     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
 106                          path=config['wiki_path'])
 107     site.login(config['wiki_user'], config['wiki_password'])
 108     generated_pages = {}
 109     generated_redirects = {}
 110
 111     for appid in sortedids:
 112         app = metadata.App(apps[appid])
 113
 114         wikidata = ''
 115         if app.Disabled:
 116             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 117         if app.AntiFeatures:
 118             for af in app.AntiFeatures:
 119                 wikidata += '{{AntiFeature|' + af + '}}\n'
 120         if app.RequiresRoot:
 121             requiresroot = 'Yes'
 122         else:
 123             requiresroot = 'No'
 124         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 125             appid,
 126             app.Name,
 127             app.added.strftime('%Y-%m-%d') if app.added else '',
 128             app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
 129             app.SourceCode,
 130             app.IssueTracker,
 131             app.WebSite,
 132             app.Changelog,
 133             app.Donate,
 134             app.FlattrID,
 135             app.Bitcoin,
 136             app.Litecoin,
 137             app.License,
 138             requiresroot,
 139             app.AuthorName,
 140             app.AuthorEmail)
 141
 142         if app.Provides:
 143             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 144
 145         wikidata += app.Summary
 146         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 147
 148         wikidata += "=Description=\n"
 149         wikidata += metadata.description_wiki(app.Description) + "\n"
 150
 151         wikidata += "=Maintainer Notes=\n"
 152         if app.MaintainerNotes:
 153             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 154         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 155
 156         # Get a list of all packages for this application...
 157         apklist = []
 158         gotcurrentver = False
 159         cantupdate = False
 160         buildfails = False
 161         for apk in apks:
 162             if apk['packageName'] == appid:
 163                 if str(apk['versionCode']) == app.CurrentVersionCode:
 164                     gotcurrentver = True
 165                 apklist.append(apk)
 166         # Include ones we can't build, as a special case...
 167         for build in app.builds:
 168             if build.disable:
 169                 if build.versionCode == app.CurrentVersionCode:
 170                     cantupdate = True
 171                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 172                 apklist.append({'versionCode': int(build.versionCode),
 173                                 'versionName': build.versionName,
 174                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 175                                 })
 176             else:
 177                 builtit = False
 178                 for apk in apklist:
 179                     if apk['versionCode'] == int(build.versionCode):
 180                         builtit = True
 181                         break
 182                 if not builtit:
 183                     buildfails = True
 184                     apklist.append({'versionCode': int(build.versionCode),
 185                                     'versionName': build.versionName,
 186                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
 187                                     })
 188         if app.CurrentVersionCode == '0':
 189             cantupdate = True
 190         # Sort with most recent first...
 191         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 192
 193         wikidata += "=Versions=\n"
 194         if len(apklist) == 0:
 195             wikidata += "We currently have no versions of this app available."
 196         elif not gotcurrentver:
 197             wikidata += "We don't have the current version of this app."
 198         else:
 199             wikidata += "We have the current version of this app."
 200         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 201         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 202         if len(app.NoSourceSince) > 0:
 203             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 204         if len(app.CurrentVersion) > 0:
 205             wikidata += "The current (recommended) version is " + app.CurrentVersion
 206             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 207         validapks = 0
 208         for apk in apklist:
 209             wikidata += "==" + apk['versionName'] + "==\n"
 210
 211             if 'buildproblem' in apk:
 212                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 213             else:
 214                 validapks += 1
 215                 wikidata += "This version is built and signed by "
 216                 if 'srcname' in apk:
 217                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 218                 else:
 219                     wikidata += "the original developer.\n\n"
 220             wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 221
 222         wikidata += '\n[[Category:' + wikicat + ']]\n'
 223         if len(app.NoSourceSince) > 0:
 224             wikidata += '\n[[Category:Apps missing source code]]\n'
 225         if validapks == 0 and not app.Disabled:
 226             wikidata += '\n[[Category:Apps with no packages]]\n'
 227         if cantupdate and not app.Disabled:
 228             wikidata += "\n[[Category:Apps we cannot update]]\n"
 229         if buildfails and not app.Disabled:
 230             wikidata += "\n[[Category:Apps with failing builds]]\n"
 231         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 232             wikidata += '\n[[Category:Apps to Update]]\n'
 233         if app.Disabled:
 234             wikidata += '\n[[Category:Apps that are disabled]]\n'
 235         if app.UpdateCheckMode == 'None' and not app.Disabled:
 236             wikidata += '\n[[Category:Apps with no update check]]\n'
 237         for appcat in app.Categories:
 238             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 239
 240         # We can't have underscores in the page name, even if they're in
 241         # the package ID, because MediaWiki messes with them...
 242         pagename = appid.replace('_', ' ')
 243
 244         # Drop a trailing newline, because mediawiki is going to drop it anyway
 245         # and it we don't we'll think the page has changed when it hasn't...
 246         if wikidata.endswith('\n'):
 247             wikidata = wikidata[:-1]
 248
 249         generated_pages[pagename] = wikidata
 250
 251         # Make a redirect from the name to the ID too, unless there's
 252         # already an existing page with the name and it isn't a redirect.
 253         noclobber = False
 254         apppagename = app.Name.replace('_', ' ')
 255         apppagename = apppagename.replace('{', '')
 256         apppagename = apppagename.replace('}', ' ')
 257         apppagename = apppagename.replace(':', ' ')
 258         apppagename = apppagename.replace('[', ' ')
 259         apppagename = apppagename.replace(']', ' ')
 260         # Drop double spaces caused mostly by replacing ':' above
 261         apppagename = apppagename.replace('  ', ' ')
 262         for expagename in site.allpages(prefix=apppagename,
 263                                         filterredir='nonredirects',
 264                                         generator=False):
 265             if expagename == apppagename:
 266                 noclobber = True
 267         # Another reason not to make the redirect page is if the app name
 268         # is the same as it's ID, because that will overwrite the real page
 269         # with an redirect to itself! (Although it seems like an odd
 270         # scenario this happens a lot, e.g. where there is metadata but no
 271         # builds or binaries to extract a name from.
 272         if apppagename == pagename:
 273             noclobber = True
 274         if not noclobber:
 275             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 276
 277     for tcat, genp in [(wikicat, generated_pages),
 278                        (wikiredircat, generated_redirects)]:
 279         catpages = site.Pages['Category:' + tcat]
 280         existingpages = []
 281         for page in catpages:
 282             existingpages.append(page.name)
 283             if page.name in genp:
 284                 pagetxt = page.edit()
 285                 if pagetxt != genp[page.name]:
 286                     logging.debug("Updating modified page " + page.name)
 287                     page.save(genp[page.name], summary='Auto-updated')
 288                 else:
 289                     logging.debug("Page " + page.name + " is unchanged")
 290             else:
 291                 logging.warn("Deleting page " + page.name)
 292                 page.delete('No longer published')
 293         for pagename, text in genp.items():
 294             logging.debug("Checking " + pagename)
 295             if pagename not in existingpages:
 296                 logging.debug("Creating page " + pagename)
 297                 try:
 298                     newpage = site.Pages[pagename]
 299                     newpage.save(text, summary='Auto-created')
 300                 except Exception as e:
 301                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 302
 303     # Purge server cache to ensure counts are up to date
 304     site.pages['Repository Maintenance'].purge()
 305
 306
 307 def delete_disabled_builds(apps, apkcache, repodirs):
 308     """Delete disabled build outputs.
 309
 310     :param apps: list of all applications, as per metadata.read_metadata
 311     :param apkcache: current apk cache information
 312     :param repodirs: the repo directories to process
 313     """
 314     for appid, app in apps.items():
 315         for build in app['builds']:
 316             if not build.disable:
 317                 continue
 318             apkfilename = appid + '_' + str(build.versionCode) + '.apk'
 319             iconfilename = "%s.%s.png" % (
 320                 appid,
 321                 build.versionCode)
 322             for repodir in repodirs:
 323                 files = [
 324                     os.path.join(repodir, apkfilename),
 325                     os.path.join(repodir, apkfilename + '.asc'),
 326                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 327                 ]
 328                 for density in all_screen_densities:
 329                     repo_dir = get_icon_dir(repodir, density)
 330                     files.append(os.path.join(repo_dir, iconfilename))
 331
 332                 for f in files:
 333                     if os.path.exists(f):
 334                         logging.info("Deleting disabled build output " + f)
 335                         os.remove(f)
 336             if apkfilename in apkcache:
 337                 del apkcache[apkfilename]
 338
 339
 340 def resize_icon(iconpath, density):
 341
 342     if not os.path.isfile(iconpath):
 343         return
 344
 345     fp = None
 346     try:
 347         fp = open(iconpath, 'rb')
 348         im = Image.open(fp)
 349         size = dpi_to_px(density)
 350
 351         if any(length > size for length in im.size):
 352             oldsize = im.size
 353             im.thumbnail((size, size), Image.ANTIALIAS)
 354             logging.debug("%s was too large at %s - new size is %s" % (
 355                 iconpath, oldsize, im.size))
 356             im.save(iconpath, "PNG")
 357
 358     except Exception as e:
 359         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 360
 361     finally:
 362         if fp:
 363             fp.close()
 364
 365
 366 def resize_all_icons(repodirs):
 367     """Resize all icons that exceed the max size
 368
 369     :param repodirs: the repo directories to process
 370     """
 371     for repodir in repodirs:
 372         for density in screen_densities:
 373             icon_dir = get_icon_dir(repodir, density)
 374             icon_glob = os.path.join(icon_dir, '*.png')
 375             for iconpath in glob.glob(icon_glob):
 376                 resize_icon(iconpath, density)
 377
 378
 379 def getsig(apkpath):
 380     """ Get the signing certificate of an apk. To get the same md5 has that
 381     Android gets, we encode the .RSA certificate in a specific format and pass
 382     it hex-encoded to the md5 digest algorithm.
 383
 384     :param apkpath: path to the apk
 385     :returns: A string containing the md5 of the signature of the apk or None
 386               if an error occurred.
 387     """
 388
 389     # verify the jar signature is correct
 390     if not common.verify_apk_signature(apkpath):
 391         return None
 392
 393     with zipfile.ZipFile(apkpath, 'r') as apk:
 394         certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 395
 396         if len(certs) < 1:
 397             logging.error("Found no signing certificates on %s" % apkpath)
 398             return None
 399         if len(certs) > 1:
 400             logging.error("Found multiple signing certificates on %s" % apkpath)
 401             return None
 402
 403         cert = apk.read(certs[0])
 404
 405     cert_encoded = common.get_certificate(cert)
 406
 407     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 408
 409
 410 def get_cache_file():
 411     return os.path.join('tmp', 'apkcache')
 412
 413
 414 def get_cache():
 415     """
 416     Gather information about all the apk files in the repo directory,
 417     using cached data if possible.
 418     :return: apkcache
 419     """
 420     apkcachefile = get_cache_file()
 421     if not options.clean and os.path.exists(apkcachefile):
 422         with open(apkcachefile, 'rb') as cf:
 423             apkcache = pickle.load(cf, encoding='utf-8')
 424         if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
 425             apkcache = {}
 426     else:
 427         apkcache = {}
 428
 429     return apkcache
 430
 431
 432 def write_cache(apkcache):
 433     apkcachefile = get_cache_file()
 434     cache_path = os.path.dirname(apkcachefile)
 435     if not os.path.exists(cache_path):
 436         os.makedirs(cache_path)
 437     apkcache["METADATA_VERSION"] = METADATA_VERSION
 438     with open(apkcachefile, 'wb') as cf:
 439         pickle.dump(apkcache, cf)
 440
 441
 442 def get_icon_bytes(apkzip, iconsrc):
 443     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 444     try:
 445         return apkzip.read(iconsrc)
 446     except KeyError:
 447         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 448
 449
 450 def sha256sum(filename):
 451     '''Calculate the sha256 of the given file'''
 452     sha = hashlib.sha256()
 453     with open(filename, 'rb') as f:
 454         while True:
 455             t = f.read(16384)
 456             if len(t) == 0:
 457                 break
 458             sha.update(t)
 459     return sha.hexdigest()
 460
 461
 462 def has_old_openssl(filename):
 463     '''checks for known vulnerable openssl versions in the APK'''
 464
 465     # statically load this pattern
 466     if not hasattr(has_old_openssl, "pattern"):
 467         has_old_openssl.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 468
 469     with zipfile.ZipFile(filename) as zf:
 470         for name in zf.namelist():
 471             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
 472                 lib = zf.open(name)
 473                 while True:
 474                     chunk = lib.read(4096)
 475                     if chunk == b'':
 476                         break
 477                     m = has_old_openssl.pattern.search(chunk)
 478                     if m:
 479                         version = m.group(1).decode('ascii')
 480                         if version.startswith('1.0.1') and version[5] >= 'r' \
 481                            or version.startswith('1.0.2') and version[5] >= 'f':
 482                             logging.debug('"%s" contains recent %s (%s)', filename, name, version)
 483                         else:
 484                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
 485                             return True
 486                         break
 487     return False
 488
 489
 490 def insert_obbs(repodir, apps, apks):
 491     """Scans the .obb files in a given repo directory and adds them to the
 492     relevant APK instances.  OBB files have versionCodes like APK
 493     files, and they are loosely associated.  If there is an OBB file
 494     present, then any APK with the same or higher versionCode will use
 495     that OBB file.  There are two OBB types: main and patch, each APK
 496     can only have only have one of each.
 497
 498     https://developer.android.com/google/play/expansion-files.html
 499
 500     :param repodir: repo directory to scan
 501     :param apps: list of current, valid apps
 502     :param apks: current information on all APKs
 503
 504     """
 505
 506     def obbWarnDelete(f, msg):
 507         logging.warning(msg + f)
 508         if options.delete_unknown:
 509             logging.error("Deleting unknown file: " + f)
 510             os.remove(f)
 511
 512     obbs = []
 513     java_Integer_MIN_VALUE = -pow(2, 31)
 514     currentPackageNames = apps.keys()
 515     for f in glob.glob(os.path.join(repodir, '*.obb')):
 516         obbfile = os.path.basename(f)
 517         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 518         chunks = obbfile.split('.')
 519         if chunks[0] != 'main' and chunks[0] != 'patch':
 520             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 521             continue
 522         if not re.match(r'^-?[0-9]+$', chunks[1]):
 523             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 524             continue
 525         versionCode = int(chunks[1])
 526         packagename = ".".join(chunks[2:-1])
 527
 528         highestVersionCode = java_Integer_MIN_VALUE
 529         if packagename not in currentPackageNames:
 530             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 531             continue
 532         for apk in apks:
 533             if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
 534                 highestVersionCode = apk['versionCode']
 535         if versionCode > highestVersionCode:
 536             obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
 537                           + ') than any APK: ')
 538             continue
 539         obbsha256 = sha256sum(f)
 540         obbs.append((packagename, versionCode, obbfile, obbsha256))
 541
 542     for apk in apks:
 543         for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 544             if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
 545                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 546                     apk['obbMainFile'] = obbfile
 547                     apk['obbMainFileSha256'] = obbsha256
 548                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 549                     apk['obbPatchFile'] = obbfile
 550                     apk['obbPatchFileSha256'] = obbsha256
 551             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 552                 break
 553
 554
 555 def insert_graphics(repodir, apps):
 556     """Scans for screenshot PNG files in statically defined screenshots
 557     directory and adds them to the app metadata.  The screenshots and
 558     graphic must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
 559     and must be in the following layout:
 560
 561     repo/packageName/locale/featureGraphic.png
 562     repo/packageName/locale/phoneScreenshots/1.png
 563     repo/packageName/locale/phoneScreenshots/2.png
 564
 565     Where "packageName" is the app's packageName and "locale" is the locale
 566     of the graphics, e.g. what language they are in, using the IETF RFC5646
 567     format (en-US, fr-CA, es-MX, etc).  This is following this pattern:
 568     https://github.com/fastlane/fastlane/blob/1.109.0/supply/README.md#images-and-screenshots
 569
 570     This will also scan the metadata/ folder and the apps' source repos
 571     for standard locations of graphic and screenshot files.  If it finds
 572     them, it will copy them into the repo.
 573
 574     :param repodir: repo directory to scan
 575
 576     """
 577
 578     allowed_extensions = ('png', 'jpg', 'jpeg')
 579     graphicnames = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
 580     screenshotdirs = ('phoneScreenshots', 'sevenInchScreenshots',
 581                       'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
 582
 583     sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z][A-Z-.@]*'))
 584     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*'))
 585
 586     for d in sorted(sourcedirs):
 587         if not os.path.isdir(d):
 588             continue
 589         for root, dirs, files in os.walk(d):
 590             segments = root.split('/')
 591             destdir = os.path.join('repo', segments[1], segments[-1])  # repo/packageName/locale
 592             for f in files:
 593                 base, extension = common.get_extension(f)
 594                 if base in graphicnames and extension in allowed_extensions:
 595                     os.makedirs(destdir, mode=0o755, exist_ok=True)
 596                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
 597                     shutil.copy(os.path.join(root, f), destdir)
 598             for d in dirs:
 599                 if d in screenshotdirs:
 600                     for f in glob.glob(os.path.join(root, d, '*.*')):
 601                         _, extension = common.get_extension(f)
 602                         if extension in allowed_extensions:
 603                             screenshotdestdir = os.path.join(destdir, d)
 604                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
 605                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
 606                             shutil.copy(f, screenshotdestdir)
 607
 608     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
 609     for d in repofiles:
 610         if not os.path.isdir(d):
 611             continue
 612         for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
 613             if not os.path.isfile(f):
 614                 continue
 615             segments = f.split('/')
 616             packageName = segments[1]
 617             locale = segments[2]
 618             screenshotdir = segments[3]
 619             filename = os.path.basename(f)
 620             base, extension = common.get_extension(filename)
 621
 622             if packageName not in apps:
 623                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
 624                                 % (filename, packageName))
 625                 continue
 626             if 'localized' not in apps[packageName]:
 627                 apps[packageName]['localized'] = collections.OrderedDict()
 628             if locale not in apps[packageName]['localized']:
 629                 apps[packageName]['localized'][locale] = collections.OrderedDict()
 630             graphics = apps[packageName]['localized'][locale]
 631
 632             if extension not in allowed_extensions:
 633                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
 634             elif base in graphicnames:
 635                 # there can only be zero or one of these per locale
 636                 graphics[base] = filename
 637             elif screenshotdir in screenshotdirs:
 638                 # there can any number of these per locale
 639                 logging.debug('adding ' + base + ':' + f)
 640                 if screenshotdir not in graphics:
 641                     graphics[screenshotdir] = []
 642                 graphics[screenshotdir].append(filename)
 643             else:
 644                 logging.warning('Unsupported graphics file found: ' + f)
 645
 646
 647 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 648     """Scan a repo for all files with an extension except APK/OBB
 649
 650     :param apkcache: current cached info about all repo files
 651     :param repodir: repo directory to scan
 652     :param knownapks: list of all known files, as per metadata.read_metadata
 653     :param use_date_from_file: use date from file (instead of current date)
 654                                for newly added files
 655     """
 656
 657     cachechanged = False
 658     repo_files = []
 659     for name in os.listdir(repodir):
 660         file_extension = common.get_file_extension(name)
 661         if file_extension == 'apk' or file_extension == 'obb':
 662             continue
 663         filename = os.path.join(repodir, name)
 664         if filename.endswith('_src.tar.gz'):
 665             logging.debug('skipping source tarball: ' + filename)
 666             continue
 667         if not common.is_repo_file(filename):
 668             continue
 669         stat = os.stat(filename)
 670         if stat.st_size == 0:
 671             logging.error(filename + ' is zero size!')
 672             sys.exit(1)
 673
 674         shasum = sha256sum(filename)
 675         usecache = False
 676         if name in apkcache:
 677             repo_file = apkcache[name]
 678             # added time is cached as tuple but used here as datetime instance
 679             if 'added' in repo_file:
 680                 a = repo_file['added']
 681                 if isinstance(a, datetime):
 682                     repo_file['added'] = a
 683                 else:
 684                     repo_file['added'] = datetime(*a[:6])
 685             if repo_file['hash'] == shasum:
 686                 logging.debug("Reading " + name + " from cache")
 687                 usecache = True
 688             else:
 689                 logging.debug("Ignoring stale cache data for " + name)
 690
 691         if not usecache:
 692             logging.debug("Processing " + name)
 693             repo_file = {}
 694             # TODO rename apkname globally to something more generic
 695             repo_file['name'] = name
 696             repo_file['apkName'] = name
 697             repo_file['hash'] = shasum
 698             repo_file['hashType'] = 'sha256'
 699             repo_file['versionCode'] = 0
 700             repo_file['versionName'] = shasum
 701             # the static ID is the SHA256 unless it is set in the metadata
 702             repo_file['packageName'] = shasum
 703             n = name.split('_')
 704             if len(n) == 2:
 705                 packageName = n[0]
 706                 versionCode = n[1].split('.')[0]
 707                 if re.match(r'^-?[0-9]+$', versionCode) \
 708                    and common.is_valid_package_name(name.split('_')[0]):
 709                     repo_file['packageName'] = packageName
 710                     repo_file['versionCode'] = int(versionCode)
 711             srcfilename = name + "_src.tar.gz"
 712             if os.path.exists(os.path.join(repodir, srcfilename)):
 713                 repo_file['srcname'] = srcfilename
 714             repo_file['size'] = stat.st_size
 715
 716             apkcache[name] = repo_file
 717             cachechanged = True
 718
 719         if use_date_from_file:
 720             timestamp = stat.st_ctime
 721             default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
 722         else:
 723             default_date_param = None
 724
 725         # Record in knownapks, getting the added date at the same time..
 726         added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
 727                                     default_date=default_date_param)
 728         if added:
 729             repo_file['added'] = added
 730
 731         repo_files.append(repo_file)
 732
 733     return repo_files, cachechanged
 734
 735
 736 def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
 737     """Scan the apk with the given filename in the given repo directory.
 738
 739     This also extracts the icons.
 740
 741     :param apkcache: current apk cache information
 742     :param apkfilename: the filename of the apk to scan
 743     :param repodir: repo directory to scan
 744     :param knownapks: known apks info
 745     :param use_date_from_apk: use date from APK (instead of current date)
 746                               for newly added APKs
 747     :returns: (skip, apk, cachechanged) where skip is a boolean indicating whether to skip this apk,
 748      apk is the scanned apk information, and cachechanged is True if the apkcache got changed.
 749     """
 750
 751     if ' ' in apkfilename:
 752         logging.critical("Spaces in filenames are not allowed.")
 753         sys.exit(1)
 754
 755     apkfile = os.path.join(repodir, apkfilename)
 756     shasum = sha256sum(apkfile)
 757
 758     cachechanged = False
 759     usecache = False
 760     if apkfilename in apkcache:
 761         apk = apkcache[apkfilename]
 762         if apk['hash'] == shasum:
 763             logging.debug("Reading " + apkfilename + " from cache")
 764             usecache = True
 765         else:
 766             logging.debug("Ignoring stale cache data for " + apkfilename)
 767
 768     if not usecache:
 769         logging.debug("Processing " + apkfilename)
 770         apk = {}
 771         apk['apkName'] = apkfilename
 772         apk['hash'] = shasum
 773         apk['hashType'] = 'sha256'
 774         srcfilename = apkfilename[:-4] + "_src.tar.gz"
 775         if os.path.exists(os.path.join(repodir, srcfilename)):
 776             apk['srcname'] = srcfilename
 777         apk['size'] = os.path.getsize(apkfile)
 778         apk['uses-permission'] = set()
 779         apk['uses-permission-sdk-23'] = set()
 780         apk['features'] = set()
 781         apk['icons_src'] = {}
 782         apk['icons'] = {}
 783         apk['antiFeatures'] = set()
 784         if has_old_openssl(apkfile):
 785             apk['antiFeatures'].add('KnownVuln')
 786         p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 787         if p.returncode != 0:
 788             if options.delete_unknown:
 789                 if os.path.exists(apkfile):
 790                     logging.error("Failed to get apk information, deleting " + apkfile)
 791                     os.remove(apkfile)
 792                 else:
 793                     logging.error("Could not find {0} to remove it".format(apkfile))
 794             else:
 795                 logging.error("Failed to get apk information, skipping " + apkfile)
 796             return True
 797         for line in p.output.splitlines():
 798             if line.startswith("package:"):
 799                 try:
 800                     apk['packageName'] = re.match(APK_NAME_PAT, line).group(1)
 801                     apk['versionCode'] = int(re.match(APK_VERCODE_PAT, line).group(1))
 802                     apk['versionName'] = re.match(APK_VERNAME_PAT, line).group(1)
 803                 except Exception as e:
 804                     logging.error("Package matching failed: " + str(e))
 805                     logging.info("Line was: " + line)
 806                     sys.exit(1)
 807             elif line.startswith("application:"):
 808                 apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
 809                 # Keep path to non-dpi icon in case we need it
 810                 match = re.match(APK_ICON_PAT_NODPI, line)
 811                 if match:
 812                     apk['icons_src']['-1'] = match.group(1)
 813             elif line.startswith("launchable-activity:"):
 814                 # Only use launchable-activity as fallback to application
 815                 if not apk['name']:
 816                     apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
 817                 if '-1' not in apk['icons_src']:
 818                     match = re.match(APK_ICON_PAT_NODPI, line)
 819                     if match:
 820                         apk['icons_src']['-1'] = match.group(1)
 821             elif line.startswith("application-icon-"):
 822                 match = re.match(APK_ICON_PAT, line)
 823                 if match:
 824                     density = match.group(1)
 825                     path = match.group(2)
 826                     apk['icons_src'][density] = path
 827             elif line.startswith("sdkVersion:"):
 828                 m = re.match(APK_SDK_VERSION_PAT, line)
 829                 if m is None:
 830                     logging.error(line.replace('sdkVersion:', '')
 831                                   + ' is not a valid minSdkVersion!')
 832                 else:
 833                     apk['minSdkVersion'] = m.group(1)
 834                     # if target not set, default to min
 835                     if 'targetSdkVersion' not in apk:
 836                         apk['targetSdkVersion'] = m.group(1)
 837             elif line.startswith("targetSdkVersion:"):
 838                 m = re.match(APK_SDK_VERSION_PAT, line)
 839                 if m is None:
 840                     logging.error(line.replace('targetSdkVersion:', '')
 841                                   + ' is not a valid targetSdkVersion!')
 842                 else:
 843                     apk['targetSdkVersion'] = m.group(1)
 844             elif line.startswith("maxSdkVersion:"):
 845                 apk['maxSdkVersion'] = re.match(APK_SDK_VERSION_PAT, line).group(1)
 846             elif line.startswith("native-code:"):
 847                 apk['nativecode'] = []
 848                 for arch in line[13:].split(' '):
 849                     apk['nativecode'].append(arch[1:-1])
 850             elif line.startswith('uses-permission:'):
 851                 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
 852                 if perm_match['maxSdkVersion']:
 853                     perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
 854                 permission = UsesPermission(
 855                     perm_match['name'],
 856                     perm_match['maxSdkVersion']
 857                 )
 858
 859                 apk['uses-permission'].add(permission)
 860             elif line.startswith('uses-permission-sdk-23:'):
 861                 perm_match = re.match(APK_PERMISSION_PAT, line).groupdict()
 862                 if perm_match['maxSdkVersion']:
 863                     perm_match['maxSdkVersion'] = int(perm_match['maxSdkVersion'])
 864                 permission_sdk_23 = UsesPermissionSdk23(
 865                     perm_match['name'],
 866                     perm_match['maxSdkVersion']
 867                 )
 868
 869                 apk['uses-permission-sdk-23'].add(permission_sdk_23)
 870
 871             elif line.startswith('uses-feature:'):
 872                 feature = re.match(APK_FEATURE_PAT, line).group(1)
 873                 # Filter out this, it's only added with the latest SDK tools and
 874                 # causes problems for lots of apps.
 875                 if feature != "android.hardware.screen.portrait" \
 876                         and feature != "android.hardware.screen.landscape":
 877                     if feature.startswith("android.feature."):
 878                         feature = feature[16:]
 879                     apk['features'].add(feature)
 880
 881         if 'minSdkVersion' not in apk:
 882             logging.warn("No SDK version information found in {0}".format(apkfile))
 883             apk['minSdkVersion'] = 1
 884
 885         # Check for debuggable apks...
 886         if common.isApkAndDebuggable(apkfile, config):
 887             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
 888
 889         # Get the signature (or md5 of, to be precise)...
 890         logging.debug('Getting signature of {0}'.format(apkfile))
 891         apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 892         if not apk['sig']:
 893             logging.critical("Failed to get apk signature")
 894             sys.exit(1)
 895
 896         apkzip = zipfile.ZipFile(apkfile, 'r')
 897
 898         # if an APK has files newer than the system time, suggest updating
 899         # the system clock.  This is useful for offline systems, used for
 900         # signing, which do not have another source of clock sync info. It
 901         # has to be more than 24 hours newer because ZIP/APK files do not
 902         # store timezone info
 903         manifest = apkzip.getinfo('AndroidManifest.xml')
 904         if manifest.date_time[1] == 0:  # month can't be zero
 905             logging.debug('AndroidManifest.xml has no date')
 906         else:
 907             dt_obj = datetime(*manifest.date_time)
 908             checkdt = dt_obj - timedelta(1)
 909             if datetime.today() < checkdt:
 910                 logging.warn('System clock is older than manifest in: '
 911                              + apkfilename
 912                              + '\nSet clock to that time using:\n'
 913                              + 'sudo date -s "' + str(dt_obj) + '"')
 914
 915         iconfilename = "%s.%s.png" % (
 916             apk['packageName'],
 917             apk['versionCode'])
 918
 919         # Extract the icon file...
 920         empty_densities = []
 921         for density in screen_densities:
 922             if density not in apk['icons_src']:
 923                 empty_densities.append(density)
 924                 continue
 925             iconsrc = apk['icons_src'][density]
 926             icon_dir = get_icon_dir(repodir, density)
 927             icondest = os.path.join(icon_dir, iconfilename)
 928
 929             try:
 930                 with open(icondest, 'wb') as f:
 931                     f.write(get_icon_bytes(apkzip, iconsrc))
 932                 apk['icons'][density] = iconfilename
 933
 934             except Exception as e:
 935                 logging.warn("Error retrieving icon file: %s" % (e))
 936                 del apk['icons'][density]
 937                 del apk['icons_src'][density]
 938                 empty_densities.append(density)
 939
 940         if '-1' in apk['icons_src']:
 941             iconsrc = apk['icons_src']['-1']
 942             iconpath = os.path.join(
 943                 get_icon_dir(repodir, '0'), iconfilename)
 944             with open(iconpath, 'wb') as f:
 945                 f.write(get_icon_bytes(apkzip, iconsrc))
 946             try:
 947                 im = Image.open(iconpath)
 948                 dpi = px_to_dpi(im.size[0])
 949                 for density in screen_densities:
 950                     if density in apk['icons']:
 951                         break
 952                     if density == screen_densities[-1] or dpi >= int(density):
 953                         apk['icons'][density] = iconfilename
 954                         shutil.move(iconpath,
 955                                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 956                         empty_densities.remove(density)
 957                         break
 958             except Exception as e:
 959                 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 960
 961         if apk['icons']:
 962             apk['icon'] = iconfilename
 963
 964         apkzip.close()
 965
 966         # First try resizing down to not lose quality
 967         last_density = None
 968         for density in screen_densities:
 969             if density not in empty_densities:
 970                 last_density = density
 971                 continue
 972             if last_density is None:
 973                 continue
 974             logging.debug("Density %s not available, resizing down from %s"
 975                           % (density, last_density))
 976
 977             last_iconpath = os.path.join(
 978                 get_icon_dir(repodir, last_density), iconfilename)
 979             iconpath = os.path.join(
 980                 get_icon_dir(repodir, density), iconfilename)
 981             fp = None
 982             try:
 983                 fp = open(last_iconpath, 'rb')
 984                 im = Image.open(fp)
 985
 986                 size = dpi_to_px(density)
 987
 988                 im.thumbnail((size, size), Image.ANTIALIAS)
 989                 im.save(iconpath, "PNG")
 990                 empty_densities.remove(density)
 991             except Exception as e:
 992                 logging.warning("Invalid image file at %s: %s" % (last_iconpath, e))
 993             finally:
 994                 if fp:
 995                     fp.close()
 996
 997         # Then just copy from the highest resolution available
 998         last_density = None
 999         for density in reversed(screen_densities):
1000             if density not in empty_densities:
1001                 last_density = density
1002                 continue
1003             if last_density is None:
1004                 continue
1005             logging.debug("Density %s not available, copying from lower density %s"
1006                           % (density, last_density))
1007
1008             shutil.copyfile(
1009                 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
1010                 os.path.join(get_icon_dir(repodir, density), iconfilename))
1011
1012             empty_densities.remove(density)
1013
1014         for density in screen_densities:
1015             icon_dir = get_icon_dir(repodir, density)
1016             icondest = os.path.join(icon_dir, iconfilename)
1017             resize_icon(icondest, density)
1018
1019         # Copy from icons-mdpi to icons since mdpi is the baseline density
1020         baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
1021         if os.path.isfile(baseline):
1022             apk['icons']['0'] = iconfilename
1023             shutil.copyfile(baseline,
1024                             os.path.join(get_icon_dir(repodir, '0'), iconfilename))
1025
1026         if use_date_from_apk and manifest.date_time[1] != 0:
1027             default_date_param = datetime(*manifest.date_time)
1028         else:
1029             default_date_param = None
1030
1031         # Record in known apks, getting the added date at the same time..
1032         added = knownapks.recordapk(apk['apkName'], apk['packageName'],
1033                                     default_date=default_date_param)
1034         if added:
1035             apk['added'] = added
1036
1037         apkcache[apkfilename] = apk
1038         cachechanged = True
1039
1040     return False, apk, cachechanged
1041
1042
1043 def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
1044     """Scan the apks in the given repo directory.
1045
1046     This also extracts the icons.
1047
1048     :param apkcache: current apk cache information
1049     :param repodir: repo directory to scan
1050     :param knownapks: known apks info
1051     :param use_date_from_apk: use date from APK (instead of current date)
1052                               for newly added APKs
1053     :returns: (apks, cachechanged) where apks is a list of apk information,
1054               and cachechanged is True if the apkcache got changed.
1055     """
1056
1057     cachechanged = False
1058
1059     for icon_dir in get_all_icon_dirs(repodir):
1060         if os.path.exists(icon_dir):
1061             if options.clean:
1062                 shutil.rmtree(icon_dir)
1063                 os.makedirs(icon_dir)
1064         else:
1065             os.makedirs(icon_dir)
1066
1067     apks = []
1068     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
1069         apkfilename = apkfile[len(repodir) + 1:]
1070         (skip, apk, cachechanged) = scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk)
1071         if skip:
1072             continue
1073         apks.append(apk)
1074
1075     return apks, cachechanged
1076
1077
1078 def apply_info_from_latest_apk(apps, apks):
1079     """
1080     Some information from the apks needs to be applied up to the application level.
1081     When doing this, we use the info from the most recent version's apk.
1082     We deal with figuring out when the app was added and last updated at the same time.
1083     """
1084     for appid, app in apps.items():
1085         bestver = UNSET_VERSION_CODE
1086         for apk in apks:
1087             if apk['packageName'] == appid:
1088                 if apk['versionCode'] > bestver:
1089                     bestver = apk['versionCode']
1090                     bestapk = apk
1091
1092                 if 'added' in apk:
1093                     if not app.added or apk['added'] < app.added:
1094                         app.added = apk['added']
1095                     if not app.lastUpdated or apk['added'] > app.lastUpdated:
1096                         app.lastUpdated = apk['added']
1097
1098         if not app.added:
1099             logging.debug("Don't know when " + appid + " was added")
1100         if not app.lastUpdated:
1101             logging.debug("Don't know when " + appid + " was last updated")
1102
1103         if bestver == UNSET_VERSION_CODE:
1104
1105             if app.Name is None:
1106                 app.Name = app.AutoName or appid
1107             app.icon = None
1108             logging.debug("Application " + appid + " has no packages")
1109         else:
1110             if app.Name is None:
1111                 app.Name = bestapk['name']
1112             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1113             if app.CurrentVersionCode is None:
1114                 app.CurrentVersionCode = str(bestver)
1115
1116
1117 def make_categories_txt(repodir, categories):
1118     '''Write a category list in the repo to allow quick access'''
1119     catdata = ''
1120     for cat in sorted(categories):
1121         catdata += cat + '\n'
1122     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1123         f.write(catdata)
1124
1125
1126 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1127
1128     for appid, app in apps.items():
1129
1130         if app.ArchivePolicy:
1131             keepversions = int(app.ArchivePolicy[:-9])
1132         else:
1133             keepversions = defaultkeepversions
1134
1135         def filter_apk_list_sorted(apk_list):
1136             res = []
1137             for apk in apk_list:
1138                 if apk['packageName'] == appid:
1139                     res.append(apk)
1140
1141             # Sort the apk list by version code. First is highest/newest.
1142             return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
1143
1144         def move_file(from_dir, to_dir, filename, ignore_missing):
1145             from_path = os.path.join(from_dir, filename)
1146             if ignore_missing and not os.path.exists(from_path):
1147                 return
1148             to_path = os.path.join(to_dir, filename)
1149             shutil.move(from_path, to_path)
1150
1151         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1152                       .format(appid, len(apks), keepversions, len(archapks)))
1153
1154         if len(apks) > keepversions:
1155             apklist = filter_apk_list_sorted(apks)
1156             # Move back the ones we don't want.
1157             for apk in apklist[keepversions:]:
1158                 logging.info("Moving " + apk['apkName'] + " to archive")
1159                 move_file(repodir, archivedir, apk['apkName'], False)
1160                 move_file(repodir, archivedir, apk['apkName'] + '.asc', True)
1161                 for density in all_screen_densities:
1162                     repo_icon_dir = get_icon_dir(repodir, density)
1163                     archive_icon_dir = get_icon_dir(archivedir, density)
1164                     if density not in apk['icons']:
1165                         continue
1166                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1167                 if 'srcname' in apk:
1168                     move_file(repodir, archivedir, apk['srcname'], False)
1169                 archapks.append(apk)
1170                 apks.remove(apk)
1171         elif len(apks) < keepversions and len(archapks) > 0:
1172             required = keepversions - len(apks)
1173             archapklist = filter_apk_list_sorted(archapks)
1174             # Move forward the ones we want again.
1175             for apk in archapklist[:required]:
1176                 logging.info("Moving " + apk['apkName'] + " from archive")
1177                 move_file(archivedir, repodir, apk['apkName'], False)
1178                 move_file(archivedir, repodir, apk['apkName'] + '.asc', True)
1179                 for density in all_screen_densities:
1180                     repo_icon_dir = get_icon_dir(repodir, density)
1181                     archive_icon_dir = get_icon_dir(archivedir, density)
1182                     if density not in apk['icons']:
1183                         continue
1184                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1185                 if 'srcname' in apk:
1186                     move_file(archivedir, repodir, apk['srcname'], False)
1187                 archapks.remove(apk)
1188                 apks.append(apk)
1189
1190
1191 def add_apks_to_per_app_repos(repodir, apks):
1192     apks_per_app = dict()
1193     for apk in apks:
1194         apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
1195         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1196         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1197         apks_per_app[apk['packageName']] = apk
1198
1199         if not os.path.exists(apk['per_app_icons']):
1200             logging.info('Adding new repo for only ' + apk['packageName'])
1201             os.makedirs(apk['per_app_icons'])
1202
1203         apkpath = os.path.join(repodir, apk['apkName'])
1204         shutil.copy(apkpath, apk['per_app_repo'])
1205         apksigpath = apkpath + '.sig'
1206         if os.path.exists(apksigpath):
1207             shutil.copy(apksigpath, apk['per_app_repo'])
1208         apkascpath = apkpath + '.asc'
1209         if os.path.exists(apkascpath):
1210             shutil.copy(apkascpath, apk['per_app_repo'])
1211
1212
1213 def make_binary_transparency_log(repodirs):
1214     '''Log the indexes in a standalone git repo to serve as a "binary
1215     transparency" log.
1216
1217     see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies
1218
1219     '''
1220
1221     import git
1222     btrepo = 'binary_transparency'
1223     if os.path.exists(os.path.join(btrepo, '.git')):
1224         gitrepo = git.Repo(btrepo)
1225     else:
1226         if not os.path.exists(btrepo):
1227             os.mkdir(btrepo)
1228         gitrepo = git.Repo.init(btrepo)
1229
1230         gitconfig = gitrepo.config_writer()
1231         gitconfig.set_value('user', 'name', 'fdroid update')
1232         gitconfig.set_value('user', 'email', 'fdroid@' + platform.node())
1233
1234         url = config['repo_url'].rstrip('/')
1235         with open(os.path.join(btrepo, 'README.md'), 'w') as fp:
1236             fp.write("""
1237 # Binary Transparency Log for %s
1238
1239 """ % url[:url.rindex('/')])  # strip '/repo'
1240         gitrepo.index.add(['README.md', ])
1241         gitrepo.index.commit('add README')
1242
1243     for repodir in repodirs:
1244         cpdir = os.path.join(btrepo, repodir)
1245         if not os.path.exists(cpdir):
1246             os.mkdir(cpdir)
1247         for f in ('index.xml', 'index-v1.json'):
1248             dest = os.path.join(cpdir, f)
1249             shutil.copyfile(os.path.join(repodir, f), dest)
1250             gitrepo.index.add([os.path.join(repodir, f), ])
1251         for f in ('index.jar', 'index-v1.jar'):
1252             repof = os.path.join(repodir, f)
1253             dest = os.path.join(cpdir, f)
1254             jarin = zipfile.ZipFile(repof, 'r')
1255             jarout = zipfile.ZipFile(dest, 'w')
1256             for info in jarin.infolist():
1257                 if info.filename.startswith('META-INF/'):
1258                     jarout.writestr(info, jarin.read(info.filename))
1259             jarout.close()
1260             jarin.close()
1261             gitrepo.index.add([repof, ])
1262
1263         files = []
1264         for root, dirs, filenames in os.walk(repodir):
1265             for f in filenames:
1266                 files.append(os.path.relpath(os.path.join(root, f), repodir))
1267         output = collections.OrderedDict()
1268         for f in sorted(files):
1269             repofile = os.path.join(repodir, f)
1270             stat = os.stat(repofile)
1271             output[f] = (
1272                 stat.st_size,
1273                 stat.st_ctime_ns,
1274                 stat.st_mtime_ns,
1275                 stat.st_mode,
1276                 stat.st_uid,
1277                 stat.st_gid,
1278             )
1279         fslogfile = os.path.join(cpdir, 'filesystemlog.json')
1280         with open(fslogfile, 'w') as fp:
1281             json.dump(output, fp, indent=2)
1282         gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ])
1283
1284     gitrepo.index.commit('fdroid update')
1285
1286
1287 config = None
1288 options = None
1289
1290
1291 def main():
1292
1293     global config, options
1294
1295     # Parse command line...
1296     parser = ArgumentParser()
1297     common.setup_global_opts(parser)
1298     parser.add_argument("--create-key", action="store_true", default=False,
1299                         help="Create a repo signing key in a keystore")
1300     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1301                         help="Create skeleton metadata files that are missing")
1302     parser.add_argument("--delete-unknown", action="store_true", default=False,
1303                         help="Delete APKs and/or OBBs without metadata from the repo")
1304     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1305                         help="Report on build data status")
1306     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1307                         help="Interactively ask about things that need updating.")
1308     parser.add_argument("-I", "--icons", action="store_true", default=False,
1309                         help="Resize all the icons exceeding the max pixel size and exit")
1310     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1311                         help="Specify editor to use in interactive mode. Default " +
1312                         "is /etc/alternatives/editor")
1313     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1314                         help="Update the wiki")
1315     parser.add_argument("--pretty", action="store_true", default=False,
1316                         help="Produce human-readable index.xml")
1317     parser.add_argument("--clean", action="store_true", default=False,
1318                         help="Clean update - don't uses caches, reprocess all apks")
1319     parser.add_argument("--nosign", action="store_true", default=False,
1320                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1321     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1322                         help="Use date from apk instead of current time for newly added apks")
1323     metadata.add_metadata_arguments(parser)
1324     options = parser.parse_args()
1325     metadata.warnings_action = options.W
1326
1327     config = common.read_config(options)
1328
1329     if not ('jarsigner' in config and 'keytool' in config):
1330         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1331         sys.exit(1)
1332
1333     repodirs = ['repo']
1334     if config['archive_older'] != 0:
1335         repodirs.append('archive')
1336         if not os.path.exists('archive'):
1337             os.mkdir('archive')
1338
1339     if options.icons:
1340         resize_all_icons(repodirs)
1341         sys.exit(0)
1342
1343     # check that icons exist now, rather than fail at the end of `fdroid update`
1344     for k in ['repo_icon', 'archive_icon']:
1345         if k in config:
1346             if not os.path.exists(config[k]):
1347                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1348                 sys.exit(1)
1349
1350     # if the user asks to create a keystore, do it now, reusing whatever it can
1351     if options.create_key:
1352         if os.path.exists(config['keystore']):
1353             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1354             logging.critical("\t'" + config['keystore'] + "'")
1355             sys.exit(1)
1356
1357         if 'repo_keyalias' not in config:
1358             config['repo_keyalias'] = socket.getfqdn()
1359             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1360         if 'keydname' not in config:
1361             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1362             common.write_to_config(config, 'keydname', config['keydname'])
1363         if 'keystore' not in config:
1364             config['keystore'] = common.default_config.keystore
1365             common.write_to_config(config, 'keystore', config['keystore'])
1366
1367         password = common.genpassword()
1368         if 'keystorepass' not in config:
1369             config['keystorepass'] = password
1370             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1371         if 'keypass' not in config:
1372             config['keypass'] = password
1373             common.write_to_config(config, 'keypass', config['keypass'])
1374         common.genkeystore(config)
1375
1376     # Get all apps...
1377     apps = metadata.read_metadata()
1378
1379     # Generate a list of categories...
1380     categories = set()
1381     for app in apps.values():
1382         categories.update(app.Categories)
1383
1384     # Read known apks data (will be updated and written back when we've finished)
1385     knownapks = common.KnownApks()
1386
1387     # Get APK cache
1388     apkcache = get_cache()
1389
1390     # Delete builds for disabled apps
1391     delete_disabled_builds(apps, apkcache, repodirs)
1392
1393     # Scan all apks in the main repo
1394     apks, cachechanged = scan_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1395
1396     files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
1397                                            options.use_date_from_apk)
1398     cachechanged = cachechanged or fcachechanged
1399     apks += files
1400     # Generate warnings for apk's with no metadata (or create skeleton
1401     # metadata files, if requested on the command line)
1402     newmetadata = False
1403     for apk in apks:
1404         if apk['packageName'] not in apps:
1405             if options.create_metadata:
1406                 if 'name' not in apk:
1407                     logging.error(apk['packageName'] + ' does not have a name! Skipping...')
1408                     continue
1409                 f = open(os.path.join('metadata', apk['packageName'] + '.txt'), 'w', encoding='utf8')
1410                 f.write("License:Unknown\n")
1411                 f.write("Web Site:\n")
1412                 f.write("Source Code:\n")
1413                 f.write("Issue Tracker:\n")
1414                 f.write("Changelog:\n")
1415                 f.write("Summary:" + apk['name'] + "\n")
1416                 f.write("Description:\n")
1417                 f.write(apk['name'] + "\n")
1418                 f.write(".\n")
1419                 f.write("Name:" + apk['name'] + "\n")
1420                 f.close()
1421                 logging.info("Generated skeleton metadata for " + apk['packageName'])
1422                 newmetadata = True
1423             else:
1424                 msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
1425                 if options.delete_unknown:
1426                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
1427                     rmf = os.path.join(repodirs[0], apk['apkName'])
1428                     if not os.path.exists(rmf):
1429                         logging.error("Could not find {0} to remove it".format(rmf))
1430                     else:
1431                         os.remove(rmf)
1432                 else:
1433                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1434
1435     # update the metadata with the newly created ones included
1436     if newmetadata:
1437         apps = metadata.read_metadata()
1438
1439     insert_obbs(repodirs[0], apps, apks)
1440     insert_graphics(repodirs[0], apps)
1441
1442     # Scan the archive repo for apks as well
1443     if len(repodirs) > 1:
1444         archapks, cc = scan_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1445         if cc:
1446             cachechanged = True
1447     else:
1448         archapks = []
1449
1450     # Apply information from latest apks to the application and update dates
1451     apply_info_from_latest_apk(apps, apks + archapks)
1452
1453     # Sort the app list by name, then the web site doesn't have to by default.
1454     # (we had to wait until we'd scanned the apks to do this, because mostly the
1455     # name comes from there!)
1456     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1457
1458     # APKs are placed into multiple repos based on the app package, providing
1459     # per-app subscription feeds for nightly builds and things like it
1460     if config['per_app_repos']:
1461         add_apks_to_per_app_repos(repodirs[0], apks)
1462         for appid, app in apps.items():
1463             repodir = os.path.join(appid, 'fdroid', 'repo')
1464             appdict = dict()
1465             appdict[appid] = app
1466             if os.path.isdir(repodir):
1467                 index.make(appdict, [appid], apks, repodir, False)
1468             else:
1469                 logging.info('Skipping index generation for ' + appid)
1470         return
1471
1472     if len(repodirs) > 1:
1473         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1474
1475     # Make the index for the main repo...
1476     index.make(apps, sortedids, apks, repodirs[0], False)
1477     make_categories_txt(repodirs[0], categories)
1478
1479     # If there's an archive repo,  make the index for it. We already scanned it
1480     # earlier on.
1481     if len(repodirs) > 1:
1482         index.make(apps, sortedids, archapks, repodirs[1], True)
1483
1484     if config.get('binary_transparency_remote'):
1485         make_binary_transparency_log(repodirs)
1486
1487     if config['update_stats']:
1488         # Update known apks info...
1489         knownapks.writeifchanged()
1490
1491         # Generate latest apps data for widget
1492         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1493             data = ''
1494             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1495                 for line in f:
1496                     appid = line.rstrip()
1497                     data += appid + "\t"
1498                     app = apps[appid]
1499                     data += app.Name + "\t"
1500                     if app.icon is not None:
1501                         data += app.icon + "\t"
1502                     data += app.License + "\n"
1503             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1504                 f.write(data)
1505
1506     if cachechanged:
1507         write_cache(apkcache)
1508
1509     # Update the wiki...
1510     if options.wiki:
1511         update_wiki(apps, sortedids, apks + archapks)
1512
1513     logging.info("Finished.")
1514
1515
1516 if __name__ == "__main__":
1517     main()