fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   5 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU Affero General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Affero General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Affero General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 import sys
  21 import os
  22 import shutil
  23 import glob
  24 import re
  25 import socket
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 import urllib.parse
  30 from datetime import datetime, timedelta
  31 from xml.dom.minidom import Document
  32 from argparse import ArgumentParser
  33 import time
  34
  35 import collections
  36 from pyasn1.error import PyAsn1Error
  37 from pyasn1.codec.der import decoder, encoder
  38 from pyasn1_modules import rfc2315
  39 from binascii import hexlify, unhexlify
  40
  41 from PIL import Image
  42 import logging
  43
  44 from . import common
  45 from . import metadata
  46 from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
  47 from .metadata import MetaDataException
  48
  49 METADATA_VERSION = 17
  50
  51 screen_densities = ['640', '480', '320', '240', '160', '120']
  52
  53 all_screen_densities = ['0'] + screen_densities
  54
  55 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  56 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  57
  58
  59 def dpi_to_px(density):
  60     return (int(density) * 48) / 160
  61
  62
  63 def px_to_dpi(px):
  64     return (int(px) * 160) / 48
  65
  66
  67 def get_icon_dir(repodir, density):
  68     if density == '0':
  69         return os.path.join(repodir, "icons")
  70     return os.path.join(repodir, "icons-%s" % density)
  71
  72
  73 def get_icon_dirs(repodir):
  74     for density in screen_densities:
  75         yield get_icon_dir(repodir, density)
  76
  77
  78 def get_all_icon_dirs(repodir):
  79     for density in all_screen_densities:
  80         yield get_icon_dir(repodir, density)
  81
  82
  83 def update_wiki(apps, sortedids, apks):
  84     """Update the wiki
  85
  86     :param apps: fully populated list of all applications
  87     :param apks: all apks, except...
  88     """
  89     logging.info("Updating wiki")
  90     wikicat = 'Apps'
  91     wikiredircat = 'App Redirects'
  92     import mwclient
  93     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  94                          path=config['wiki_path'])
  95     site.login(config['wiki_user'], config['wiki_password'])
  96     generated_pages = {}
  97     generated_redirects = {}
  98
  99     for appid in sortedids:
 100         app = apps[appid]
 101
 102         wikidata = ''
 103         if app.Disabled:
 104             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 105         if app.AntiFeatures:
 106             for af in app.AntiFeatures:
 107                 wikidata += '{{AntiFeature|' + af + '}}\n'
 108         if app.RequiresRoot:
 109             requiresroot = 'Yes'
 110         else:
 111             requiresroot = 'No'
 112         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 113             appid,
 114             app.Name,
 115             time.strftime('%Y-%m-%d', app.added) if app.added else '',
 116             time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
 117             app.SourceCode,
 118             app.IssueTracker,
 119             app.WebSite,
 120             app.Changelog,
 121             app.Donate,
 122             app.FlattrID,
 123             app.Bitcoin,
 124             app.Litecoin,
 125             app.License,
 126             requiresroot,
 127             app.AuthorName,
 128             app.AuthorEmail)
 129
 130         if app.Provides:
 131             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 132
 133         wikidata += app.Summary
 134         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 135
 136         wikidata += "=Description=\n"
 137         wikidata += metadata.description_wiki(app.Description) + "\n"
 138
 139         wikidata += "=Maintainer Notes=\n"
 140         if app.MaintainerNotes:
 141             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 142         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 143
 144         # Get a list of all packages for this application...
 145         apklist = []
 146         gotcurrentver = False
 147         cantupdate = False
 148         buildfails = False
 149         for apk in apks:
 150             if apk['id'] == appid:
 151                 if str(apk['versioncode']) == app.CurrentVersionCode:
 152                     gotcurrentver = True
 153                 apklist.append(apk)
 154         # Include ones we can't build, as a special case...
 155         for build in app.builds:
 156             if build.disable:
 157                 if build.vercode == app.CurrentVersionCode:
 158                     cantupdate = True
 159                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 160                 apklist.append({'versioncode': int(build.vercode),
 161                                 'version': build.version,
 162                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 163                                 })
 164             else:
 165                 builtit = False
 166                 for apk in apklist:
 167                     if apk['versioncode'] == int(build.vercode):
 168                         builtit = True
 169                         break
 170                 if not builtit:
 171                     buildfails = True
 172                     apklist.append({'versioncode': int(build.vercode),
 173                                     'version': build.version,
 174                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
 175                                     })
 176         if app.CurrentVersionCode == '0':
 177             cantupdate = True
 178         # Sort with most recent first...
 179         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 180
 181         wikidata += "=Versions=\n"
 182         if len(apklist) == 0:
 183             wikidata += "We currently have no versions of this app available."
 184         elif not gotcurrentver:
 185             wikidata += "We don't have the current version of this app."
 186         else:
 187             wikidata += "We have the current version of this app."
 188         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 189         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 190         if len(app.NoSourceSince) > 0:
 191             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 192         if len(app.CurrentVersion) > 0:
 193             wikidata += "The current (recommended) version is " + app.CurrentVersion
 194             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 195         validapks = 0
 196         for apk in apklist:
 197             wikidata += "==" + apk['version'] + "==\n"
 198
 199             if 'buildproblem' in apk:
 200                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 201             else:
 202                 validapks += 1
 203                 wikidata += "This version is built and signed by "
 204                 if 'srcname' in apk:
 205                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 206                 else:
 207                     wikidata += "the original developer.\n\n"
 208             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 209
 210         wikidata += '\n[[Category:' + wikicat + ']]\n'
 211         if len(app.NoSourceSince) > 0:
 212             wikidata += '\n[[Category:Apps missing source code]]\n'
 213         if validapks == 0 and not app.Disabled:
 214             wikidata += '\n[[Category:Apps with no packages]]\n'
 215         if cantupdate and not app.Disabled:
 216             wikidata += "\n[[Category:Apps we cannot update]]\n"
 217         if buildfails and not app.Disabled:
 218             wikidata += "\n[[Category:Apps with failing builds]]\n"
 219         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 220             wikidata += '\n[[Category:Apps to Update]]\n'
 221         if app.Disabled:
 222             wikidata += '\n[[Category:Apps that are disabled]]\n'
 223         if app.UpdateCheckMode == 'None' and not app.Disabled:
 224             wikidata += '\n[[Category:Apps with no update check]]\n'
 225         for appcat in app.Categories:
 226             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 227
 228         # We can't have underscores in the page name, even if they're in
 229         # the package ID, because MediaWiki messes with them...
 230         pagename = appid.replace('_', ' ')
 231
 232         # Drop a trailing newline, because mediawiki is going to drop it anyway
 233         # and it we don't we'll think the page has changed when it hasn't...
 234         if wikidata.endswith('\n'):
 235             wikidata = wikidata[:-1]
 236
 237         generated_pages[pagename] = wikidata
 238
 239         # Make a redirect from the name to the ID too, unless there's
 240         # already an existing page with the name and it isn't a redirect.
 241         noclobber = False
 242         apppagename = app.Name.replace('_', ' ')
 243         apppagename = apppagename.replace('{', '')
 244         apppagename = apppagename.replace('}', ' ')
 245         apppagename = apppagename.replace(':', ' ')
 246         # Drop double spaces caused mostly by replacing ':' above
 247         apppagename = apppagename.replace('  ', ' ')
 248         for expagename in site.allpages(prefix=apppagename,
 249                                         filterredir='nonredirects',
 250                                         generator=False):
 251             if expagename == apppagename:
 252                 noclobber = True
 253         # Another reason not to make the redirect page is if the app name
 254         # is the same as it's ID, because that will overwrite the real page
 255         # with an redirect to itself! (Although it seems like an odd
 256         # scenario this happens a lot, e.g. where there is metadata but no
 257         # builds or binaries to extract a name from.
 258         if apppagename == pagename:
 259             noclobber = True
 260         if not noclobber:
 261             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 262
 263     for tcat, genp in [(wikicat, generated_pages),
 264                        (wikiredircat, generated_redirects)]:
 265         catpages = site.Pages['Category:' + tcat]
 266         existingpages = []
 267         for page in catpages:
 268             existingpages.append(page.name)
 269             if page.name in genp:
 270                 pagetxt = page.edit()
 271                 if pagetxt != genp[page.name]:
 272                     logging.debug("Updating modified page " + page.name)
 273                     page.save(genp[page.name], summary='Auto-updated')
 274                 else:
 275                     logging.debug("Page " + page.name + " is unchanged")
 276             else:
 277                 logging.warn("Deleting page " + page.name)
 278                 page.delete('No longer published')
 279         for pagename, text in genp.items():
 280             logging.debug("Checking " + pagename)
 281             if pagename not in existingpages:
 282                 logging.debug("Creating page " + pagename)
 283                 try:
 284                     newpage = site.Pages[pagename]
 285                     newpage.save(text, summary='Auto-created')
 286                 except:
 287                     logging.error("...FAILED to create page '{0}'".format(pagename))
 288
 289     # Purge server cache to ensure counts are up to date
 290     site.pages['Repository Maintenance'].purge()
 291
 292
 293 def delete_disabled_builds(apps, apkcache, repodirs):
 294     """Delete disabled build outputs.
 295
 296     :param apps: list of all applications, as per metadata.read_metadata
 297     :param apkcache: current apk cache information
 298     :param repodirs: the repo directories to process
 299     """
 300     for appid, app in apps.items():
 301         for build in app.builds:
 302             if not build.disable:
 303                 continue
 304             apkfilename = appid + '_' + str(build.vercode) + '.apk'
 305             iconfilename = "%s.%s.png" % (
 306                 appid,
 307                 build.vercode)
 308             for repodir in repodirs:
 309                 files = [
 310                     os.path.join(repodir, apkfilename),
 311                     os.path.join(repodir, apkfilename + '.asc'),
 312                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 313                 ]
 314                 for density in all_screen_densities:
 315                     repo_dir = get_icon_dir(repodir, density)
 316                     files.append(os.path.join(repo_dir, iconfilename))
 317
 318                 for f in files:
 319                     if os.path.exists(f):
 320                         logging.info("Deleting disabled build output " + f)
 321                         os.remove(f)
 322             if apkfilename in apkcache:
 323                 del apkcache[apkfilename]
 324
 325
 326 def resize_icon(iconpath, density):
 327
 328     if not os.path.isfile(iconpath):
 329         return
 330
 331     fp = None
 332     try:
 333         fp = open(iconpath, 'rb')
 334         im = Image.open(fp)
 335         size = dpi_to_px(density)
 336
 337         if any(length > size for length in im.size):
 338             oldsize = im.size
 339             im.thumbnail((size, size), Image.ANTIALIAS)
 340             logging.debug("%s was too large at %s - new size is %s" % (
 341                 iconpath, oldsize, im.size))
 342             im.save(iconpath, "PNG")
 343
 344     except Exception as e:
 345         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 346
 347     finally:
 348         if fp:
 349             fp.close()
 350
 351
 352 def resize_all_icons(repodirs):
 353     """Resize all icons that exceed the max size
 354
 355     :param repodirs: the repo directories to process
 356     """
 357     for repodir in repodirs:
 358         for density in screen_densities:
 359             icon_dir = get_icon_dir(repodir, density)
 360             icon_glob = os.path.join(icon_dir, '*.png')
 361             for iconpath in glob.glob(icon_glob):
 362                 resize_icon(iconpath, density)
 363
 364
 365 # A signature block file with a .DSA, .RSA, or .EC extension
 366 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 367
 368
 369 def getsig(apkpath):
 370     """ Get the signing certificate of an apk. To get the same md5 has that
 371     Android gets, we encode the .RSA certificate in a specific format and pass
 372     it hex-encoded to the md5 digest algorithm.
 373
 374     :param apkpath: path to the apk
 375     :returns: A string containing the md5 of the signature of the apk or None
 376               if an error occurred.
 377     """
 378
 379     cert = None
 380
 381     # verify the jar signature is correct
 382     args = [config['jarsigner'], '-verify', apkpath]
 383     p = FDroidPopen(args)
 384     if p.returncode != 0:
 385         logging.critical(apkpath + " has a bad signature!")
 386         return None
 387
 388     with zipfile.ZipFile(apkpath, 'r') as apk:
 389
 390         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 391
 392         if len(certs) < 1:
 393             logging.error("Found no signing certificates on %s" % apkpath)
 394             return None
 395         if len(certs) > 1:
 396             logging.error("Found multiple signing certificates on %s" % apkpath)
 397             return None
 398
 399         cert = apk.read(certs[0])
 400
 401     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 402     if content.getComponentByName('contentType') != rfc2315.signedData:
 403         logging.error("Unexpected format.")
 404         return None
 405
 406     content = decoder.decode(content.getComponentByName('content'),
 407                              asn1Spec=rfc2315.SignedData())[0]
 408     try:
 409         certificates = content.getComponentByName('certificates')
 410     except PyAsn1Error:
 411         logging.error("Certificates not found.")
 412         return None
 413
 414     cert_encoded = encoder.encode(certificates)[4:]
 415
 416     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 417
 418
 419 def get_icon_bytes(apkzip, iconsrc):
 420     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 421     try:
 422         return apkzip.read(iconsrc)
 423     except KeyError:
 424         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 425
 426
 427 def sha256sum(filename):
 428     '''Calculate the sha256 of the given file'''
 429     sha = hashlib.sha256()
 430     with open(filename, 'rb') as f:
 431         while True:
 432             t = f.read(16384)
 433             if len(t) == 0:
 434                 break
 435             sha.update(t)
 436     return sha.hexdigest()
 437
 438
 439 def insert_obbs(repodir, apps, apks):
 440     """Scans the .obb files in a given repo directory and adds them to the
 441     relevant APK instances.  OBB files have versionCodes like APK
 442     files, and they are loosely associated.  If there is an OBB file
 443     present, then any APK with the same or higher versionCode will use
 444     that OBB file.  There are two OBB types: main and patch, each APK
 445     can only have only have one of each.
 446
 447     https://developer.android.com/google/play/expansion-files.html
 448
 449     :param repodir: repo directory to scan
 450     :param apps: list of current, valid apps
 451     :param apks: current information on all APKs
 452
 453     """
 454
 455     def obbWarnDelete(f, msg):
 456         logging.warning(msg + f)
 457         if options.delete_unknown:
 458             logging.error("Deleting unknown file: " + f)
 459             os.remove(f)
 460
 461     obbs = []
 462     java_Integer_MIN_VALUE = -pow(2, 31)
 463     for f in glob.glob(os.path.join(repodir, '*.obb')):
 464         obbfile = os.path.basename(f)
 465         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 466         chunks = obbfile.split('.')
 467         if chunks[0] != 'main' and chunks[0] != 'patch':
 468             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 469             continue
 470         if not re.match(r'^-?[0-9]+$', chunks[1]):
 471             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 472             continue
 473         versioncode = int(chunks[1])
 474         packagename = ".".join(chunks[2:-1])
 475
 476         highestVersionCode = java_Integer_MIN_VALUE
 477         if packagename not in apps.keys():
 478             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 479             continue
 480         for apk in apks:
 481             if packagename == apk['id'] and apk['versioncode'] > highestVersionCode:
 482                 highestVersionCode = apk['versioncode']
 483         if versioncode > highestVersionCode:
 484             obbWarnDelete(f, 'OBB file has newer versioncode(' + str(versioncode)
 485                           + ') than any APK: ')
 486             continue
 487         obbsha256 = sha256sum(f)
 488         obbs.append((packagename, versioncode, obbfile, obbsha256))
 489
 490     for apk in apks:
 491         for (packagename, versioncode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 492             if versioncode <= apk['versioncode'] and packagename == apk['id']:
 493                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 494                     apk['obbMainFile'] = obbfile
 495                     apk['obbMainFileSha256'] = obbsha256
 496                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 497                     apk['obbPatchFile'] = obbfile
 498                     apk['obbPatchFileSha256'] = obbsha256
 499             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 500                 break
 501
 502
 503 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
 504     """Scan the apks in the given repo directory.
 505
 506     This also extracts the icons.
 507
 508     :param apps: list of all applications, as per metadata.read_metadata
 509     :param apkcache: current apk cache information
 510     :param repodir: repo directory to scan
 511     :param knownapks: known apks info
 512     :param use_date_from_apk: use date from APK (instead of current date)
 513                               for newly added APKs
 514     :returns: (apks, cachechanged) where apks is a list of apk information,
 515               and cachechanged is True if the apkcache got changed.
 516     """
 517
 518     cachechanged = False
 519
 520     for icon_dir in get_all_icon_dirs(repodir):
 521         if os.path.exists(icon_dir):
 522             if options.clean:
 523                 shutil.rmtree(icon_dir)
 524                 os.makedirs(icon_dir)
 525         else:
 526             os.makedirs(icon_dir)
 527
 528     apks = []
 529     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 530     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 531     vername_pat = re.compile(".*versionName='([^']*)'.*")
 532     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 533     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 534     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 535     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 536     permission_pat = re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
 537     feature_pat = re.compile(".*name='([^']*)'.*")
 538     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 539
 540         apkfilename = apkfile[len(repodir) + 1:]
 541         if ' ' in apkfilename:
 542             logging.critical("Spaces in filenames are not allowed.")
 543             sys.exit(1)
 544
 545         shasum = sha256sum(apkfile)
 546
 547         usecache = False
 548         if apkfilename in apkcache:
 549             apk = apkcache[apkfilename]
 550             if apk['sha256'] == shasum:
 551                 logging.debug("Reading " + apkfilename + " from cache")
 552                 usecache = True
 553             else:
 554                 logging.debug("Ignoring stale cache data for " + apkfilename)
 555
 556         if not usecache:
 557             logging.debug("Processing " + apkfilename)
 558             apk = {}
 559             apk['apkname'] = apkfilename
 560             apk['sha256'] = shasum
 561             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 562             if os.path.exists(os.path.join(repodir, srcfilename)):
 563                 apk['srcname'] = srcfilename
 564             apk['size'] = os.path.getsize(apkfile)
 565             apk['uses-permission'] = set()
 566             apk['uses-permission-sdk-23'] = set()
 567             apk['features'] = set()
 568             apk['icons_src'] = {}
 569             apk['icons'] = {}
 570             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 571             if p.returncode != 0:
 572                 if options.delete_unknown:
 573                     if os.path.exists(apkfile):
 574                         logging.error("Failed to get apk information, deleting " + apkfile)
 575                         os.remove(apkfile)
 576                     else:
 577                         logging.error("Could not find {0} to remove it".format(apkfile))
 578                 else:
 579                     logging.error("Failed to get apk information, skipping " + apkfile)
 580                 continue
 581             for line in p.output.splitlines():
 582                 if line.startswith("package:"):
 583                     try:
 584                         apk['id'] = re.match(name_pat, line).group(1)
 585                         apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
 586                         apk['version'] = re.match(vername_pat, line).group(1)
 587                     except Exception as e:
 588                         logging.error("Package matching failed: " + str(e))
 589                         logging.info("Line was: " + line)
 590                         sys.exit(1)
 591                 elif line.startswith("application:"):
 592                     apk['name'] = re.match(label_pat, line).group(1)
 593                     # Keep path to non-dpi icon in case we need it
 594                     match = re.match(icon_pat_nodpi, line)
 595                     if match:
 596                         apk['icons_src']['-1'] = match.group(1)
 597                 elif line.startswith("launchable-activity:"):
 598                     # Only use launchable-activity as fallback to application
 599                     if not apk['name']:
 600                         apk['name'] = re.match(label_pat, line).group(1)
 601                     if '-1' not in apk['icons_src']:
 602                         match = re.match(icon_pat_nodpi, line)
 603                         if match:
 604                             apk['icons_src']['-1'] = match.group(1)
 605                 elif line.startswith("application-icon-"):
 606                     match = re.match(icon_pat, line)
 607                     if match:
 608                         density = match.group(1)
 609                         path = match.group(2)
 610                         apk['icons_src'][density] = path
 611                 elif line.startswith("sdkVersion:"):
 612                     m = re.match(sdkversion_pat, line)
 613                     if m is None:
 614                         logging.error(line.replace('sdkVersion:', '')
 615                                       + ' is not a valid minSdkVersion!')
 616                     else:
 617                         apk['minSdkVersion'] = m.group(1)
 618                         # if target not set, default to min
 619                         if 'targetSdkVersion' not in apk:
 620                             apk['targetSdkVersion'] = m.group(1)
 621                 elif line.startswith("targetSdkVersion:"):
 622                     m = re.match(sdkversion_pat, line)
 623                     if m is None:
 624                         logging.error(line.replace('targetSdkVersion:', '')
 625                                       + ' is not a valid targetSdkVersion!')
 626                     else:
 627                         apk['targetSdkVersion'] = m.group(1)
 628                 elif line.startswith("maxSdkVersion:"):
 629                     apk['maxSdkVersion'] = re.match(sdkversion_pat, line).group(1)
 630                 elif line.startswith("native-code:"):
 631                     apk['nativecode'] = []
 632                     for arch in line[13:].split(' '):
 633                         apk['nativecode'].append(arch[1:-1])
 634                 elif line.startswith('uses-permission:'):
 635                     perm_match = re.match(permission_pat, line).groupdict()
 636
 637                     permission = UsesPermission(
 638                         perm_match['name'],
 639                         perm_match['maxSdkVersion']
 640                     )
 641
 642                     apk['uses-permission'].add(permission)
 643                 elif line.startswith('uses-permission-sdk-23:'):
 644                     perm_match = re.match(permission_pat, line).groupdict()
 645
 646                     permission_sdk_23 = UsesPermissionSdk23(
 647                         perm_match['name'],
 648                         perm_match['maxSdkVersion']
 649                     )
 650
 651                     apk['uses-permission-sdk-23'].add(permission_sdk_23)
 652
 653                 elif line.startswith('uses-feature:'):
 654                     feature = re.match(feature_pat, line).group(1)
 655                     # Filter out this, it's only added with the latest SDK tools and
 656                     # causes problems for lots of apps.
 657                     if feature != "android.hardware.screen.portrait" \
 658                             and feature != "android.hardware.screen.landscape":
 659                         if feature.startswith("android.feature."):
 660                             feature = feature[16:]
 661                         apk['features'].add(feature)
 662
 663             if 'minSdkVersion' not in apk:
 664                 logging.warn("No SDK version information found in {0}".format(apkfile))
 665                 apk['minSdkVersion'] = 1
 666
 667             # Check for debuggable apks...
 668             if common.isApkDebuggable(apkfile, config):
 669                 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
 670
 671             # Get the signature (or md5 of, to be precise)...
 672             logging.debug('Getting signature of {0}'.format(apkfile))
 673             apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 674             if not apk['sig']:
 675                 logging.critical("Failed to get apk signature")
 676                 sys.exit(1)
 677
 678             apkzip = zipfile.ZipFile(apkfile, 'r')
 679
 680             # if an APK has files newer than the system time, suggest updating
 681             # the system clock.  This is useful for offline systems, used for
 682             # signing, which do not have another source of clock sync info. It
 683             # has to be more than 24 hours newer because ZIP/APK files do not
 684             # store timezone info
 685             manifest = apkzip.getinfo('AndroidManifest.xml')
 686             if manifest.date_time[1] == 0:  # month can't be zero
 687                 logging.debug('AndroidManifest.xml has no date')
 688             else:
 689                 dt_obj = datetime(*manifest.date_time)
 690                 checkdt = dt_obj - timedelta(1)
 691                 if datetime.today() < checkdt:
 692                     logging.warn('System clock is older than manifest in: '
 693                                  + apkfilename
 694                                  + '\nSet clock to that time using:\n'
 695                                  + 'sudo date -s "' + str(dt_obj) + '"')
 696
 697             iconfilename = "%s.%s.png" % (
 698                 apk['id'],
 699                 apk['versioncode'])
 700
 701             # Extract the icon file...
 702             empty_densities = []
 703             for density in screen_densities:
 704                 if density not in apk['icons_src']:
 705                     empty_densities.append(density)
 706                     continue
 707                 iconsrc = apk['icons_src'][density]
 708                 icon_dir = get_icon_dir(repodir, density)
 709                 icondest = os.path.join(icon_dir, iconfilename)
 710
 711                 try:
 712                     with open(icondest, 'wb') as f:
 713                         f.write(get_icon_bytes(apkzip, iconsrc))
 714                     apk['icons'][density] = iconfilename
 715
 716                 except:
 717                     logging.warn("Error retrieving icon file")
 718                     del apk['icons'][density]
 719                     del apk['icons_src'][density]
 720                     empty_densities.append(density)
 721
 722             if '-1' in apk['icons_src']:
 723                 iconsrc = apk['icons_src']['-1']
 724                 iconpath = os.path.join(
 725                     get_icon_dir(repodir, '0'), iconfilename)
 726                 with open(iconpath, 'wb') as f:
 727                     f.write(get_icon_bytes(apkzip, iconsrc))
 728                 try:
 729                     im = Image.open(iconpath)
 730                     dpi = px_to_dpi(im.size[0])
 731                     for density in screen_densities:
 732                         if density in apk['icons']:
 733                             break
 734                         if density == screen_densities[-1] or dpi >= int(density):
 735                             apk['icons'][density] = iconfilename
 736                             shutil.move(iconpath,
 737                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 738                             empty_densities.remove(density)
 739                             break
 740                 except Exception as e:
 741                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 742
 743             if apk['icons']:
 744                 apk['icon'] = iconfilename
 745
 746             apkzip.close()
 747
 748             # First try resizing down to not lose quality
 749             last_density = None
 750             for density in screen_densities:
 751                 if density not in empty_densities:
 752                     last_density = density
 753                     continue
 754                 if last_density is None:
 755                     continue
 756                 logging.debug("Density %s not available, resizing down from %s"
 757                               % (density, last_density))
 758
 759                 last_iconpath = os.path.join(
 760                     get_icon_dir(repodir, last_density), iconfilename)
 761                 iconpath = os.path.join(
 762                     get_icon_dir(repodir, density), iconfilename)
 763                 fp = None
 764                 try:
 765                     fp = open(last_iconpath, 'rb')
 766                     im = Image.open(fp)
 767
 768                     size = dpi_to_px(density)
 769
 770                     im.thumbnail((size, size), Image.ANTIALIAS)
 771                     im.save(iconpath, "PNG")
 772                     empty_densities.remove(density)
 773                 except:
 774                     logging.warning("Invalid image file at %s" % last_iconpath)
 775                 finally:
 776                     if fp:
 777                         fp.close()
 778
 779             # Then just copy from the highest resolution available
 780             last_density = None
 781             for density in reversed(screen_densities):
 782                 if density not in empty_densities:
 783                     last_density = density
 784                     continue
 785                 if last_density is None:
 786                     continue
 787                 logging.debug("Density %s not available, copying from lower density %s"
 788                               % (density, last_density))
 789
 790                 shutil.copyfile(
 791                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 792                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 793
 794                 empty_densities.remove(density)
 795
 796             for density in screen_densities:
 797                 icon_dir = get_icon_dir(repodir, density)
 798                 icondest = os.path.join(icon_dir, iconfilename)
 799                 resize_icon(icondest, density)
 800
 801             # Copy from icons-mdpi to icons since mdpi is the baseline density
 802             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 803             if os.path.isfile(baseline):
 804                 apk['icons']['0'] = iconfilename
 805                 shutil.copyfile(baseline,
 806                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 807
 808             if use_date_from_apk and manifest.date_time[1] != 0:
 809                 default_date_param = datetime(*manifest.date_time).utctimetuple()
 810             else:
 811                 default_date_param = None
 812
 813             # Record in known apks, getting the added date at the same time..
 814             added = knownapks.recordapk(apk['apkname'], apk['id'], default_date=default_date_param)
 815             if added:
 816                 apk['added'] = added
 817
 818             apkcache[apkfilename] = apk
 819             cachechanged = True
 820
 821         apks.append(apk)
 822
 823     return apks, cachechanged
 824
 825
 826 repo_pubkey_fingerprint = None
 827
 828
 829 # Generate a certificate fingerprint the same way keytool does it
 830 # (but with slightly different formatting)
 831 def cert_fingerprint(data):
 832     digest = hashlib.sha256(data).digest()
 833     ret = []
 834     ret.append(' '.join("%02X" % b for b in bytearray(digest)))
 835     return " ".join(ret)
 836
 837
 838 def extract_pubkey():
 839     global repo_pubkey_fingerprint
 840     if 'repo_pubkey' in config:
 841         pubkey = unhexlify(config['repo_pubkey'])
 842     else:
 843         p = FDroidPopenBytes([config['keytool'], '-exportcert',
 844                               '-alias', config['repo_keyalias'],
 845                               '-keystore', config['keystore'],
 846                               '-storepass:file', config['keystorepassfile']]
 847                              + config['smartcardoptions'],
 848                              output=False, stderr_to_stdout=False)
 849         if p.returncode != 0 or len(p.output) < 20:
 850             msg = "Failed to get repo pubkey!"
 851             if config['keystore'] == 'NONE':
 852                 msg += ' Is your crypto smartcard plugged in?'
 853             logging.critical(msg)
 854             sys.exit(1)
 855         pubkey = p.output
 856     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 857     return hexlify(pubkey)
 858
 859
 860 def make_index(apps, sortedids, apks, repodir, archive, categories):
 861     """Make a repo index.
 862
 863     :param apps: fully populated apps list
 864     :param apks: full populated apks list
 865     :param repodir: the repo directory
 866     :param archive: True if this is the archive repo, False if it's the
 867                     main one.
 868     :param categories: list of categories
 869     """
 870
 871     doc = Document()
 872
 873     def addElement(name, value, doc, parent):
 874         el = doc.createElement(name)
 875         el.appendChild(doc.createTextNode(value))
 876         parent.appendChild(el)
 877
 878     def addElementNonEmpty(name, value, doc, parent):
 879         if not value:
 880             return
 881         addElement(name, value, doc, parent)
 882
 883     def addElementCDATA(name, value, doc, parent):
 884         el = doc.createElement(name)
 885         el.appendChild(doc.createCDATASection(value))
 886         parent.appendChild(el)
 887
 888     root = doc.createElement("fdroid")
 889     doc.appendChild(root)
 890
 891     repoel = doc.createElement("repo")
 892
 893     mirrorcheckfailed = False
 894     for mirror in config.get('mirrors', []):
 895         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
 896         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
 897             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 898             mirrorcheckfailed = True
 899     if mirrorcheckfailed:
 900         sys.exit(1)
 901
 902     if archive:
 903         repoel.setAttribute("name", config['archive_name'])
 904         if config['repo_maxage'] != 0:
 905             repoel.setAttribute("maxage", str(config['repo_maxage']))
 906         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 907         repoel.setAttribute("url", config['archive_url'])
 908         addElement('description', config['archive_description'], doc, repoel)
 909         urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
 910         for mirror in config.get('mirrors', []):
 911             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 912
 913     else:
 914         repoel.setAttribute("name", config['repo_name'])
 915         if config['repo_maxage'] != 0:
 916             repoel.setAttribute("maxage", str(config['repo_maxage']))
 917         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 918         repoel.setAttribute("url", config['repo_url'])
 919         addElement('description', config['repo_description'], doc, repoel)
 920         urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
 921         for mirror in config.get('mirrors', []):
 922             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 923
 924     repoel.setAttribute("version", str(METADATA_VERSION))
 925     repoel.setAttribute("timestamp", str(int(time.time())))
 926
 927     nosigningkey = False
 928     if not options.nosign:
 929         if 'repo_keyalias' not in config:
 930             nosigningkey = True
 931             logging.critical("'repo_keyalias' not found in config.py!")
 932         if 'keystore' not in config:
 933             nosigningkey = True
 934             logging.critical("'keystore' not found in config.py!")
 935         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 936             nosigningkey = True
 937             logging.critical("'keystorepass' not found in config.py!")
 938         if 'keypass' not in config and 'keypassfile' not in config:
 939             nosigningkey = True
 940             logging.critical("'keypass' not found in config.py!")
 941         if not os.path.exists(config['keystore']):
 942             nosigningkey = True
 943             logging.critical("'" + config['keystore'] + "' does not exist!")
 944         if nosigningkey:
 945             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 946             logging.warning("\tfdroid update --create-key")
 947             sys.exit(1)
 948
 949     repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
 950     root.appendChild(repoel)
 951
 952     for appid in sortedids:
 953         app = apps[appid]
 954
 955         if app.Disabled is not None:
 956             continue
 957
 958         # Get a list of the apks for this app...
 959         apklist = []
 960         for apk in apks:
 961             if apk['id'] == appid:
 962                 apklist.append(apk)
 963
 964         if len(apklist) == 0:
 965             continue
 966
 967         apel = doc.createElement("application")
 968         apel.setAttribute("id", app.id)
 969         root.appendChild(apel)
 970
 971         addElement('id', app.id, doc, apel)
 972         if app.added:
 973             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
 974         if app.lastupdated:
 975             addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
 976         addElement('name', app.Name, doc, apel)
 977         addElement('summary', app.Summary, doc, apel)
 978         if app.icon:
 979             addElement('icon', app.icon, doc, apel)
 980
 981         def linkres(appid):
 982             if appid in apps:
 983                 return ("fdroid.app:" + appid, apps[appid].Name)
 984             raise MetaDataException("Cannot resolve app id " + appid)
 985
 986         addElement('desc',
 987                    metadata.description_html(app.Description, linkres),
 988                    doc, apel)
 989         addElement('license', app.License, doc, apel)
 990         if app.Categories:
 991             addElement('categories', ','.join(app.Categories), doc, apel)
 992             # We put the first (primary) category in LAST, which will have
 993             # the desired effect of making clients that only understand one
 994             # category see that one.
 995             addElement('category', app.Categories[0], doc, apel)
 996         addElement('web', app.WebSite, doc, apel)
 997         addElement('source', app.SourceCode, doc, apel)
 998         addElement('tracker', app.IssueTracker, doc, apel)
 999         addElementNonEmpty('changelog', app.Changelog, doc, apel)
1000         addElementNonEmpty('author', app.AuthorName, doc, apel)
1001         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
1002         addElementNonEmpty('donate', app.Donate, doc, apel)
1003         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
1004         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
1005         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
1006
1007         # These elements actually refer to the current version (i.e. which
1008         # one is recommended. They are historically mis-named, and need
1009         # changing, but stay like this for now to support existing clients.
1010         addElement('marketversion', app.CurrentVersion, doc, apel)
1011         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
1012
1013         if app.AntiFeatures:
1014             af = app.AntiFeatures
1015             if af:
1016                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
1017         if app.Provides:
1018             pv = app.Provides.split(',')
1019             addElementNonEmpty('provides', ','.join(pv), doc, apel)
1020         if app.RequiresRoot:
1021             addElement('requirements', 'root', doc, apel)
1022
1023         # Sort the apk list into version order, just so the web site
1024         # doesn't have to do any work by default...
1025         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
1026
1027         # Check for duplicates - they will make the client unhappy...
1028         for i in range(len(apklist) - 1):
1029             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
1030                 logging.critical("duplicate versions: '%s' - '%s'" % (
1031                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
1032                 sys.exit(1)
1033
1034         current_version_code = 0
1035         current_version_file = None
1036         for apk in apklist:
1037             # find the APK for the "Current Version"
1038             if current_version_code < apk['versioncode']:
1039                 current_version_code = apk['versioncode']
1040             if current_version_code < int(app.CurrentVersionCode):
1041                 current_version_file = apk['apkname']
1042
1043             apkel = doc.createElement("package")
1044             apel.appendChild(apkel)
1045             addElement('version', apk['version'], doc, apkel)
1046             addElement('versioncode', str(apk['versioncode']), doc, apkel)
1047             addElement('apkname', apk['apkname'], doc, apkel)
1048             if 'srcname' in apk:
1049                 addElement('srcname', apk['srcname'], doc, apkel)
1050             for hash_type in ['sha256']:
1051                 if hash_type not in apk:
1052                     continue
1053                 hashel = doc.createElement("hash")
1054                 hashel.setAttribute("type", hash_type)
1055                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
1056                 apkel.appendChild(hashel)
1057             addElement('sig', apk['sig'], doc, apkel)
1058             addElement('size', str(apk['size']), doc, apkel)
1059             addElement('sdkver', str(apk['minSdkVersion']), doc, apkel)
1060             if 'targetSdkVersion' in apk:
1061                 addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
1062             if 'maxSdkVersion' in apk:
1063                 addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
1064             addElementNonEmpty('obbMainFile', apk.get('obbMainFile'), doc, apkel)
1065             addElementNonEmpty('obbMainFileSha256', apk.get('obbMainFileSha256'), doc, apkel)
1066             addElementNonEmpty('obbPatchFile', apk.get('obbPatchFile'), doc, apkel)
1067             addElementNonEmpty('obbPatchFileSha256', apk.get('obbPatchFileSha256'), doc, apkel)
1068             if 'added' in apk:
1069                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
1070
1071             # TODO: remove old permission format
1072             old_permissions = set()
1073             for perm in apk['uses-permission']:
1074                 perm_name = perm.name
1075                 if perm_name.startswith("android.permission."):
1076                     perm_name = perm_name[19:]
1077                 old_permissions.add(perm_name)
1078             addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel)
1079
1080             for permission in apk['uses-permission']:
1081                 permel = doc.createElement('uses-permission')
1082                 permel.setAttribute('name', permission.name)
1083                 if permission.maxSdkVersion is not None:
1084                     permel.setAttribute('maxSdkVersion', permission.maxSdkVersion)
1085                     apkel.appendChild(permel)
1086             for permission_sdk_23 in apk['uses-permission-sdk-23']:
1087                 permel = doc.createElement('uses-permission-sdk-23')
1088                 permel.setAttribute('name', permission_sdk_23.name)
1089                 if permission_sdk_23.maxSdkVersion is not None:
1090                     permel.setAttribute('maxSdkVersion', permission_sdk_23.maxSdkVersion)
1091                     apkel.appendChild(permel)
1092             if 'nativecode' in apk:
1093                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
1094             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
1095
1096         if current_version_file is not None \
1097                 and config['make_current_version_link'] \
1098                 and repodir == 'repo':  # only create these
1099             namefield = config['current_version_name_source']
1100             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
1101             apklinkname = sanitized_name + '.apk'
1102             current_version_path = os.path.join(repodir, current_version_file)
1103             if os.path.islink(apklinkname):
1104                 os.remove(apklinkname)
1105             os.symlink(current_version_path, apklinkname)
1106             # also symlink gpg signature, if it exists
1107             for extension in ('.asc', '.sig'):
1108                 sigfile_path = current_version_path + extension
1109                 if os.path.exists(sigfile_path):
1110                     siglinkname = apklinkname + extension
1111                     if os.path.islink(siglinkname):
1112                         os.remove(siglinkname)
1113                     os.symlink(sigfile_path, siglinkname)
1114
1115     if options.pretty:
1116         output = doc.toprettyxml(encoding='utf-8')
1117     else:
1118         output = doc.toxml(encoding='utf-8')
1119
1120     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
1121         f.write(output)
1122
1123     if 'repo_keyalias' in config:
1124
1125         if options.nosign:
1126             logging.info("Creating unsigned index in preparation for signing")
1127         else:
1128             logging.info("Creating signed index with this key (SHA256):")
1129             logging.info("%s" % repo_pubkey_fingerprint)
1130
1131         # Create a jar of the index...
1132         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
1133         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
1134         if p.returncode != 0:
1135             logging.critical("Failed to create {0}".format(jar_output))
1136             sys.exit(1)
1137
1138         # Sign the index...
1139         signed = os.path.join(repodir, 'index.jar')
1140         if options.nosign:
1141             # Remove old signed index if not signing
1142             if os.path.exists(signed):
1143                 os.remove(signed)
1144         else:
1145             args = [config['jarsigner'], '-keystore', config['keystore'],
1146                     '-storepass:file', config['keystorepassfile'],
1147                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1148                     signed, config['repo_keyalias']]
1149             if config['keystore'] == 'NONE':
1150                 args += config['smartcardoptions']
1151             else:  # smardcards never use -keypass
1152                 args += ['-keypass:file', config['keypassfile']]
1153             p = FDroidPopen(args)
1154             if p.returncode != 0:
1155                 logging.critical("Failed to sign index")
1156                 sys.exit(1)
1157
1158     # Copy the repo icon into the repo directory...
1159     icon_dir = os.path.join(repodir, 'icons')
1160     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1161     shutil.copyfile(config['repo_icon'], iconfilename)
1162
1163     # Write a category list in the repo to allow quick access...
1164     catdata = ''
1165     for cat in categories:
1166         catdata += cat + '\n'
1167     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1168         f.write(catdata)
1169
1170
1171 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1172
1173     for appid, app in apps.items():
1174
1175         if app.ArchivePolicy:
1176             keepversions = int(app.ArchivePolicy[:-9])
1177         else:
1178             keepversions = defaultkeepversions
1179
1180         def filter_apk_list_sorted(apk_list):
1181             res = []
1182             for apk in apk_list:
1183                 if apk['id'] == appid:
1184                     res.append(apk)
1185
1186             # Sort the apk list by version code. First is highest/newest.
1187             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1188
1189         def move_file(from_dir, to_dir, filename, ignore_missing):
1190             from_path = os.path.join(from_dir, filename)
1191             if ignore_missing and not os.path.exists(from_path):
1192                 return
1193             to_path = os.path.join(to_dir, filename)
1194             shutil.move(from_path, to_path)
1195
1196         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1197                       .format(appid, len(apks), keepversions, len(archapks)))
1198
1199         if len(apks) > keepversions:
1200             apklist = filter_apk_list_sorted(apks)
1201             # Move back the ones we don't want.
1202             for apk in apklist[keepversions:]:
1203                 logging.info("Moving " + apk['apkname'] + " to archive")
1204                 move_file(repodir, archivedir, apk['apkname'], False)
1205                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1206                 for density in all_screen_densities:
1207                     repo_icon_dir = get_icon_dir(repodir, density)
1208                     archive_icon_dir = get_icon_dir(archivedir, density)
1209                     if density not in apk['icons']:
1210                         continue
1211                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1212                 if 'srcname' in apk:
1213                     move_file(repodir, archivedir, apk['srcname'], False)
1214                 archapks.append(apk)
1215                 apks.remove(apk)
1216         elif len(apks) < keepversions and len(archapks) > 0:
1217             required = keepversions - len(apks)
1218             archapklist = filter_apk_list_sorted(archapks)
1219             # Move forward the ones we want again.
1220             for apk in archapklist[:required]:
1221                 logging.info("Moving " + apk['apkname'] + " from archive")
1222                 move_file(archivedir, repodir, apk['apkname'], False)
1223                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1224                 for density in all_screen_densities:
1225                     repo_icon_dir = get_icon_dir(repodir, density)
1226                     archive_icon_dir = get_icon_dir(archivedir, density)
1227                     if density not in apk['icons']:
1228                         continue
1229                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1230                 if 'srcname' in apk:
1231                     move_file(archivedir, repodir, apk['srcname'], False)
1232                 archapks.remove(apk)
1233                 apks.append(apk)
1234
1235
1236 def add_apks_to_per_app_repos(repodir, apks):
1237     apks_per_app = dict()
1238     for apk in apks:
1239         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1240         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1241         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1242         apks_per_app[apk['id']] = apk
1243
1244         if not os.path.exists(apk['per_app_icons']):
1245             logging.info('Adding new repo for only ' + apk['id'])
1246             os.makedirs(apk['per_app_icons'])
1247
1248         apkpath = os.path.join(repodir, apk['apkname'])
1249         shutil.copy(apkpath, apk['per_app_repo'])
1250         apksigpath = apkpath + '.sig'
1251         if os.path.exists(apksigpath):
1252             shutil.copy(apksigpath, apk['per_app_repo'])
1253         apkascpath = apkpath + '.asc'
1254         if os.path.exists(apkascpath):
1255             shutil.copy(apkascpath, apk['per_app_repo'])
1256
1257
1258 config = None
1259 options = None
1260
1261
1262 def main():
1263
1264     global config, options
1265
1266     # Parse command line...
1267     parser = ArgumentParser()
1268     common.setup_global_opts(parser)
1269     parser.add_argument("--create-key", action="store_true", default=False,
1270                         help="Create a repo signing key in a keystore")
1271     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1272                         help="Create skeleton metadata files that are missing")
1273     parser.add_argument("--delete-unknown", action="store_true", default=False,
1274                         help="Delete APKs and/or OBBs without metadata from the repo")
1275     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1276                         help="Report on build data status")
1277     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1278                         help="Interactively ask about things that need updating.")
1279     parser.add_argument("-I", "--icons", action="store_true", default=False,
1280                         help="Resize all the icons exceeding the max pixel size and exit")
1281     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1282                         help="Specify editor to use in interactive mode. Default " +
1283                         "is /etc/alternatives/editor")
1284     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1285                         help="Update the wiki")
1286     parser.add_argument("--pretty", action="store_true", default=False,
1287                         help="Produce human-readable index.xml")
1288     parser.add_argument("--clean", action="store_true", default=False,
1289                         help="Clean update - don't uses caches, reprocess all apks")
1290     parser.add_argument("--nosign", action="store_true", default=False,
1291                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1292     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1293                         help="Use date from apk instead of current time for newly added apks")
1294     options = parser.parse_args()
1295
1296     config = common.read_config(options)
1297
1298     if not ('jarsigner' in config and 'keytool' in config):
1299         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1300         sys.exit(1)
1301
1302     repodirs = ['repo']
1303     if config['archive_older'] != 0:
1304         repodirs.append('archive')
1305         if not os.path.exists('archive'):
1306             os.mkdir('archive')
1307
1308     if options.icons:
1309         resize_all_icons(repodirs)
1310         sys.exit(0)
1311
1312     # check that icons exist now, rather than fail at the end of `fdroid update`
1313     for k in ['repo_icon', 'archive_icon']:
1314         if k in config:
1315             if not os.path.exists(config[k]):
1316                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1317                 sys.exit(1)
1318
1319     # if the user asks to create a keystore, do it now, reusing whatever it can
1320     if options.create_key:
1321         if os.path.exists(config['keystore']):
1322             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1323             logging.critical("\t'" + config['keystore'] + "'")
1324             sys.exit(1)
1325
1326         if 'repo_keyalias' not in config:
1327             config['repo_keyalias'] = socket.getfqdn()
1328             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1329         if 'keydname' not in config:
1330             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1331             common.write_to_config(config, 'keydname', config['keydname'])
1332         if 'keystore' not in config:
1333             config['keystore'] = common.default_config.keystore
1334             common.write_to_config(config, 'keystore', config['keystore'])
1335
1336         password = common.genpassword()
1337         if 'keystorepass' not in config:
1338             config['keystorepass'] = password
1339             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1340         if 'keypass' not in config:
1341             config['keypass'] = password
1342             common.write_to_config(config, 'keypass', config['keypass'])
1343         common.genkeystore(config)
1344
1345     # Get all apps...
1346     apps = metadata.read_metadata()
1347
1348     # Generate a list of categories...
1349     categories = set()
1350     for app in apps.values():
1351         categories.update(app.Categories)
1352
1353     # Read known apks data (will be updated and written back when we've finished)
1354     knownapks = common.KnownApks()
1355
1356     # Gather information about all the apk files in the repo directory, using
1357     # cached data if possible.
1358     apkcachefile = os.path.join('tmp', 'apkcache')
1359     if not options.clean and os.path.exists(apkcachefile):
1360         with open(apkcachefile, 'rb') as cf:
1361             apkcache = pickle.load(cf, encoding='utf-8')
1362         if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
1363             apkcache = {}
1364     else:
1365         apkcache = {}
1366
1367     delete_disabled_builds(apps, apkcache, repodirs)
1368
1369     # Scan all apks in the main repo
1370     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1371
1372     # Generate warnings for apk's with no metadata (or create skeleton
1373     # metadata files, if requested on the command line)
1374     newmetadata = False
1375     for apk in apks:
1376         if apk['id'] not in apps:
1377             if options.create_metadata:
1378                 if 'name' not in apk:
1379                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1380                     continue
1381                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
1382                 f.write("License:Unknown\n")
1383                 f.write("Web Site:\n")
1384                 f.write("Source Code:\n")
1385                 f.write("Issue Tracker:\n")
1386                 f.write("Changelog:\n")
1387                 f.write("Summary:" + apk['name'] + "\n")
1388                 f.write("Description:\n")
1389                 f.write(apk['name'] + "\n")
1390                 f.write(".\n")
1391                 f.close()
1392                 logging.info("Generated skeleton metadata for " + apk['id'])
1393                 newmetadata = True
1394             else:
1395                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1396                 if options.delete_unknown:
1397                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1398                     rmf = os.path.join(repodirs[0], apk['apkname'])
1399                     if not os.path.exists(rmf):
1400                         logging.error("Could not find {0} to remove it".format(rmf))
1401                     else:
1402                         os.remove(rmf)
1403                 else:
1404                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1405
1406     # update the metadata with the newly created ones included
1407     if newmetadata:
1408         apps = metadata.read_metadata()
1409
1410     insert_obbs(repodirs[0], apps, apks)
1411
1412     # Scan the archive repo for apks as well
1413     if len(repodirs) > 1:
1414         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1415         if cc:
1416             cachechanged = True
1417     else:
1418         archapks = []
1419
1420     # Some information from the apks needs to be applied up to the application
1421     # level. When doing this, we use the info from the most recent version's apk.
1422     # We deal with figuring out when the app was added and last updated at the
1423     # same time.
1424     for appid, app in apps.items():
1425         bestver = 0
1426         for apk in apks + archapks:
1427             if apk['id'] == appid:
1428                 if apk['versioncode'] > bestver:
1429                     bestver = apk['versioncode']
1430                     bestapk = apk
1431
1432                 if 'added' in apk:
1433                     if not app.added or apk['added'] < app.added:
1434                         app.added = apk['added']
1435                     if not app.lastupdated or apk['added'] > app.lastupdated:
1436                         app.lastupdated = apk['added']
1437
1438         if not app.added:
1439             logging.debug("Don't know when " + appid + " was added")
1440         if not app.lastupdated:
1441             logging.debug("Don't know when " + appid + " was last updated")
1442
1443         if bestver == 0:
1444             if app.Name is None:
1445                 app.Name = app.AutoName or appid
1446             app.icon = None
1447             logging.debug("Application " + appid + " has no packages")
1448         else:
1449             if app.Name is None:
1450                 app.Name = bestapk['name']
1451             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1452             if app.CurrentVersionCode is None:
1453                 app.CurrentVersionCode = str(bestver)
1454
1455     # Sort the app list by name, then the web site doesn't have to by default.
1456     # (we had to wait until we'd scanned the apks to do this, because mostly the
1457     # name comes from there!)
1458     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1459
1460     # APKs are placed into multiple repos based on the app package, providing
1461     # per-app subscription feeds for nightly builds and things like it
1462     if config['per_app_repos']:
1463         add_apks_to_per_app_repos(repodirs[0], apks)
1464         for appid, app in apps.items():
1465             repodir = os.path.join(appid, 'fdroid', 'repo')
1466             appdict = dict()
1467             appdict[appid] = app
1468             if os.path.isdir(repodir):
1469                 make_index(appdict, [appid], apks, repodir, False, categories)
1470             else:
1471                 logging.info('Skipping index generation for ' + appid)
1472         return
1473
1474     if len(repodirs) > 1:
1475         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1476
1477     # Make the index for the main repo...
1478     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1479
1480     # If there's an archive repo,  make the index for it. We already scanned it
1481     # earlier on.
1482     if len(repodirs) > 1:
1483         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1484
1485     if config['update_stats']:
1486
1487         # Update known apks info...
1488         knownapks.writeifchanged()
1489
1490         # Generate latest apps data for widget
1491         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1492             data = ''
1493             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1494                 for line in f:
1495                     appid = line.rstrip()
1496                     data += appid + "\t"
1497                     app = apps[appid]
1498                     data += app.Name + "\t"
1499                     if app.icon is not None:
1500                         data += app.icon + "\t"
1501                     data += app.License + "\n"
1502             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1503                 f.write(data)
1504
1505     if cachechanged:
1506         apkcache["METADATA_VERSION"] = METADATA_VERSION
1507         with open(apkcachefile, 'wb') as cf:
1508             pickle.dump(apkcache, cf)
1509
1510     # Update the wiki...
1511     if options.wiki:
1512         update_wiki(apps, sortedids, apks + archapks)
1513
1514     logging.info("Finished.")
1515
1516 if __name__ == "__main__":
1517     main()