fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import urllib.parse
  32 from datetime import datetime, timedelta
  33 from xml.dom.minidom import Document
  34 from argparse import ArgumentParser
  35 import time
  36
  37 import collections
  38 from pyasn1.error import PyAsn1Error
  39 from pyasn1.codec.der import decoder, encoder
  40 from pyasn1_modules import rfc2315
  41 from binascii import hexlify, unhexlify
  42
  43 from PIL import Image
  44 import logging
  45
  46 from . import common
  47 from . import metadata
  48 from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
  49 from .metadata import MetaDataException
  50
  51 METADATA_VERSION = 17
  52
  53 screen_densities = ['640', '480', '320', '240', '160', '120']
  54
  55 all_screen_densities = ['0'] + screen_densities
  56
  57 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  58 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  59
  60
  61 def dpi_to_px(density):
  62     return (int(density) * 48) / 160
  63
  64
  65 def px_to_dpi(px):
  66     return (int(px) * 160) / 48
  67
  68
  69 def get_icon_dir(repodir, density):
  70     if density == '0':
  71         return os.path.join(repodir, "icons")
  72     return os.path.join(repodir, "icons-%s" % density)
  73
  74
  75 def get_icon_dirs(repodir):
  76     for density in screen_densities:
  77         yield get_icon_dir(repodir, density)
  78
  79
  80 def get_all_icon_dirs(repodir):
  81     for density in all_screen_densities:
  82         yield get_icon_dir(repodir, density)
  83
  84
  85 def update_wiki(apps, sortedids, apks):
  86     """Update the wiki
  87
  88     :param apps: fully populated list of all applications
  89     :param apks: all apks, except...
  90     """
  91     logging.info("Updating wiki")
  92     wikicat = 'Apps'
  93     wikiredircat = 'App Redirects'
  94     import mwclient
  95     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  96                          path=config['wiki_path'])
  97     site.login(config['wiki_user'], config['wiki_password'])
  98     generated_pages = {}
  99     generated_redirects = {}
 100
 101     for appid in sortedids:
 102         app = apps[appid]
 103
 104         wikidata = ''
 105         if app.Disabled:
 106             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 107         if app.AntiFeatures:
 108             for af in app.AntiFeatures:
 109                 wikidata += '{{AntiFeature|' + af + '}}\n'
 110         if app.RequiresRoot:
 111             requiresroot = 'Yes'
 112         else:
 113             requiresroot = 'No'
 114         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 115             appid,
 116             app.Name,
 117             time.strftime('%Y-%m-%d', app.added) if app.added else '',
 118             time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
 119             app.SourceCode,
 120             app.IssueTracker,
 121             app.WebSite,
 122             app.Changelog,
 123             app.Donate,
 124             app.FlattrID,
 125             app.Bitcoin,
 126             app.Litecoin,
 127             app.License,
 128             requiresroot,
 129             app.AuthorName,
 130             app.AuthorEmail)
 131
 132         if app.Provides:
 133             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 134
 135         wikidata += app.Summary
 136         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 137
 138         wikidata += "=Description=\n"
 139         wikidata += metadata.description_wiki(app.Description) + "\n"
 140
 141         wikidata += "=Maintainer Notes=\n"
 142         if app.MaintainerNotes:
 143             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 144         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 145
 146         # Get a list of all packages for this application...
 147         apklist = []
 148         gotcurrentver = False
 149         cantupdate = False
 150         buildfails = False
 151         for apk in apks:
 152             if apk['id'] == appid:
 153                 if str(apk['versioncode']) == app.CurrentVersionCode:
 154                     gotcurrentver = True
 155                 apklist.append(apk)
 156         # Include ones we can't build, as a special case...
 157         for build in app.builds:
 158             if build.disable:
 159                 if build.vercode == app.CurrentVersionCode:
 160                     cantupdate = True
 161                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 162                 apklist.append({'versioncode': int(build.vercode),
 163                                 'version': build.version,
 164                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 165                                 })
 166             else:
 167                 builtit = False
 168                 for apk in apklist:
 169                     if apk['versioncode'] == int(build.vercode):
 170                         builtit = True
 171                         break
 172                 if not builtit:
 173                     buildfails = True
 174                     apklist.append({'versioncode': int(build.vercode),
 175                                     'version': build.version,
 176                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
 177                                     })
 178         if app.CurrentVersionCode == '0':
 179             cantupdate = True
 180         # Sort with most recent first...
 181         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 182
 183         wikidata += "=Versions=\n"
 184         if len(apklist) == 0:
 185             wikidata += "We currently have no versions of this app available."
 186         elif not gotcurrentver:
 187             wikidata += "We don't have the current version of this app."
 188         else:
 189             wikidata += "We have the current version of this app."
 190         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 191         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 192         if len(app.NoSourceSince) > 0:
 193             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 194         if len(app.CurrentVersion) > 0:
 195             wikidata += "The current (recommended) version is " + app.CurrentVersion
 196             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 197         validapks = 0
 198         for apk in apklist:
 199             wikidata += "==" + apk['version'] + "==\n"
 200
 201             if 'buildproblem' in apk:
 202                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 203             else:
 204                 validapks += 1
 205                 wikidata += "This version is built and signed by "
 206                 if 'srcname' in apk:
 207                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 208                 else:
 209                     wikidata += "the original developer.\n\n"
 210             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 211
 212         wikidata += '\n[[Category:' + wikicat + ']]\n'
 213         if len(app.NoSourceSince) > 0:
 214             wikidata += '\n[[Category:Apps missing source code]]\n'
 215         if validapks == 0 and not app.Disabled:
 216             wikidata += '\n[[Category:Apps with no packages]]\n'
 217         if cantupdate and not app.Disabled:
 218             wikidata += "\n[[Category:Apps we cannot update]]\n"
 219         if buildfails and not app.Disabled:
 220             wikidata += "\n[[Category:Apps with failing builds]]\n"
 221         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 222             wikidata += '\n[[Category:Apps to Update]]\n'
 223         if app.Disabled:
 224             wikidata += '\n[[Category:Apps that are disabled]]\n'
 225         if app.UpdateCheckMode == 'None' and not app.Disabled:
 226             wikidata += '\n[[Category:Apps with no update check]]\n'
 227         for appcat in app.Categories:
 228             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 229
 230         # We can't have underscores in the page name, even if they're in
 231         # the package ID, because MediaWiki messes with them...
 232         pagename = appid.replace('_', ' ')
 233
 234         # Drop a trailing newline, because mediawiki is going to drop it anyway
 235         # and it we don't we'll think the page has changed when it hasn't...
 236         if wikidata.endswith('\n'):
 237             wikidata = wikidata[:-1]
 238
 239         generated_pages[pagename] = wikidata
 240
 241         # Make a redirect from the name to the ID too, unless there's
 242         # already an existing page with the name and it isn't a redirect.
 243         noclobber = False
 244         apppagename = app.Name.replace('_', ' ')
 245         apppagename = apppagename.replace('{', '')
 246         apppagename = apppagename.replace('}', ' ')
 247         apppagename = apppagename.replace(':', ' ')
 248         apppagename = apppagename.replace('[', ' ')
 249         apppagename = apppagename.replace(']', ' ')
 250         # Drop double spaces caused mostly by replacing ':' above
 251         apppagename = apppagename.replace('  ', ' ')
 252         for expagename in site.allpages(prefix=apppagename,
 253                                         filterredir='nonredirects',
 254                                         generator=False):
 255             if expagename == apppagename:
 256                 noclobber = True
 257         # Another reason not to make the redirect page is if the app name
 258         # is the same as it's ID, because that will overwrite the real page
 259         # with an redirect to itself! (Although it seems like an odd
 260         # scenario this happens a lot, e.g. where there is metadata but no
 261         # builds or binaries to extract a name from.
 262         if apppagename == pagename:
 263             noclobber = True
 264         if not noclobber:
 265             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 266
 267     for tcat, genp in [(wikicat, generated_pages),
 268                        (wikiredircat, generated_redirects)]:
 269         catpages = site.Pages['Category:' + tcat]
 270         existingpages = []
 271         for page in catpages:
 272             existingpages.append(page.name)
 273             if page.name in genp:
 274                 pagetxt = page.edit()
 275                 if pagetxt != genp[page.name]:
 276                     logging.debug("Updating modified page " + page.name)
 277                     page.save(genp[page.name], summary='Auto-updated')
 278                 else:
 279                     logging.debug("Page " + page.name + " is unchanged")
 280             else:
 281                 logging.warn("Deleting page " + page.name)
 282                 page.delete('No longer published')
 283         for pagename, text in genp.items():
 284             logging.debug("Checking " + pagename)
 285             if pagename not in existingpages:
 286                 logging.debug("Creating page " + pagename)
 287                 try:
 288                     newpage = site.Pages[pagename]
 289                     newpage.save(text, summary='Auto-created')
 290                 except:
 291                     logging.error("...FAILED to create page '{0}'".format(pagename))
 292
 293     # Purge server cache to ensure counts are up to date
 294     site.pages['Repository Maintenance'].purge()
 295
 296
 297 def delete_disabled_builds(apps, apkcache, repodirs):
 298     """Delete disabled build outputs.
 299
 300     :param apps: list of all applications, as per metadata.read_metadata
 301     :param apkcache: current apk cache information
 302     :param repodirs: the repo directories to process
 303     """
 304     for appid, app in apps.items():
 305         for build in app.builds:
 306             if not build.disable:
 307                 continue
 308             apkfilename = appid + '_' + str(build.vercode) + '.apk'
 309             iconfilename = "%s.%s.png" % (
 310                 appid,
 311                 build.vercode)
 312             for repodir in repodirs:
 313                 files = [
 314                     os.path.join(repodir, apkfilename),
 315                     os.path.join(repodir, apkfilename + '.asc'),
 316                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 317                 ]
 318                 for density in all_screen_densities:
 319                     repo_dir = get_icon_dir(repodir, density)
 320                     files.append(os.path.join(repo_dir, iconfilename))
 321
 322                 for f in files:
 323                     if os.path.exists(f):
 324                         logging.info("Deleting disabled build output " + f)
 325                         os.remove(f)
 326             if apkfilename in apkcache:
 327                 del apkcache[apkfilename]
 328
 329
 330 def resize_icon(iconpath, density):
 331
 332     if not os.path.isfile(iconpath):
 333         return
 334
 335     fp = None
 336     try:
 337         fp = open(iconpath, 'rb')
 338         im = Image.open(fp)
 339         size = dpi_to_px(density)
 340
 341         if any(length > size for length in im.size):
 342             oldsize = im.size
 343             im.thumbnail((size, size), Image.ANTIALIAS)
 344             logging.debug("%s was too large at %s - new size is %s" % (
 345                 iconpath, oldsize, im.size))
 346             im.save(iconpath, "PNG")
 347
 348     except Exception as e:
 349         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 350
 351     finally:
 352         if fp:
 353             fp.close()
 354
 355
 356 def resize_all_icons(repodirs):
 357     """Resize all icons that exceed the max size
 358
 359     :param repodirs: the repo directories to process
 360     """
 361     for repodir in repodirs:
 362         for density in screen_densities:
 363             icon_dir = get_icon_dir(repodir, density)
 364             icon_glob = os.path.join(icon_dir, '*.png')
 365             for iconpath in glob.glob(icon_glob):
 366                 resize_icon(iconpath, density)
 367
 368
 369 # A signature block file with a .DSA, .RSA, or .EC extension
 370 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 371
 372
 373 def getsig(apkpath):
 374     """ Get the signing certificate of an apk. To get the same md5 has that
 375     Android gets, we encode the .RSA certificate in a specific format and pass
 376     it hex-encoded to the md5 digest algorithm.
 377
 378     :param apkpath: path to the apk
 379     :returns: A string containing the md5 of the signature of the apk or None
 380               if an error occurred.
 381     """
 382
 383     cert = None
 384
 385     # verify the jar signature is correct
 386     args = [config['jarsigner'], '-verify', apkpath]
 387     p = FDroidPopen(args)
 388     if p.returncode != 0:
 389         logging.critical(apkpath + " has a bad signature!")
 390         return None
 391
 392     with zipfile.ZipFile(apkpath, 'r') as apk:
 393
 394         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 395
 396         if len(certs) < 1:
 397             logging.error("Found no signing certificates on %s" % apkpath)
 398             return None
 399         if len(certs) > 1:
 400             logging.error("Found multiple signing certificates on %s" % apkpath)
 401             return None
 402
 403         cert = apk.read(certs[0])
 404
 405     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 406     if content.getComponentByName('contentType') != rfc2315.signedData:
 407         logging.error("Unexpected format.")
 408         return None
 409
 410     content = decoder.decode(content.getComponentByName('content'),
 411                              asn1Spec=rfc2315.SignedData())[0]
 412     try:
 413         certificates = content.getComponentByName('certificates')
 414     except PyAsn1Error:
 415         logging.error("Certificates not found.")
 416         return None
 417
 418     cert_encoded = encoder.encode(certificates)[4:]
 419
 420     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 421
 422
 423 def get_icon_bytes(apkzip, iconsrc):
 424     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 425     try:
 426         return apkzip.read(iconsrc)
 427     except KeyError:
 428         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 429
 430
 431 def sha256sum(filename):
 432     '''Calculate the sha256 of the given file'''
 433     sha = hashlib.sha256()
 434     with open(filename, 'rb') as f:
 435         while True:
 436             t = f.read(16384)
 437             if len(t) == 0:
 438                 break
 439             sha.update(t)
 440     return sha.hexdigest()
 441
 442
 443 def insert_obbs(repodir, apps, apks):
 444     """Scans the .obb files in a given repo directory and adds them to the
 445     relevant APK instances.  OBB files have versionCodes like APK
 446     files, and they are loosely associated.  If there is an OBB file
 447     present, then any APK with the same or higher versionCode will use
 448     that OBB file.  There are two OBB types: main and patch, each APK
 449     can only have only have one of each.
 450
 451     https://developer.android.com/google/play/expansion-files.html
 452
 453     :param repodir: repo directory to scan
 454     :param apps: list of current, valid apps
 455     :param apks: current information on all APKs
 456
 457     """
 458
 459     def obbWarnDelete(f, msg):
 460         logging.warning(msg + f)
 461         if options.delete_unknown:
 462             logging.error("Deleting unknown file: " + f)
 463             os.remove(f)
 464
 465     obbs = []
 466     java_Integer_MIN_VALUE = -pow(2, 31)
 467     for f in glob.glob(os.path.join(repodir, '*.obb')):
 468         obbfile = os.path.basename(f)
 469         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 470         chunks = obbfile.split('.')
 471         if chunks[0] != 'main' and chunks[0] != 'patch':
 472             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 473             continue
 474         if not re.match(r'^-?[0-9]+$', chunks[1]):
 475             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 476             continue
 477         versioncode = int(chunks[1])
 478         packagename = ".".join(chunks[2:-1])
 479
 480         highestVersionCode = java_Integer_MIN_VALUE
 481         if packagename not in apps.keys():
 482             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 483             continue
 484         for apk in apks:
 485             if packagename == apk['id'] and apk['versioncode'] > highestVersionCode:
 486                 highestVersionCode = apk['versioncode']
 487         if versioncode > highestVersionCode:
 488             obbWarnDelete(f, 'OBB file has newer versioncode(' + str(versioncode)
 489                           + ') than any APK: ')
 490             continue
 491         obbsha256 = sha256sum(f)
 492         obbs.append((packagename, versioncode, obbfile, obbsha256))
 493
 494     for apk in apks:
 495         for (packagename, versioncode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 496             if versioncode <= apk['versioncode'] and packagename == apk['id']:
 497                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 498                     apk['obbMainFile'] = obbfile
 499                     apk['obbMainFileSha256'] = obbsha256
 500                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 501                     apk['obbPatchFile'] = obbfile
 502                     apk['obbPatchFileSha256'] = obbsha256
 503             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 504                 break
 505
 506
 507 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
 508     """Scan the apks in the given repo directory.
 509
 510     This also extracts the icons.
 511
 512     :param apps: list of all applications, as per metadata.read_metadata
 513     :param apkcache: current apk cache information
 514     :param repodir: repo directory to scan
 515     :param knownapks: known apks info
 516     :param use_date_from_apk: use date from APK (instead of current date)
 517                               for newly added APKs
 518     :returns: (apks, cachechanged) where apks is a list of apk information,
 519               and cachechanged is True if the apkcache got changed.
 520     """
 521
 522     cachechanged = False
 523
 524     for icon_dir in get_all_icon_dirs(repodir):
 525         if os.path.exists(icon_dir):
 526             if options.clean:
 527                 shutil.rmtree(icon_dir)
 528                 os.makedirs(icon_dir)
 529         else:
 530             os.makedirs(icon_dir)
 531
 532     apks = []
 533     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 534     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 535     vername_pat = re.compile(".*versionName='([^']*)'.*")
 536     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 537     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 538     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 539     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 540     permission_pat = re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
 541     feature_pat = re.compile(".*name='([^']*)'.*")
 542     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 543
 544         apkfilename = apkfile[len(repodir) + 1:]
 545         if ' ' in apkfilename:
 546             logging.critical("Spaces in filenames are not allowed.")
 547             sys.exit(1)
 548
 549         shasum = sha256sum(apkfile)
 550
 551         usecache = False
 552         if apkfilename in apkcache:
 553             apk = apkcache[apkfilename]
 554             if apk['sha256'] == shasum:
 555                 logging.debug("Reading " + apkfilename + " from cache")
 556                 usecache = True
 557             else:
 558                 logging.debug("Ignoring stale cache data for " + apkfilename)
 559
 560         if not usecache:
 561             logging.debug("Processing " + apkfilename)
 562             apk = {}
 563             apk['apkname'] = apkfilename
 564             apk['sha256'] = shasum
 565             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 566             if os.path.exists(os.path.join(repodir, srcfilename)):
 567                 apk['srcname'] = srcfilename
 568             apk['size'] = os.path.getsize(apkfile)
 569             apk['uses-permission'] = set()
 570             apk['uses-permission-sdk-23'] = set()
 571             apk['features'] = set()
 572             apk['icons_src'] = {}
 573             apk['icons'] = {}
 574             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 575             if p.returncode != 0:
 576                 if options.delete_unknown:
 577                     if os.path.exists(apkfile):
 578                         logging.error("Failed to get apk information, deleting " + apkfile)
 579                         os.remove(apkfile)
 580                     else:
 581                         logging.error("Could not find {0} to remove it".format(apkfile))
 582                 else:
 583                     logging.error("Failed to get apk information, skipping " + apkfile)
 584                 continue
 585             for line in p.output.splitlines():
 586                 if line.startswith("package:"):
 587                     try:
 588                         apk['id'] = re.match(name_pat, line).group(1)
 589                         apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
 590                         apk['version'] = re.match(vername_pat, line).group(1)
 591                     except Exception as e:
 592                         logging.error("Package matching failed: " + str(e))
 593                         logging.info("Line was: " + line)
 594                         sys.exit(1)
 595                 elif line.startswith("application:"):
 596                     apk['name'] = re.match(label_pat, line).group(1)
 597                     # Keep path to non-dpi icon in case we need it
 598                     match = re.match(icon_pat_nodpi, line)
 599                     if match:
 600                         apk['icons_src']['-1'] = match.group(1)
 601                 elif line.startswith("launchable-activity:"):
 602                     # Only use launchable-activity as fallback to application
 603                     if not apk['name']:
 604                         apk['name'] = re.match(label_pat, line).group(1)
 605                     if '-1' not in apk['icons_src']:
 606                         match = re.match(icon_pat_nodpi, line)
 607                         if match:
 608                             apk['icons_src']['-1'] = match.group(1)
 609                 elif line.startswith("application-icon-"):
 610                     match = re.match(icon_pat, line)
 611                     if match:
 612                         density = match.group(1)
 613                         path = match.group(2)
 614                         apk['icons_src'][density] = path
 615                 elif line.startswith("sdkVersion:"):
 616                     m = re.match(sdkversion_pat, line)
 617                     if m is None:
 618                         logging.error(line.replace('sdkVersion:', '')
 619                                       + ' is not a valid minSdkVersion!')
 620                     else:
 621                         apk['minSdkVersion'] = m.group(1)
 622                         # if target not set, default to min
 623                         if 'targetSdkVersion' not in apk:
 624                             apk['targetSdkVersion'] = m.group(1)
 625                 elif line.startswith("targetSdkVersion:"):
 626                     m = re.match(sdkversion_pat, line)
 627                     if m is None:
 628                         logging.error(line.replace('targetSdkVersion:', '')
 629                                       + ' is not a valid targetSdkVersion!')
 630                     else:
 631                         apk['targetSdkVersion'] = m.group(1)
 632                 elif line.startswith("maxSdkVersion:"):
 633                     apk['maxSdkVersion'] = re.match(sdkversion_pat, line).group(1)
 634                 elif line.startswith("native-code:"):
 635                     apk['nativecode'] = []
 636                     for arch in line[13:].split(' '):
 637                         apk['nativecode'].append(arch[1:-1])
 638                 elif line.startswith('uses-permission:'):
 639                     perm_match = re.match(permission_pat, line).groupdict()
 640
 641                     permission = UsesPermission(
 642                         perm_match['name'],
 643                         perm_match['maxSdkVersion']
 644                     )
 645
 646                     apk['uses-permission'].add(permission)
 647                 elif line.startswith('uses-permission-sdk-23:'):
 648                     perm_match = re.match(permission_pat, line).groupdict()
 649
 650                     permission_sdk_23 = UsesPermissionSdk23(
 651                         perm_match['name'],
 652                         perm_match['maxSdkVersion']
 653                     )
 654
 655                     apk['uses-permission-sdk-23'].add(permission_sdk_23)
 656
 657                 elif line.startswith('uses-feature:'):
 658                     feature = re.match(feature_pat, line).group(1)
 659                     # Filter out this, it's only added with the latest SDK tools and
 660                     # causes problems for lots of apps.
 661                     if feature != "android.hardware.screen.portrait" \
 662                             and feature != "android.hardware.screen.landscape":
 663                         if feature.startswith("android.feature."):
 664                             feature = feature[16:]
 665                         apk['features'].add(feature)
 666
 667             if 'minSdkVersion' not in apk:
 668                 logging.warn("No SDK version information found in {0}".format(apkfile))
 669                 apk['minSdkVersion'] = 1
 670
 671             # Check for debuggable apks...
 672             if common.isApkDebuggable(apkfile, config):
 673                 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
 674
 675             # Get the signature (or md5 of, to be precise)...
 676             logging.debug('Getting signature of {0}'.format(apkfile))
 677             apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 678             if not apk['sig']:
 679                 logging.critical("Failed to get apk signature")
 680                 sys.exit(1)
 681
 682             apkzip = zipfile.ZipFile(apkfile, 'r')
 683
 684             # if an APK has files newer than the system time, suggest updating
 685             # the system clock.  This is useful for offline systems, used for
 686             # signing, which do not have another source of clock sync info. It
 687             # has to be more than 24 hours newer because ZIP/APK files do not
 688             # store timezone info
 689             manifest = apkzip.getinfo('AndroidManifest.xml')
 690             if manifest.date_time[1] == 0:  # month can't be zero
 691                 logging.debug('AndroidManifest.xml has no date')
 692             else:
 693                 dt_obj = datetime(*manifest.date_time)
 694                 checkdt = dt_obj - timedelta(1)
 695                 if datetime.today() < checkdt:
 696                     logging.warn('System clock is older than manifest in: '
 697                                  + apkfilename
 698                                  + '\nSet clock to that time using:\n'
 699                                  + 'sudo date -s "' + str(dt_obj) + '"')
 700
 701             iconfilename = "%s.%s.png" % (
 702                 apk['id'],
 703                 apk['versioncode'])
 704
 705             # Extract the icon file...
 706             empty_densities = []
 707             for density in screen_densities:
 708                 if density not in apk['icons_src']:
 709                     empty_densities.append(density)
 710                     continue
 711                 iconsrc = apk['icons_src'][density]
 712                 icon_dir = get_icon_dir(repodir, density)
 713                 icondest = os.path.join(icon_dir, iconfilename)
 714
 715                 try:
 716                     with open(icondest, 'wb') as f:
 717                         f.write(get_icon_bytes(apkzip, iconsrc))
 718                     apk['icons'][density] = iconfilename
 719
 720                 except:
 721                     logging.warn("Error retrieving icon file")
 722                     del apk['icons'][density]
 723                     del apk['icons_src'][density]
 724                     empty_densities.append(density)
 725
 726             if '-1' in apk['icons_src']:
 727                 iconsrc = apk['icons_src']['-1']
 728                 iconpath = os.path.join(
 729                     get_icon_dir(repodir, '0'), iconfilename)
 730                 with open(iconpath, 'wb') as f:
 731                     f.write(get_icon_bytes(apkzip, iconsrc))
 732                 try:
 733                     im = Image.open(iconpath)
 734                     dpi = px_to_dpi(im.size[0])
 735                     for density in screen_densities:
 736                         if density in apk['icons']:
 737                             break
 738                         if density == screen_densities[-1] or dpi >= int(density):
 739                             apk['icons'][density] = iconfilename
 740                             shutil.move(iconpath,
 741                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 742                             empty_densities.remove(density)
 743                             break
 744                 except Exception as e:
 745                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 746
 747             if apk['icons']:
 748                 apk['icon'] = iconfilename
 749
 750             apkzip.close()
 751
 752             # First try resizing down to not lose quality
 753             last_density = None
 754             for density in screen_densities:
 755                 if density not in empty_densities:
 756                     last_density = density
 757                     continue
 758                 if last_density is None:
 759                     continue
 760                 logging.debug("Density %s not available, resizing down from %s"
 761                               % (density, last_density))
 762
 763                 last_iconpath = os.path.join(
 764                     get_icon_dir(repodir, last_density), iconfilename)
 765                 iconpath = os.path.join(
 766                     get_icon_dir(repodir, density), iconfilename)
 767                 fp = None
 768                 try:
 769                     fp = open(last_iconpath, 'rb')
 770                     im = Image.open(fp)
 771
 772                     size = dpi_to_px(density)
 773
 774                     im.thumbnail((size, size), Image.ANTIALIAS)
 775                     im.save(iconpath, "PNG")
 776                     empty_densities.remove(density)
 777                 except:
 778                     logging.warning("Invalid image file at %s" % last_iconpath)
 779                 finally:
 780                     if fp:
 781                         fp.close()
 782
 783             # Then just copy from the highest resolution available
 784             last_density = None
 785             for density in reversed(screen_densities):
 786                 if density not in empty_densities:
 787                     last_density = density
 788                     continue
 789                 if last_density is None:
 790                     continue
 791                 logging.debug("Density %s not available, copying from lower density %s"
 792                               % (density, last_density))
 793
 794                 shutil.copyfile(
 795                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 796                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 797
 798                 empty_densities.remove(density)
 799
 800             for density in screen_densities:
 801                 icon_dir = get_icon_dir(repodir, density)
 802                 icondest = os.path.join(icon_dir, iconfilename)
 803                 resize_icon(icondest, density)
 804
 805             # Copy from icons-mdpi to icons since mdpi is the baseline density
 806             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 807             if os.path.isfile(baseline):
 808                 apk['icons']['0'] = iconfilename
 809                 shutil.copyfile(baseline,
 810                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 811
 812             if use_date_from_apk and manifest.date_time[1] != 0:
 813                 default_date_param = datetime(*manifest.date_time).utctimetuple()
 814             else:
 815                 default_date_param = None
 816
 817             # Record in known apks, getting the added date at the same time..
 818             added = knownapks.recordapk(apk['apkname'], apk['id'], default_date=default_date_param)
 819             if added:
 820                 apk['added'] = added
 821
 822             apkcache[apkfilename] = apk
 823             cachechanged = True
 824
 825         apks.append(apk)
 826
 827     return apks, cachechanged
 828
 829
 830 repo_pubkey_fingerprint = None
 831
 832
 833 # Generate a certificate fingerprint the same way keytool does it
 834 # (but with slightly different formatting)
 835 def cert_fingerprint(data):
 836     digest = hashlib.sha256(data).digest()
 837     ret = []
 838     ret.append(' '.join("%02X" % b for b in bytearray(digest)))
 839     return " ".join(ret)
 840
 841
 842 def extract_pubkey():
 843     global repo_pubkey_fingerprint
 844     if 'repo_pubkey' in config:
 845         pubkey = unhexlify(config['repo_pubkey'])
 846     else:
 847         p = FDroidPopenBytes([config['keytool'], '-exportcert',
 848                               '-alias', config['repo_keyalias'],
 849                               '-keystore', config['keystore'],
 850                               '-storepass:file', config['keystorepassfile']]
 851                              + config['smartcardoptions'],
 852                              output=False, stderr_to_stdout=False)
 853         if p.returncode != 0 or len(p.output) < 20:
 854             msg = "Failed to get repo pubkey!"
 855             if config['keystore'] == 'NONE':
 856                 msg += ' Is your crypto smartcard plugged in?'
 857             logging.critical(msg)
 858             sys.exit(1)
 859         pubkey = p.output
 860     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 861     return hexlify(pubkey)
 862
 863
 864 def make_index(apps, sortedids, apks, repodir, archive, categories):
 865     """Make a repo index.
 866
 867     :param apps: fully populated apps list
 868     :param apks: full populated apks list
 869     :param repodir: the repo directory
 870     :param archive: True if this is the archive repo, False if it's the
 871                     main one.
 872     :param categories: list of categories
 873     """
 874
 875     doc = Document()
 876
 877     def addElement(name, value, doc, parent):
 878         el = doc.createElement(name)
 879         el.appendChild(doc.createTextNode(value))
 880         parent.appendChild(el)
 881
 882     def addElementNonEmpty(name, value, doc, parent):
 883         if not value:
 884             return
 885         addElement(name, value, doc, parent)
 886
 887     def addElementCDATA(name, value, doc, parent):
 888         el = doc.createElement(name)
 889         el.appendChild(doc.createCDATASection(value))
 890         parent.appendChild(el)
 891
 892     root = doc.createElement("fdroid")
 893     doc.appendChild(root)
 894
 895     repoel = doc.createElement("repo")
 896
 897     mirrorcheckfailed = False
 898     mirrors = []
 899     for mirror in config.get('mirrors', []):
 900         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
 901         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
 902             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 903             mirrorcheckfailed = True
 904         # must end with / or urljoin strips a whole path segment
 905         if mirror.endswith('/'):
 906             mirrors.append(mirror)
 907         else:
 908             mirrors.append(mirror + '/')
 909     if mirrorcheckfailed:
 910         sys.exit(1)
 911
 912     if archive:
 913         repoel.setAttribute("name", config['archive_name'])
 914         if config['repo_maxage'] != 0:
 915             repoel.setAttribute("maxage", str(config['repo_maxage']))
 916         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 917         repoel.setAttribute("url", config['archive_url'])
 918         addElement('description', config['archive_description'], doc, repoel)
 919         urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
 920         for mirror in mirrors:
 921             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 922
 923     else:
 924         repoel.setAttribute("name", config['repo_name'])
 925         if config['repo_maxage'] != 0:
 926             repoel.setAttribute("maxage", str(config['repo_maxage']))
 927         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 928         repoel.setAttribute("url", config['repo_url'])
 929         addElement('description', config['repo_description'], doc, repoel)
 930         urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
 931         for mirror in mirrors:
 932             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 933
 934     repoel.setAttribute("version", str(METADATA_VERSION))
 935     repoel.setAttribute("timestamp", str(int(time.time())))
 936
 937     nosigningkey = False
 938     if not options.nosign:
 939         if 'repo_keyalias' not in config:
 940             nosigningkey = True
 941             logging.critical("'repo_keyalias' not found in config.py!")
 942         if 'keystore' not in config:
 943             nosigningkey = True
 944             logging.critical("'keystore' not found in config.py!")
 945         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 946             nosigningkey = True
 947             logging.critical("'keystorepass' not found in config.py!")
 948         if 'keypass' not in config and 'keypassfile' not in config:
 949             nosigningkey = True
 950             logging.critical("'keypass' not found in config.py!")
 951         if not os.path.exists(config['keystore']):
 952             nosigningkey = True
 953             logging.critical("'" + config['keystore'] + "' does not exist!")
 954         if nosigningkey:
 955             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 956             logging.warning("\tfdroid update --create-key")
 957             sys.exit(1)
 958
 959     repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
 960     root.appendChild(repoel)
 961
 962     for command in ('install', 'uninstall'):
 963         packageNames = []
 964         key = command + '_list'
 965         if key in config:
 966             if isinstance(config[key], str):
 967                 packageNames = [config[key]]
 968             elif all(isinstance(item, str) for item in config[key]):
 969                 packageNames = config[key]
 970             else:
 971                 raise TypeError('only accepts strings, lists, and tuples')
 972         for packageName in packageNames:
 973             element = doc.createElement(command)
 974             root.appendChild(element)
 975             element.setAttribute('packageName', packageName)
 976
 977     for appid in sortedids:
 978         app = apps[appid]
 979
 980         if app.Disabled is not None:
 981             continue
 982
 983         # Get a list of the apks for this app...
 984         apklist = []
 985         for apk in apks:
 986             if apk['id'] == appid:
 987                 apklist.append(apk)
 988
 989         if len(apklist) == 0:
 990             continue
 991
 992         apel = doc.createElement("application")
 993         apel.setAttribute("id", app.id)
 994         root.appendChild(apel)
 995
 996         addElement('id', app.id, doc, apel)
 997         if app.added:
 998             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
 999         if app.lastupdated:
1000             addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
1001         addElement('name', app.Name, doc, apel)
1002         addElement('summary', app.Summary, doc, apel)
1003         if app.icon:
1004             addElement('icon', app.icon, doc, apel)
1005
1006         def linkres(appid):
1007             if appid in apps:
1008                 return ("fdroid.app:" + appid, apps[appid].Name)
1009             raise MetaDataException("Cannot resolve app id " + appid)
1010
1011         addElement('desc',
1012                    metadata.description_html(app.Description, linkres),
1013                    doc, apel)
1014         addElement('license', app.License, doc, apel)
1015         if app.Categories:
1016             addElement('categories', ','.join(app.Categories), doc, apel)
1017             # We put the first (primary) category in LAST, which will have
1018             # the desired effect of making clients that only understand one
1019             # category see that one.
1020             addElement('category', app.Categories[0], doc, apel)
1021         addElement('web', app.WebSite, doc, apel)
1022         addElement('source', app.SourceCode, doc, apel)
1023         addElement('tracker', app.IssueTracker, doc, apel)
1024         addElementNonEmpty('changelog', app.Changelog, doc, apel)
1025         addElementNonEmpty('author', app.AuthorName, doc, apel)
1026         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
1027         addElementNonEmpty('donate', app.Donate, doc, apel)
1028         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
1029         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
1030         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
1031
1032         # These elements actually refer to the current version (i.e. which
1033         # one is recommended. They are historically mis-named, and need
1034         # changing, but stay like this for now to support existing clients.
1035         addElement('marketversion', app.CurrentVersion, doc, apel)
1036         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
1037
1038         if app.AntiFeatures:
1039             af = app.AntiFeatures
1040             if af:
1041                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
1042         if app.Provides:
1043             pv = app.Provides.split(',')
1044             addElementNonEmpty('provides', ','.join(pv), doc, apel)
1045         if app.RequiresRoot:
1046             addElement('requirements', 'root', doc, apel)
1047
1048         # Sort the apk list into version order, just so the web site
1049         # doesn't have to do any work by default...
1050         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
1051
1052         # Check for duplicates - they will make the client unhappy...
1053         for i in range(len(apklist) - 1):
1054             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
1055                 logging.critical("duplicate versions: '%s' - '%s'" % (
1056                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
1057                 sys.exit(1)
1058
1059         current_version_code = 0
1060         current_version_file = None
1061         for apk in apklist:
1062             # find the APK for the "Current Version"
1063             if current_version_code < apk['versioncode']:
1064                 current_version_code = apk['versioncode']
1065             if current_version_code < int(app.CurrentVersionCode):
1066                 current_version_file = apk['apkname']
1067
1068             apkel = doc.createElement("package")
1069             apel.appendChild(apkel)
1070             addElement('version', apk['version'], doc, apkel)
1071             addElement('versioncode', str(apk['versioncode']), doc, apkel)
1072             addElement('apkname', apk['apkname'], doc, apkel)
1073             if 'srcname' in apk:
1074                 addElement('srcname', apk['srcname'], doc, apkel)
1075             for hash_type in ['sha256']:
1076                 if hash_type not in apk:
1077                     continue
1078                 hashel = doc.createElement("hash")
1079                 hashel.setAttribute("type", hash_type)
1080                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
1081                 apkel.appendChild(hashel)
1082             addElement('sig', apk['sig'], doc, apkel)
1083             addElement('size', str(apk['size']), doc, apkel)
1084             addElement('sdkver', str(apk['minSdkVersion']), doc, apkel)
1085             if 'targetSdkVersion' in apk:
1086                 addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
1087             if 'maxSdkVersion' in apk:
1088                 addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
1089             addElementNonEmpty('obbMainFile', apk.get('obbMainFile'), doc, apkel)
1090             addElementNonEmpty('obbMainFileSha256', apk.get('obbMainFileSha256'), doc, apkel)
1091             addElementNonEmpty('obbPatchFile', apk.get('obbPatchFile'), doc, apkel)
1092             addElementNonEmpty('obbPatchFileSha256', apk.get('obbPatchFileSha256'), doc, apkel)
1093             if 'added' in apk:
1094                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
1095
1096             # TODO: remove old permission format
1097             old_permissions = set()
1098             for perm in apk['uses-permission']:
1099                 perm_name = perm.name
1100                 if perm_name.startswith("android.permission."):
1101                     perm_name = perm_name[19:]
1102                 old_permissions.add(perm_name)
1103             addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel)
1104
1105             for permission in apk['uses-permission']:
1106                 permel = doc.createElement('uses-permission')
1107                 permel.setAttribute('name', permission.name)
1108                 if permission.maxSdkVersion is not None:
1109                     permel.setAttribute('maxSdkVersion', permission.maxSdkVersion)
1110                     apkel.appendChild(permel)
1111             for permission_sdk_23 in apk['uses-permission-sdk-23']:
1112                 permel = doc.createElement('uses-permission-sdk-23')
1113                 permel.setAttribute('name', permission_sdk_23.name)
1114                 if permission_sdk_23.maxSdkVersion is not None:
1115                     permel.setAttribute('maxSdkVersion', permission_sdk_23.maxSdkVersion)
1116                     apkel.appendChild(permel)
1117             if 'nativecode' in apk:
1118                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
1119             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
1120
1121         if current_version_file is not None \
1122                 and config['make_current_version_link'] \
1123                 and repodir == 'repo':  # only create these
1124             namefield = config['current_version_name_source']
1125             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
1126             apklinkname = sanitized_name + '.apk'
1127             current_version_path = os.path.join(repodir, current_version_file)
1128             if os.path.islink(apklinkname):
1129                 os.remove(apklinkname)
1130             os.symlink(current_version_path, apklinkname)
1131             # also symlink gpg signature, if it exists
1132             for extension in ('.asc', '.sig'):
1133                 sigfile_path = current_version_path + extension
1134                 if os.path.exists(sigfile_path):
1135                     siglinkname = apklinkname + extension
1136                     if os.path.islink(siglinkname):
1137                         os.remove(siglinkname)
1138                     os.symlink(sigfile_path, siglinkname)
1139
1140     if options.pretty:
1141         output = doc.toprettyxml(encoding='utf-8')
1142     else:
1143         output = doc.toxml(encoding='utf-8')
1144
1145     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
1146         f.write(output)
1147
1148     if 'repo_keyalias' in config:
1149
1150         if options.nosign:
1151             logging.info("Creating unsigned index in preparation for signing")
1152         else:
1153             logging.info("Creating signed index with this key (SHA256):")
1154             logging.info("%s" % repo_pubkey_fingerprint)
1155
1156         # Create a jar of the index...
1157         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
1158         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
1159         if p.returncode != 0:
1160             logging.critical("Failed to create {0}".format(jar_output))
1161             sys.exit(1)
1162
1163         # Sign the index...
1164         signed = os.path.join(repodir, 'index.jar')
1165         if options.nosign:
1166             # Remove old signed index if not signing
1167             if os.path.exists(signed):
1168                 os.remove(signed)
1169         else:
1170             args = [config['jarsigner'], '-keystore', config['keystore'],
1171                     '-storepass:file', config['keystorepassfile'],
1172                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1173                     signed, config['repo_keyalias']]
1174             if config['keystore'] == 'NONE':
1175                 args += config['smartcardoptions']
1176             else:  # smardcards never use -keypass
1177                 args += ['-keypass:file', config['keypassfile']]
1178             p = FDroidPopen(args)
1179             if p.returncode != 0:
1180                 logging.critical("Failed to sign index")
1181                 sys.exit(1)
1182
1183     # Copy the repo icon into the repo directory...
1184     icon_dir = os.path.join(repodir, 'icons')
1185     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1186     shutil.copyfile(config['repo_icon'], iconfilename)
1187
1188     # Write a category list in the repo to allow quick access...
1189     catdata = ''
1190     for cat in categories:
1191         catdata += cat + '\n'
1192     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1193         f.write(catdata)
1194
1195
1196 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1197
1198     for appid, app in apps.items():
1199
1200         if app.ArchivePolicy:
1201             keepversions = int(app.ArchivePolicy[:-9])
1202         else:
1203             keepversions = defaultkeepversions
1204
1205         def filter_apk_list_sorted(apk_list):
1206             res = []
1207             for apk in apk_list:
1208                 if apk['id'] == appid:
1209                     res.append(apk)
1210
1211             # Sort the apk list by version code. First is highest/newest.
1212             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1213
1214         def move_file(from_dir, to_dir, filename, ignore_missing):
1215             from_path = os.path.join(from_dir, filename)
1216             if ignore_missing and not os.path.exists(from_path):
1217                 return
1218             to_path = os.path.join(to_dir, filename)
1219             shutil.move(from_path, to_path)
1220
1221         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1222                       .format(appid, len(apks), keepversions, len(archapks)))
1223
1224         if len(apks) > keepversions:
1225             apklist = filter_apk_list_sorted(apks)
1226             # Move back the ones we don't want.
1227             for apk in apklist[keepversions:]:
1228                 logging.info("Moving " + apk['apkname'] + " to archive")
1229                 move_file(repodir, archivedir, apk['apkname'], False)
1230                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1231                 for density in all_screen_densities:
1232                     repo_icon_dir = get_icon_dir(repodir, density)
1233                     archive_icon_dir = get_icon_dir(archivedir, density)
1234                     if density not in apk['icons']:
1235                         continue
1236                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1237                 if 'srcname' in apk:
1238                     move_file(repodir, archivedir, apk['srcname'], False)
1239                 archapks.append(apk)
1240                 apks.remove(apk)
1241         elif len(apks) < keepversions and len(archapks) > 0:
1242             required = keepversions - len(apks)
1243             archapklist = filter_apk_list_sorted(archapks)
1244             # Move forward the ones we want again.
1245             for apk in archapklist[:required]:
1246                 logging.info("Moving " + apk['apkname'] + " from archive")
1247                 move_file(archivedir, repodir, apk['apkname'], False)
1248                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1249                 for density in all_screen_densities:
1250                     repo_icon_dir = get_icon_dir(repodir, density)
1251                     archive_icon_dir = get_icon_dir(archivedir, density)
1252                     if density not in apk['icons']:
1253                         continue
1254                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1255                 if 'srcname' in apk:
1256                     move_file(archivedir, repodir, apk['srcname'], False)
1257                 archapks.remove(apk)
1258                 apks.append(apk)
1259
1260
1261 def add_apks_to_per_app_repos(repodir, apks):
1262     apks_per_app = dict()
1263     for apk in apks:
1264         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1265         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1266         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1267         apks_per_app[apk['id']] = apk
1268
1269         if not os.path.exists(apk['per_app_icons']):
1270             logging.info('Adding new repo for only ' + apk['id'])
1271             os.makedirs(apk['per_app_icons'])
1272
1273         apkpath = os.path.join(repodir, apk['apkname'])
1274         shutil.copy(apkpath, apk['per_app_repo'])
1275         apksigpath = apkpath + '.sig'
1276         if os.path.exists(apksigpath):
1277             shutil.copy(apksigpath, apk['per_app_repo'])
1278         apkascpath = apkpath + '.asc'
1279         if os.path.exists(apkascpath):
1280             shutil.copy(apkascpath, apk['per_app_repo'])
1281
1282
1283 config = None
1284 options = None
1285
1286
1287 def main():
1288
1289     global config, options
1290
1291     # Parse command line...
1292     parser = ArgumentParser()
1293     common.setup_global_opts(parser)
1294     parser.add_argument("--create-key", action="store_true", default=False,
1295                         help="Create a repo signing key in a keystore")
1296     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1297                         help="Create skeleton metadata files that are missing")
1298     parser.add_argument("--delete-unknown", action="store_true", default=False,
1299                         help="Delete APKs and/or OBBs without metadata from the repo")
1300     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1301                         help="Report on build data status")
1302     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1303                         help="Interactively ask about things that need updating.")
1304     parser.add_argument("-I", "--icons", action="store_true", default=False,
1305                         help="Resize all the icons exceeding the max pixel size and exit")
1306     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1307                         help="Specify editor to use in interactive mode. Default " +
1308                         "is /etc/alternatives/editor")
1309     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1310                         help="Update the wiki")
1311     parser.add_argument("--pretty", action="store_true", default=False,
1312                         help="Produce human-readable index.xml")
1313     parser.add_argument("--clean", action="store_true", default=False,
1314                         help="Clean update - don't uses caches, reprocess all apks")
1315     parser.add_argument("--nosign", action="store_true", default=False,
1316                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1317     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1318                         help="Use date from apk instead of current time for newly added apks")
1319     options = parser.parse_args()
1320
1321     config = common.read_config(options)
1322
1323     if not ('jarsigner' in config and 'keytool' in config):
1324         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1325         sys.exit(1)
1326
1327     repodirs = ['repo']
1328     if config['archive_older'] != 0:
1329         repodirs.append('archive')
1330         if not os.path.exists('archive'):
1331             os.mkdir('archive')
1332
1333     if options.icons:
1334         resize_all_icons(repodirs)
1335         sys.exit(0)
1336
1337     # check that icons exist now, rather than fail at the end of `fdroid update`
1338     for k in ['repo_icon', 'archive_icon']:
1339         if k in config:
1340             if not os.path.exists(config[k]):
1341                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1342                 sys.exit(1)
1343
1344     # if the user asks to create a keystore, do it now, reusing whatever it can
1345     if options.create_key:
1346         if os.path.exists(config['keystore']):
1347             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1348             logging.critical("\t'" + config['keystore'] + "'")
1349             sys.exit(1)
1350
1351         if 'repo_keyalias' not in config:
1352             config['repo_keyalias'] = socket.getfqdn()
1353             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1354         if 'keydname' not in config:
1355             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1356             common.write_to_config(config, 'keydname', config['keydname'])
1357         if 'keystore' not in config:
1358             config['keystore'] = common.default_config.keystore
1359             common.write_to_config(config, 'keystore', config['keystore'])
1360
1361         password = common.genpassword()
1362         if 'keystorepass' not in config:
1363             config['keystorepass'] = password
1364             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1365         if 'keypass' not in config:
1366             config['keypass'] = password
1367             common.write_to_config(config, 'keypass', config['keypass'])
1368         common.genkeystore(config)
1369
1370     # Get all apps...
1371     apps = metadata.read_metadata()
1372
1373     # Generate a list of categories...
1374     categories = set()
1375     for app in apps.values():
1376         categories.update(app.Categories)
1377
1378     # Read known apks data (will be updated and written back when we've finished)
1379     knownapks = common.KnownApks()
1380
1381     # Gather information about all the apk files in the repo directory, using
1382     # cached data if possible.
1383     apkcachefile = os.path.join('tmp', 'apkcache')
1384     if not options.clean and os.path.exists(apkcachefile):
1385         with open(apkcachefile, 'rb') as cf:
1386             apkcache = pickle.load(cf, encoding='utf-8')
1387         if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
1388             apkcache = {}
1389     else:
1390         apkcache = {}
1391
1392     delete_disabled_builds(apps, apkcache, repodirs)
1393
1394     # Scan all apks in the main repo
1395     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1396
1397     # Generate warnings for apk's with no metadata (or create skeleton
1398     # metadata files, if requested on the command line)
1399     newmetadata = False
1400     for apk in apks:
1401         if apk['id'] not in apps:
1402             if options.create_metadata:
1403                 if 'name' not in apk:
1404                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1405                     continue
1406                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
1407                 f.write("License:Unknown\n")
1408                 f.write("Web Site:\n")
1409                 f.write("Source Code:\n")
1410                 f.write("Issue Tracker:\n")
1411                 f.write("Changelog:\n")
1412                 f.write("Summary:" + apk['name'] + "\n")
1413                 f.write("Description:\n")
1414                 f.write(apk['name'] + "\n")
1415                 f.write(".\n")
1416                 f.close()
1417                 logging.info("Generated skeleton metadata for " + apk['id'])
1418                 newmetadata = True
1419             else:
1420                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1421                 if options.delete_unknown:
1422                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1423                     rmf = os.path.join(repodirs[0], apk['apkname'])
1424                     if not os.path.exists(rmf):
1425                         logging.error("Could not find {0} to remove it".format(rmf))
1426                     else:
1427                         os.remove(rmf)
1428                 else:
1429                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1430
1431     # update the metadata with the newly created ones included
1432     if newmetadata:
1433         apps = metadata.read_metadata()
1434
1435     insert_obbs(repodirs[0], apps, apks)
1436
1437     # Scan the archive repo for apks as well
1438     if len(repodirs) > 1:
1439         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1440         if cc:
1441             cachechanged = True
1442     else:
1443         archapks = []
1444
1445     # Some information from the apks needs to be applied up to the application
1446     # level. When doing this, we use the info from the most recent version's apk.
1447     # We deal with figuring out when the app was added and last updated at the
1448     # same time.
1449     for appid, app in apps.items():
1450         bestver = 0
1451         for apk in apks + archapks:
1452             if apk['id'] == appid:
1453                 if apk['versioncode'] > bestver:
1454                     bestver = apk['versioncode']
1455                     bestapk = apk
1456
1457                 if 'added' in apk:
1458                     if not app.added or apk['added'] < app.added:
1459                         app.added = apk['added']
1460                     if not app.lastupdated or apk['added'] > app.lastupdated:
1461                         app.lastupdated = apk['added']
1462
1463         if not app.added:
1464             logging.debug("Don't know when " + appid + " was added")
1465         if not app.lastupdated:
1466             logging.debug("Don't know when " + appid + " was last updated")
1467
1468         if bestver == 0:
1469             if app.Name is None:
1470                 app.Name = app.AutoName or appid
1471             app.icon = None
1472             logging.debug("Application " + appid + " has no packages")
1473         else:
1474             if app.Name is None:
1475                 app.Name = bestapk['name']
1476             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1477             if app.CurrentVersionCode is None:
1478                 app.CurrentVersionCode = str(bestver)
1479
1480     # Sort the app list by name, then the web site doesn't have to by default.
1481     # (we had to wait until we'd scanned the apks to do this, because mostly the
1482     # name comes from there!)
1483     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1484
1485     # APKs are placed into multiple repos based on the app package, providing
1486     # per-app subscription feeds for nightly builds and things like it
1487     if config['per_app_repos']:
1488         add_apks_to_per_app_repos(repodirs[0], apks)
1489         for appid, app in apps.items():
1490             repodir = os.path.join(appid, 'fdroid', 'repo')
1491             appdict = dict()
1492             appdict[appid] = app
1493             if os.path.isdir(repodir):
1494                 make_index(appdict, [appid], apks, repodir, False, categories)
1495             else:
1496                 logging.info('Skipping index generation for ' + appid)
1497         return
1498
1499     if len(repodirs) > 1:
1500         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1501
1502     # Make the index for the main repo...
1503     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1504
1505     # If there's an archive repo,  make the index for it. We already scanned it
1506     # earlier on.
1507     if len(repodirs) > 1:
1508         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1509
1510     if config['update_stats']:
1511
1512         # Update known apks info...
1513         knownapks.writeifchanged()
1514
1515         # Generate latest apps data for widget
1516         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1517             data = ''
1518             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1519                 for line in f:
1520                     appid = line.rstrip()
1521                     data += appid + "\t"
1522                     app = apps[appid]
1523                     data += app.Name + "\t"
1524                     if app.icon is not None:
1525                         data += app.icon + "\t"
1526                     data += app.License + "\n"
1527             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1528                 f.write(data)
1529
1530     if cachechanged:
1531         apkcache["METADATA_VERSION"] = METADATA_VERSION
1532         with open(apkcachefile, 'wb') as cf:
1533             pickle.dump(apkcache, cf)
1534
1535     # Update the wiki...
1536     if options.wiki:
1537         update_wiki(apps, sortedids, apks + archapks)
1538
1539     logging.info("Finished.")
1540
1541 if __name__ == "__main__":
1542     main()