fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2016, Blue Jay Wireless
   5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
   6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
   7 # Copyright (C) 2013-2014, Daniel Martí <mvdan@mvdan.cc>
   8 #
   9 # This program is free software: you can redistribute it and/or modify
  10 # it under the terms of the GNU Affero General Public License as published by
  11 # the Free Software Foundation, either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Affero General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU Affero General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 import sys
  23 import os
  24 import shutil
  25 import glob
  26 import re
  27 import socket
  28 import zipfile
  29 import hashlib
  30 import pickle
  31 import urllib.parse
  32 from datetime import datetime, timedelta
  33 from xml.dom.minidom import Document
  34 from argparse import ArgumentParser
  35 import time
  36
  37 import collections
  38 from pyasn1.error import PyAsn1Error
  39 from pyasn1.codec.der import decoder, encoder
  40 from pyasn1_modules import rfc2315
  41 from binascii import hexlify, unhexlify
  42
  43 from PIL import Image
  44 import logging
  45
  46 from . import common
  47 from . import metadata
  48 from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
  49 from .metadata import MetaDataException
  50
  51 METADATA_VERSION = 17
  52
  53 screen_densities = ['640', '480', '320', '240', '160', '120']
  54
  55 all_screen_densities = ['0'] + screen_densities
  56
  57 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  58 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  59
  60
  61 def dpi_to_px(density):
  62     return (int(density) * 48) / 160
  63
  64
  65 def px_to_dpi(px):
  66     return (int(px) * 160) / 48
  67
  68
  69 def get_icon_dir(repodir, density):
  70     if density == '0':
  71         return os.path.join(repodir, "icons")
  72     return os.path.join(repodir, "icons-%s" % density)
  73
  74
  75 def get_icon_dirs(repodir):
  76     for density in screen_densities:
  77         yield get_icon_dir(repodir, density)
  78
  79
  80 def get_all_icon_dirs(repodir):
  81     for density in all_screen_densities:
  82         yield get_icon_dir(repodir, density)
  83
  84
  85 def update_wiki(apps, sortedids, apks):
  86     """Update the wiki
  87
  88     :param apps: fully populated list of all applications
  89     :param apks: all apks, except...
  90     """
  91     logging.info("Updating wiki")
  92     wikicat = 'Apps'
  93     wikiredircat = 'App Redirects'
  94     import mwclient
  95     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  96                          path=config['wiki_path'])
  97     site.login(config['wiki_user'], config['wiki_password'])
  98     generated_pages = {}
  99     generated_redirects = {}
 100
 101     for appid in sortedids:
 102         app = apps[appid]
 103
 104         wikidata = ''
 105         if app.Disabled:
 106             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 107         if app.AntiFeatures:
 108             for af in app.AntiFeatures:
 109                 wikidata += '{{AntiFeature|' + af + '}}\n'
 110         if app.RequiresRoot:
 111             requiresroot = 'Yes'
 112         else:
 113             requiresroot = 'No'
 114         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 115             appid,
 116             app.Name,
 117             time.strftime('%Y-%m-%d', app.added) if app.added else '',
 118             time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
 119             app.SourceCode,
 120             app.IssueTracker,
 121             app.WebSite,
 122             app.Changelog,
 123             app.Donate,
 124             app.FlattrID,
 125             app.Bitcoin,
 126             app.Litecoin,
 127             app.License,
 128             requiresroot,
 129             app.AuthorName,
 130             app.AuthorEmail)
 131
 132         if app.Provides:
 133             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 134
 135         wikidata += app.Summary
 136         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 137
 138         wikidata += "=Description=\n"
 139         wikidata += metadata.description_wiki(app.Description) + "\n"
 140
 141         wikidata += "=Maintainer Notes=\n"
 142         if app.MaintainerNotes:
 143             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 144         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 145
 146         # Get a list of all packages for this application...
 147         apklist = []
 148         gotcurrentver = False
 149         cantupdate = False
 150         buildfails = False
 151         for apk in apks:
 152             if apk['id'] == appid:
 153                 if str(apk['versioncode']) == app.CurrentVersionCode:
 154                     gotcurrentver = True
 155                 apklist.append(apk)
 156         # Include ones we can't build, as a special case...
 157         for build in app.builds:
 158             if build.disable:
 159                 if build.vercode == app.CurrentVersionCode:
 160                     cantupdate = True
 161                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 162                 apklist.append({'versioncode': int(build.vercode),
 163                                 'version': build.version,
 164                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 165                                 })
 166             else:
 167                 builtit = False
 168                 for apk in apklist:
 169                     if apk['versioncode'] == int(build.vercode):
 170                         builtit = True
 171                         break
 172                 if not builtit:
 173                     buildfails = True
 174                     apklist.append({'versioncode': int(build.vercode),
 175                                     'version': build.version,
 176                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
 177                                     })
 178         if app.CurrentVersionCode == '0':
 179             cantupdate = True
 180         # Sort with most recent first...
 181         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 182
 183         wikidata += "=Versions=\n"
 184         if len(apklist) == 0:
 185             wikidata += "We currently have no versions of this app available."
 186         elif not gotcurrentver:
 187             wikidata += "We don't have the current version of this app."
 188         else:
 189             wikidata += "We have the current version of this app."
 190         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 191         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 192         if len(app.NoSourceSince) > 0:
 193             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 194         if len(app.CurrentVersion) > 0:
 195             wikidata += "The current (recommended) version is " + app.CurrentVersion
 196             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 197         validapks = 0
 198         for apk in apklist:
 199             wikidata += "==" + apk['version'] + "==\n"
 200
 201             if 'buildproblem' in apk:
 202                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 203             else:
 204                 validapks += 1
 205                 wikidata += "This version is built and signed by "
 206                 if 'srcname' in apk:
 207                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 208                 else:
 209                     wikidata += "the original developer.\n\n"
 210             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 211
 212         wikidata += '\n[[Category:' + wikicat + ']]\n'
 213         if len(app.NoSourceSince) > 0:
 214             wikidata += '\n[[Category:Apps missing source code]]\n'
 215         if validapks == 0 and not app.Disabled:
 216             wikidata += '\n[[Category:Apps with no packages]]\n'
 217         if cantupdate and not app.Disabled:
 218             wikidata += "\n[[Category:Apps we cannot update]]\n"
 219         if buildfails and not app.Disabled:
 220             wikidata += "\n[[Category:Apps with failing builds]]\n"
 221         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 222             wikidata += '\n[[Category:Apps to Update]]\n'
 223         if app.Disabled:
 224             wikidata += '\n[[Category:Apps that are disabled]]\n'
 225         if app.UpdateCheckMode == 'None' and not app.Disabled:
 226             wikidata += '\n[[Category:Apps with no update check]]\n'
 227         for appcat in app.Categories:
 228             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 229
 230         # We can't have underscores in the page name, even if they're in
 231         # the package ID, because MediaWiki messes with them...
 232         pagename = appid.replace('_', ' ')
 233
 234         # Drop a trailing newline, because mediawiki is going to drop it anyway
 235         # and it we don't we'll think the page has changed when it hasn't...
 236         if wikidata.endswith('\n'):
 237             wikidata = wikidata[:-1]
 238
 239         generated_pages[pagename] = wikidata
 240
 241         # Make a redirect from the name to the ID too, unless there's
 242         # already an existing page with the name and it isn't a redirect.
 243         noclobber = False
 244         apppagename = app.Name.replace('_', ' ')
 245         apppagename = apppagename.replace('{', '')
 246         apppagename = apppagename.replace('}', ' ')
 247         apppagename = apppagename.replace(':', ' ')
 248         apppagename = apppagename.replace('[', ' ')
 249         apppagename = apppagename.replace(']', ' ')
 250         # Drop double spaces caused mostly by replacing ':' above
 251         apppagename = apppagename.replace('  ', ' ')
 252         for expagename in site.allpages(prefix=apppagename,
 253                                         filterredir='nonredirects',
 254                                         generator=False):
 255             if expagename == apppagename:
 256                 noclobber = True
 257         # Another reason not to make the redirect page is if the app name
 258         # is the same as it's ID, because that will overwrite the real page
 259         # with an redirect to itself! (Although it seems like an odd
 260         # scenario this happens a lot, e.g. where there is metadata but no
 261         # builds or binaries to extract a name from.
 262         if apppagename == pagename:
 263             noclobber = True
 264         if not noclobber:
 265             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 266
 267     for tcat, genp in [(wikicat, generated_pages),
 268                        (wikiredircat, generated_redirects)]:
 269         catpages = site.Pages['Category:' + tcat]
 270         existingpages = []
 271         for page in catpages:
 272             existingpages.append(page.name)
 273             if page.name in genp:
 274                 pagetxt = page.edit()
 275                 if pagetxt != genp[page.name]:
 276                     logging.debug("Updating modified page " + page.name)
 277                     page.save(genp[page.name], summary='Auto-updated')
 278                 else:
 279                     logging.debug("Page " + page.name + " is unchanged")
 280             else:
 281                 logging.warn("Deleting page " + page.name)
 282                 page.delete('No longer published')
 283         for pagename, text in genp.items():
 284             logging.debug("Checking " + pagename)
 285             if pagename not in existingpages:
 286                 logging.debug("Creating page " + pagename)
 287                 try:
 288                     newpage = site.Pages[pagename]
 289                     newpage.save(text, summary='Auto-created')
 290                 except:
 291                     logging.error("...FAILED to create page '{0}'".format(pagename))
 292
 293     # Purge server cache to ensure counts are up to date
 294     site.pages['Repository Maintenance'].purge()
 295
 296
 297 def delete_disabled_builds(apps, apkcache, repodirs):
 298     """Delete disabled build outputs.
 299
 300     :param apps: list of all applications, as per metadata.read_metadata
 301     :param apkcache: current apk cache information
 302     :param repodirs: the repo directories to process
 303     """
 304     for appid, app in apps.items():
 305         for build in app.builds:
 306             if not build.disable:
 307                 continue
 308             apkfilename = appid + '_' + str(build.vercode) + '.apk'
 309             iconfilename = "%s.%s.png" % (
 310                 appid,
 311                 build.vercode)
 312             for repodir in repodirs:
 313                 files = [
 314                     os.path.join(repodir, apkfilename),
 315                     os.path.join(repodir, apkfilename + '.asc'),
 316                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 317                 ]
 318                 for density in all_screen_densities:
 319                     repo_dir = get_icon_dir(repodir, density)
 320                     files.append(os.path.join(repo_dir, iconfilename))
 321
 322                 for f in files:
 323                     if os.path.exists(f):
 324                         logging.info("Deleting disabled build output " + f)
 325                         os.remove(f)
 326             if apkfilename in apkcache:
 327                 del apkcache[apkfilename]
 328
 329
 330 def resize_icon(iconpath, density):
 331
 332     if not os.path.isfile(iconpath):
 333         return
 334
 335     fp = None
 336     try:
 337         fp = open(iconpath, 'rb')
 338         im = Image.open(fp)
 339         size = dpi_to_px(density)
 340
 341         if any(length > size for length in im.size):
 342             oldsize = im.size
 343             im.thumbnail((size, size), Image.ANTIALIAS)
 344             logging.debug("%s was too large at %s - new size is %s" % (
 345                 iconpath, oldsize, im.size))
 346             im.save(iconpath, "PNG")
 347
 348     except Exception as e:
 349         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 350
 351     finally:
 352         if fp:
 353             fp.close()
 354
 355
 356 def resize_all_icons(repodirs):
 357     """Resize all icons that exceed the max size
 358
 359     :param repodirs: the repo directories to process
 360     """
 361     for repodir in repodirs:
 362         for density in screen_densities:
 363             icon_dir = get_icon_dir(repodir, density)
 364             icon_glob = os.path.join(icon_dir, '*.png')
 365             for iconpath in glob.glob(icon_glob):
 366                 resize_icon(iconpath, density)
 367
 368
 369 # A signature block file with a .DSA, .RSA, or .EC extension
 370 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 371
 372
 373 def getsig(apkpath):
 374     """ Get the signing certificate of an apk. To get the same md5 has that
 375     Android gets, we encode the .RSA certificate in a specific format and pass
 376     it hex-encoded to the md5 digest algorithm.
 377
 378     :param apkpath: path to the apk
 379     :returns: A string containing the md5 of the signature of the apk or None
 380               if an error occurred.
 381     """
 382
 383     cert = None
 384
 385     # verify the jar signature is correct
 386     args = [config['jarsigner'], '-verify', apkpath]
 387     p = FDroidPopen(args)
 388     if p.returncode != 0:
 389         logging.critical(apkpath + " has a bad signature!")
 390         return None
 391
 392     with zipfile.ZipFile(apkpath, 'r') as apk:
 393
 394         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 395
 396         if len(certs) < 1:
 397             logging.error("Found no signing certificates on %s" % apkpath)
 398             return None
 399         if len(certs) > 1:
 400             logging.error("Found multiple signing certificates on %s" % apkpath)
 401             return None
 402
 403         cert = apk.read(certs[0])
 404
 405     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 406     if content.getComponentByName('contentType') != rfc2315.signedData:
 407         logging.error("Unexpected format.")
 408         return None
 409
 410     content = decoder.decode(content.getComponentByName('content'),
 411                              asn1Spec=rfc2315.SignedData())[0]
 412     try:
 413         certificates = content.getComponentByName('certificates')
 414     except PyAsn1Error:
 415         logging.error("Certificates not found.")
 416         return None
 417
 418     cert_encoded = encoder.encode(certificates)[4:]
 419
 420     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 421
 422
 423 def get_icon_bytes(apkzip, iconsrc):
 424     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 425     try:
 426         return apkzip.read(iconsrc)
 427     except KeyError:
 428         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 429
 430
 431 def sha256sum(filename):
 432     '''Calculate the sha256 of the given file'''
 433     sha = hashlib.sha256()
 434     with open(filename, 'rb') as f:
 435         while True:
 436             t = f.read(16384)
 437             if len(t) == 0:
 438                 break
 439             sha.update(t)
 440     return sha.hexdigest()
 441
 442
 443 def insert_obbs(repodir, apps, apks):
 444     """Scans the .obb files in a given repo directory and adds them to the
 445     relevant APK instances.  OBB files have versionCodes like APK
 446     files, and they are loosely associated.  If there is an OBB file
 447     present, then any APK with the same or higher versionCode will use
 448     that OBB file.  There are two OBB types: main and patch, each APK
 449     can only have only have one of each.
 450
 451     https://developer.android.com/google/play/expansion-files.html
 452
 453     :param repodir: repo directory to scan
 454     :param apps: list of current, valid apps
 455     :param apks: current information on all APKs
 456
 457     """
 458
 459     def obbWarnDelete(f, msg):
 460         logging.warning(msg + f)
 461         if options.delete_unknown:
 462             logging.error("Deleting unknown file: " + f)
 463             os.remove(f)
 464
 465     obbs = []
 466     java_Integer_MIN_VALUE = -pow(2, 31)
 467     for f in glob.glob(os.path.join(repodir, '*.obb')):
 468         obbfile = os.path.basename(f)
 469         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 470         chunks = obbfile.split('.')
 471         if chunks[0] != 'main' and chunks[0] != 'patch':
 472             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 473             continue
 474         if not re.match(r'^-?[0-9]+$', chunks[1]):
 475             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 476             continue
 477         versioncode = int(chunks[1])
 478         packagename = ".".join(chunks[2:-1])
 479
 480         highestVersionCode = java_Integer_MIN_VALUE
 481         if packagename not in apps.keys():
 482             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 483             continue
 484         for apk in apks:
 485             if packagename == apk['id'] and apk['versioncode'] > highestVersionCode:
 486                 highestVersionCode = apk['versioncode']
 487         if versioncode > highestVersionCode:
 488             obbWarnDelete(f, 'OBB file has newer versioncode(' + str(versioncode)
 489                           + ') than any APK: ')
 490             continue
 491         obbsha256 = sha256sum(f)
 492         obbs.append((packagename, versioncode, obbfile, obbsha256))
 493
 494     for apk in apks:
 495         for (packagename, versioncode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 496             if versioncode <= apk['versioncode'] and packagename == apk['id']:
 497                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 498                     apk['obbMainFile'] = obbfile
 499                     apk['obbMainFileSha256'] = obbsha256
 500                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 501                     apk['obbPatchFile'] = obbfile
 502                     apk['obbPatchFileSha256'] = obbsha256
 503             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 504                 break
 505
 506
 507 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
 508     """Scan the apks in the given repo directory.
 509
 510     This also extracts the icons.
 511
 512     :param apps: list of all applications, as per metadata.read_metadata
 513     :param apkcache: current apk cache information
 514     :param repodir: repo directory to scan
 515     :param knownapks: known apks info
 516     :param use_date_from_apk: use date from APK (instead of current date)
 517                               for newly added APKs
 518     :returns: (apks, cachechanged) where apks is a list of apk information,
 519               and cachechanged is True if the apkcache got changed.
 520     """
 521
 522     cachechanged = False
 523
 524     for icon_dir in get_all_icon_dirs(repodir):
 525         if os.path.exists(icon_dir):
 526             if options.clean:
 527                 shutil.rmtree(icon_dir)
 528                 os.makedirs(icon_dir)
 529         else:
 530             os.makedirs(icon_dir)
 531
 532     apks = []
 533     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 534     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 535     vername_pat = re.compile(".*versionName='([^']*)'.*")
 536     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 537     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 538     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 539     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 540     permission_pat = re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
 541     feature_pat = re.compile(".*name='([^']*)'.*")
 542     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 543
 544         apkfilename = apkfile[len(repodir) + 1:]
 545         if ' ' in apkfilename:
 546             logging.critical("Spaces in filenames are not allowed.")
 547             sys.exit(1)
 548
 549         shasum = sha256sum(apkfile)
 550
 551         usecache = False
 552         if apkfilename in apkcache:
 553             apk = apkcache[apkfilename]
 554             if apk['sha256'] == shasum:
 555                 logging.debug("Reading " + apkfilename + " from cache")
 556                 usecache = True
 557             else:
 558                 logging.debug("Ignoring stale cache data for " + apkfilename)
 559
 560         if not usecache:
 561             logging.debug("Processing " + apkfilename)
 562             apk = {}
 563             apk['apkname'] = apkfilename
 564             apk['sha256'] = shasum
 565             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 566             if os.path.exists(os.path.join(repodir, srcfilename)):
 567                 apk['srcname'] = srcfilename
 568             apk['size'] = os.path.getsize(apkfile)
 569             apk['uses-permission'] = set()
 570             apk['uses-permission-sdk-23'] = set()
 571             apk['features'] = set()
 572             apk['icons_src'] = {}
 573             apk['icons'] = {}
 574             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 575             if p.returncode != 0:
 576                 if options.delete_unknown:
 577                     if os.path.exists(apkfile):
 578                         logging.error("Failed to get apk information, deleting " + apkfile)
 579                         os.remove(apkfile)
 580                     else:
 581                         logging.error("Could not find {0} to remove it".format(apkfile))
 582                 else:
 583                     logging.error("Failed to get apk information, skipping " + apkfile)
 584                 continue
 585             for line in p.output.splitlines():
 586                 if line.startswith("package:"):
 587                     try:
 588                         apk['id'] = re.match(name_pat, line).group(1)
 589                         apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
 590                         apk['version'] = re.match(vername_pat, line).group(1)
 591                     except Exception as e:
 592                         logging.error("Package matching failed: " + str(e))
 593                         logging.info("Line was: " + line)
 594                         sys.exit(1)
 595                 elif line.startswith("application:"):
 596                     apk['name'] = re.match(label_pat, line).group(1)
 597                     # Keep path to non-dpi icon in case we need it
 598                     match = re.match(icon_pat_nodpi, line)
 599                     if match:
 600                         apk['icons_src']['-1'] = match.group(1)
 601                 elif line.startswith("launchable-activity:"):
 602                     # Only use launchable-activity as fallback to application
 603                     if not apk['name']:
 604                         apk['name'] = re.match(label_pat, line).group(1)
 605                     if '-1' not in apk['icons_src']:
 606                         match = re.match(icon_pat_nodpi, line)
 607                         if match:
 608                             apk['icons_src']['-1'] = match.group(1)
 609                 elif line.startswith("application-icon-"):
 610                     match = re.match(icon_pat, line)
 611                     if match:
 612                         density = match.group(1)
 613                         path = match.group(2)
 614                         apk['icons_src'][density] = path
 615                 elif line.startswith("sdkVersion:"):
 616                     m = re.match(sdkversion_pat, line)
 617                     if m is None:
 618                         logging.error(line.replace('sdkVersion:', '')
 619                                       + ' is not a valid minSdkVersion!')
 620                     else:
 621                         apk['minSdkVersion'] = m.group(1)
 622                         # if target not set, default to min
 623                         if 'targetSdkVersion' not in apk:
 624                             apk['targetSdkVersion'] = m.group(1)
 625                 elif line.startswith("targetSdkVersion:"):
 626                     m = re.match(sdkversion_pat, line)
 627                     if m is None:
 628                         logging.error(line.replace('targetSdkVersion:', '')
 629                                       + ' is not a valid targetSdkVersion!')
 630                     else:
 631                         apk['targetSdkVersion'] = m.group(1)
 632                 elif line.startswith("maxSdkVersion:"):
 633                     apk['maxSdkVersion'] = re.match(sdkversion_pat, line).group(1)
 634                 elif line.startswith("native-code:"):
 635                     apk['nativecode'] = []
 636                     for arch in line[13:].split(' '):
 637                         apk['nativecode'].append(arch[1:-1])
 638                 elif line.startswith('uses-permission:'):
 639                     perm_match = re.match(permission_pat, line).groupdict()
 640
 641                     permission = UsesPermission(
 642                         perm_match['name'],
 643                         perm_match['maxSdkVersion']
 644                     )
 645
 646                     apk['uses-permission'].add(permission)
 647                 elif line.startswith('uses-permission-sdk-23:'):
 648                     perm_match = re.match(permission_pat, line).groupdict()
 649
 650                     permission_sdk_23 = UsesPermissionSdk23(
 651                         perm_match['name'],
 652                         perm_match['maxSdkVersion']
 653                     )
 654
 655                     apk['uses-permission-sdk-23'].add(permission_sdk_23)
 656
 657                 elif line.startswith('uses-feature:'):
 658                     feature = re.match(feature_pat, line).group(1)
 659                     # Filter out this, it's only added with the latest SDK tools and
 660                     # causes problems for lots of apps.
 661                     if feature != "android.hardware.screen.portrait" \
 662                             and feature != "android.hardware.screen.landscape":
 663                         if feature.startswith("android.feature."):
 664                             feature = feature[16:]
 665                         apk['features'].add(feature)
 666
 667             if 'minSdkVersion' not in apk:
 668                 logging.warn("No SDK version information found in {0}".format(apkfile))
 669                 apk['minSdkVersion'] = 1
 670
 671             # Check for debuggable apks...
 672             if common.isApkDebuggable(apkfile, config):
 673                 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
 674
 675             # Get the signature (or md5 of, to be precise)...
 676             logging.debug('Getting signature of {0}'.format(apkfile))
 677             apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 678             if not apk['sig']:
 679                 logging.critical("Failed to get apk signature")
 680                 sys.exit(1)
 681
 682             apkzip = zipfile.ZipFile(apkfile, 'r')
 683
 684             # if an APK has files newer than the system time, suggest updating
 685             # the system clock.  This is useful for offline systems, used for
 686             # signing, which do not have another source of clock sync info. It
 687             # has to be more than 24 hours newer because ZIP/APK files do not
 688             # store timezone info
 689             manifest = apkzip.getinfo('AndroidManifest.xml')
 690             if manifest.date_time[1] == 0:  # month can't be zero
 691                 logging.debug('AndroidManifest.xml has no date')
 692             else:
 693                 dt_obj = datetime(*manifest.date_time)
 694                 checkdt = dt_obj - timedelta(1)
 695                 if datetime.today() < checkdt:
 696                     logging.warn('System clock is older than manifest in: '
 697                                  + apkfilename
 698                                  + '\nSet clock to that time using:\n'
 699                                  + 'sudo date -s "' + str(dt_obj) + '"')
 700
 701             iconfilename = "%s.%s.png" % (
 702                 apk['id'],
 703                 apk['versioncode'])
 704
 705             # Extract the icon file...
 706             empty_densities = []
 707             for density in screen_densities:
 708                 if density not in apk['icons_src']:
 709                     empty_densities.append(density)
 710                     continue
 711                 iconsrc = apk['icons_src'][density]
 712                 icon_dir = get_icon_dir(repodir, density)
 713                 icondest = os.path.join(icon_dir, iconfilename)
 714
 715                 try:
 716                     with open(icondest, 'wb') as f:
 717                         f.write(get_icon_bytes(apkzip, iconsrc))
 718                     apk['icons'][density] = iconfilename
 719
 720                 except:
 721                     logging.warn("Error retrieving icon file")
 722                     del apk['icons'][density]
 723                     del apk['icons_src'][density]
 724                     empty_densities.append(density)
 725
 726             if '-1' in apk['icons_src']:
 727                 iconsrc = apk['icons_src']['-1']
 728                 iconpath = os.path.join(
 729                     get_icon_dir(repodir, '0'), iconfilename)
 730                 with open(iconpath, 'wb') as f:
 731                     f.write(get_icon_bytes(apkzip, iconsrc))
 732                 try:
 733                     im = Image.open(iconpath)
 734                     dpi = px_to_dpi(im.size[0])
 735                     for density in screen_densities:
 736                         if density in apk['icons']:
 737                             break
 738                         if density == screen_densities[-1] or dpi >= int(density):
 739                             apk['icons'][density] = iconfilename
 740                             shutil.move(iconpath,
 741                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 742                             empty_densities.remove(density)
 743                             break
 744                 except Exception as e:
 745                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 746
 747             if apk['icons']:
 748                 apk['icon'] = iconfilename
 749
 750             apkzip.close()
 751
 752             # First try resizing down to not lose quality
 753             last_density = None
 754             for density in screen_densities:
 755                 if density not in empty_densities:
 756                     last_density = density
 757                     continue
 758                 if last_density is None:
 759                     continue
 760                 logging.debug("Density %s not available, resizing down from %s"
 761                               % (density, last_density))
 762
 763                 last_iconpath = os.path.join(
 764                     get_icon_dir(repodir, last_density), iconfilename)
 765                 iconpath = os.path.join(
 766                     get_icon_dir(repodir, density), iconfilename)
 767                 fp = None
 768                 try:
 769                     fp = open(last_iconpath, 'rb')
 770                     im = Image.open(fp)
 771
 772                     size = dpi_to_px(density)
 773
 774                     im.thumbnail((size, size), Image.ANTIALIAS)
 775                     im.save(iconpath, "PNG")
 776                     empty_densities.remove(density)
 777                 except:
 778                     logging.warning("Invalid image file at %s" % last_iconpath)
 779                 finally:
 780                     if fp:
 781                         fp.close()
 782
 783             # Then just copy from the highest resolution available
 784             last_density = None
 785             for density in reversed(screen_densities):
 786                 if density not in empty_densities:
 787                     last_density = density
 788                     continue
 789                 if last_density is None:
 790                     continue
 791                 logging.debug("Density %s not available, copying from lower density %s"
 792                               % (density, last_density))
 793
 794                 shutil.copyfile(
 795                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 796                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 797
 798                 empty_densities.remove(density)
 799
 800             for density in screen_densities:
 801                 icon_dir = get_icon_dir(repodir, density)
 802                 icondest = os.path.join(icon_dir, iconfilename)
 803                 resize_icon(icondest, density)
 804
 805             # Copy from icons-mdpi to icons since mdpi is the baseline density
 806             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 807             if os.path.isfile(baseline):
 808                 apk['icons']['0'] = iconfilename
 809                 shutil.copyfile(baseline,
 810                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 811
 812             if use_date_from_apk and manifest.date_time[1] != 0:
 813                 default_date_param = datetime(*manifest.date_time).utctimetuple()
 814             else:
 815                 default_date_param = None
 816
 817             # Record in known apks, getting the added date at the same time..
 818             added = knownapks.recordapk(apk['apkname'], apk['id'], default_date=default_date_param)
 819             if added:
 820                 apk['added'] = added
 821
 822             apkcache[apkfilename] = apk
 823             cachechanged = True
 824
 825         apks.append(apk)
 826
 827     return apks, cachechanged
 828
 829
 830 repo_pubkey_fingerprint = None
 831
 832
 833 # Generate a certificate fingerprint the same way keytool does it
 834 # (but with slightly different formatting)
 835 def cert_fingerprint(data):
 836     digest = hashlib.sha256(data).digest()
 837     ret = []
 838     ret.append(' '.join("%02X" % b for b in bytearray(digest)))
 839     return " ".join(ret)
 840
 841
 842 def extract_pubkey():
 843     global repo_pubkey_fingerprint
 844     if 'repo_pubkey' in config:
 845         pubkey = unhexlify(config['repo_pubkey'])
 846     else:
 847         p = FDroidPopenBytes([config['keytool'], '-exportcert',
 848                               '-alias', config['repo_keyalias'],
 849                               '-keystore', config['keystore'],
 850                               '-storepass:file', config['keystorepassfile']]
 851                              + config['smartcardoptions'],
 852                              output=False, stderr_to_stdout=False)
 853         if p.returncode != 0 or len(p.output) < 20:
 854             msg = "Failed to get repo pubkey!"
 855             if config['keystore'] == 'NONE':
 856                 msg += ' Is your crypto smartcard plugged in?'
 857             logging.critical(msg)
 858             sys.exit(1)
 859         pubkey = p.output
 860     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 861     return hexlify(pubkey)
 862
 863
 864 def make_index(apps, sortedids, apks, repodir, archive, categories):
 865     """Make a repo index.
 866
 867     :param apps: fully populated apps list
 868     :param apks: full populated apks list
 869     :param repodir: the repo directory
 870     :param archive: True if this is the archive repo, False if it's the
 871                     main one.
 872     :param categories: list of categories
 873     """
 874
 875     doc = Document()
 876
 877     def addElement(name, value, doc, parent):
 878         el = doc.createElement(name)
 879         el.appendChild(doc.createTextNode(value))
 880         parent.appendChild(el)
 881
 882     def addElementNonEmpty(name, value, doc, parent):
 883         if not value:
 884             return
 885         addElement(name, value, doc, parent)
 886
 887     def addElementCDATA(name, value, doc, parent):
 888         el = doc.createElement(name)
 889         el.appendChild(doc.createCDATASection(value))
 890         parent.appendChild(el)
 891
 892     root = doc.createElement("fdroid")
 893     doc.appendChild(root)
 894
 895     repoel = doc.createElement("repo")
 896
 897     mirrorcheckfailed = False
 898     for mirror in config.get('mirrors', []):
 899         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
 900         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
 901             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 902             mirrorcheckfailed = True
 903     if mirrorcheckfailed:
 904         sys.exit(1)
 905
 906     if archive:
 907         repoel.setAttribute("name", config['archive_name'])
 908         if config['repo_maxage'] != 0:
 909             repoel.setAttribute("maxage", str(config['repo_maxage']))
 910         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 911         repoel.setAttribute("url", config['archive_url'])
 912         addElement('description', config['archive_description'], doc, repoel)
 913         urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
 914         for mirror in config.get('mirrors', []):
 915             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 916
 917     else:
 918         repoel.setAttribute("name", config['repo_name'])
 919         if config['repo_maxage'] != 0:
 920             repoel.setAttribute("maxage", str(config['repo_maxage']))
 921         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 922         repoel.setAttribute("url", config['repo_url'])
 923         addElement('description', config['repo_description'], doc, repoel)
 924         urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
 925         for mirror in config.get('mirrors', []):
 926             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 927
 928     repoel.setAttribute("version", str(METADATA_VERSION))
 929     repoel.setAttribute("timestamp", str(int(time.time())))
 930
 931     nosigningkey = False
 932     if not options.nosign:
 933         if 'repo_keyalias' not in config:
 934             nosigningkey = True
 935             logging.critical("'repo_keyalias' not found in config.py!")
 936         if 'keystore' not in config:
 937             nosigningkey = True
 938             logging.critical("'keystore' not found in config.py!")
 939         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 940             nosigningkey = True
 941             logging.critical("'keystorepass' not found in config.py!")
 942         if 'keypass' not in config and 'keypassfile' not in config:
 943             nosigningkey = True
 944             logging.critical("'keypass' not found in config.py!")
 945         if not os.path.exists(config['keystore']):
 946             nosigningkey = True
 947             logging.critical("'" + config['keystore'] + "' does not exist!")
 948         if nosigningkey:
 949             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 950             logging.warning("\tfdroid update --create-key")
 951             sys.exit(1)
 952
 953     repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
 954     root.appendChild(repoel)
 955
 956     for command in ('install', 'delete'):
 957         packageNames = []
 958         key = command + '_list'
 959         if key in config:
 960             if isinstance(config[key], str):
 961                 packageNames = [config[key]]
 962             elif all(isinstance(item, str) for item in config[key]):
 963                 packageNames = config[key]
 964             else:
 965                 raise TypeError('only accepts strings, lists, and tuples')
 966         for packageName in packageNames:
 967             element = doc.createElement(command)
 968             root.appendChild(element)
 969             element.setAttribute('packageName', packageName)
 970
 971     for appid in sortedids:
 972         app = apps[appid]
 973
 974         if app.Disabled is not None:
 975             continue
 976
 977         # Get a list of the apks for this app...
 978         apklist = []
 979         for apk in apks:
 980             if apk['id'] == appid:
 981                 apklist.append(apk)
 982
 983         if len(apklist) == 0:
 984             continue
 985
 986         apel = doc.createElement("application")
 987         apel.setAttribute("id", app.id)
 988         root.appendChild(apel)
 989
 990         addElement('id', app.id, doc, apel)
 991         if app.added:
 992             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
 993         if app.lastupdated:
 994             addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
 995         addElement('name', app.Name, doc, apel)
 996         addElement('summary', app.Summary, doc, apel)
 997         if app.icon:
 998             addElement('icon', app.icon, doc, apel)
 999
1000         def linkres(appid):
1001             if appid in apps:
1002                 return ("fdroid.app:" + appid, apps[appid].Name)
1003             raise MetaDataException("Cannot resolve app id " + appid)
1004
1005         addElement('desc',
1006                    metadata.description_html(app.Description, linkres),
1007                    doc, apel)
1008         addElement('license', app.License, doc, apel)
1009         if app.Categories:
1010             addElement('categories', ','.join(app.Categories), doc, apel)
1011             # We put the first (primary) category in LAST, which will have
1012             # the desired effect of making clients that only understand one
1013             # category see that one.
1014             addElement('category', app.Categories[0], doc, apel)
1015         addElement('web', app.WebSite, doc, apel)
1016         addElement('source', app.SourceCode, doc, apel)
1017         addElement('tracker', app.IssueTracker, doc, apel)
1018         addElementNonEmpty('changelog', app.Changelog, doc, apel)
1019         addElementNonEmpty('author', app.AuthorName, doc, apel)
1020         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
1021         addElementNonEmpty('donate', app.Donate, doc, apel)
1022         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
1023         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
1024         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
1025
1026         # These elements actually refer to the current version (i.e. which
1027         # one is recommended. They are historically mis-named, and need
1028         # changing, but stay like this for now to support existing clients.
1029         addElement('marketversion', app.CurrentVersion, doc, apel)
1030         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
1031
1032         if app.AntiFeatures:
1033             af = app.AntiFeatures
1034             if af:
1035                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
1036         if app.Provides:
1037             pv = app.Provides.split(',')
1038             addElementNonEmpty('provides', ','.join(pv), doc, apel)
1039         if app.RequiresRoot:
1040             addElement('requirements', 'root', doc, apel)
1041
1042         # Sort the apk list into version order, just so the web site
1043         # doesn't have to do any work by default...
1044         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
1045
1046         # Check for duplicates - they will make the client unhappy...
1047         for i in range(len(apklist) - 1):
1048             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
1049                 logging.critical("duplicate versions: '%s' - '%s'" % (
1050                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
1051                 sys.exit(1)
1052
1053         current_version_code = 0
1054         current_version_file = None
1055         for apk in apklist:
1056             # find the APK for the "Current Version"
1057             if current_version_code < apk['versioncode']:
1058                 current_version_code = apk['versioncode']
1059             if current_version_code < int(app.CurrentVersionCode):
1060                 current_version_file = apk['apkname']
1061
1062             apkel = doc.createElement("package")
1063             apel.appendChild(apkel)
1064             addElement('version', apk['version'], doc, apkel)
1065             addElement('versioncode', str(apk['versioncode']), doc, apkel)
1066             addElement('apkname', apk['apkname'], doc, apkel)
1067             if 'srcname' in apk:
1068                 addElement('srcname', apk['srcname'], doc, apkel)
1069             for hash_type in ['sha256']:
1070                 if hash_type not in apk:
1071                     continue
1072                 hashel = doc.createElement("hash")
1073                 hashel.setAttribute("type", hash_type)
1074                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
1075                 apkel.appendChild(hashel)
1076             addElement('sig', apk['sig'], doc, apkel)
1077             addElement('size', str(apk['size']), doc, apkel)
1078             addElement('sdkver', str(apk['minSdkVersion']), doc, apkel)
1079             if 'targetSdkVersion' in apk:
1080                 addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
1081             if 'maxSdkVersion' in apk:
1082                 addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
1083             addElementNonEmpty('obbMainFile', apk.get('obbMainFile'), doc, apkel)
1084             addElementNonEmpty('obbMainFileSha256', apk.get('obbMainFileSha256'), doc, apkel)
1085             addElementNonEmpty('obbPatchFile', apk.get('obbPatchFile'), doc, apkel)
1086             addElementNonEmpty('obbPatchFileSha256', apk.get('obbPatchFileSha256'), doc, apkel)
1087             if 'added' in apk:
1088                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
1089
1090             # TODO: remove old permission format
1091             old_permissions = set()
1092             for perm in apk['uses-permission']:
1093                 perm_name = perm.name
1094                 if perm_name.startswith("android.permission."):
1095                     perm_name = perm_name[19:]
1096                 old_permissions.add(perm_name)
1097             addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel)
1098
1099             for permission in apk['uses-permission']:
1100                 permel = doc.createElement('uses-permission')
1101                 permel.setAttribute('name', permission.name)
1102                 if permission.maxSdkVersion is not None:
1103                     permel.setAttribute('maxSdkVersion', permission.maxSdkVersion)
1104                     apkel.appendChild(permel)
1105             for permission_sdk_23 in apk['uses-permission-sdk-23']:
1106                 permel = doc.createElement('uses-permission-sdk-23')
1107                 permel.setAttribute('name', permission_sdk_23.name)
1108                 if permission_sdk_23.maxSdkVersion is not None:
1109                     permel.setAttribute('maxSdkVersion', permission_sdk_23.maxSdkVersion)
1110                     apkel.appendChild(permel)
1111             if 'nativecode' in apk:
1112                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
1113             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
1114
1115         if current_version_file is not None \
1116                 and config['make_current_version_link'] \
1117                 and repodir == 'repo':  # only create these
1118             namefield = config['current_version_name_source']
1119             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
1120             apklinkname = sanitized_name + '.apk'
1121             current_version_path = os.path.join(repodir, current_version_file)
1122             if os.path.islink(apklinkname):
1123                 os.remove(apklinkname)
1124             os.symlink(current_version_path, apklinkname)
1125             # also symlink gpg signature, if it exists
1126             for extension in ('.asc', '.sig'):
1127                 sigfile_path = current_version_path + extension
1128                 if os.path.exists(sigfile_path):
1129                     siglinkname = apklinkname + extension
1130                     if os.path.islink(siglinkname):
1131                         os.remove(siglinkname)
1132                     os.symlink(sigfile_path, siglinkname)
1133
1134     if options.pretty:
1135         output = doc.toprettyxml(encoding='utf-8')
1136     else:
1137         output = doc.toxml(encoding='utf-8')
1138
1139     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
1140         f.write(output)
1141
1142     if 'repo_keyalias' in config:
1143
1144         if options.nosign:
1145             logging.info("Creating unsigned index in preparation for signing")
1146         else:
1147             logging.info("Creating signed index with this key (SHA256):")
1148             logging.info("%s" % repo_pubkey_fingerprint)
1149
1150         # Create a jar of the index...
1151         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
1152         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
1153         if p.returncode != 0:
1154             logging.critical("Failed to create {0}".format(jar_output))
1155             sys.exit(1)
1156
1157         # Sign the index...
1158         signed = os.path.join(repodir, 'index.jar')
1159         if options.nosign:
1160             # Remove old signed index if not signing
1161             if os.path.exists(signed):
1162                 os.remove(signed)
1163         else:
1164             args = [config['jarsigner'], '-keystore', config['keystore'],
1165                     '-storepass:file', config['keystorepassfile'],
1166                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1167                     signed, config['repo_keyalias']]
1168             if config['keystore'] == 'NONE':
1169                 args += config['smartcardoptions']
1170             else:  # smardcards never use -keypass
1171                 args += ['-keypass:file', config['keypassfile']]
1172             p = FDroidPopen(args)
1173             if p.returncode != 0:
1174                 logging.critical("Failed to sign index")
1175                 sys.exit(1)
1176
1177     # Copy the repo icon into the repo directory...
1178     icon_dir = os.path.join(repodir, 'icons')
1179     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1180     shutil.copyfile(config['repo_icon'], iconfilename)
1181
1182     # Write a category list in the repo to allow quick access...
1183     catdata = ''
1184     for cat in categories:
1185         catdata += cat + '\n'
1186     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1187         f.write(catdata)
1188
1189
1190 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1191
1192     for appid, app in apps.items():
1193
1194         if app.ArchivePolicy:
1195             keepversions = int(app.ArchivePolicy[:-9])
1196         else:
1197             keepversions = defaultkeepversions
1198
1199         def filter_apk_list_sorted(apk_list):
1200             res = []
1201             for apk in apk_list:
1202                 if apk['id'] == appid:
1203                     res.append(apk)
1204
1205             # Sort the apk list by version code. First is highest/newest.
1206             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1207
1208         def move_file(from_dir, to_dir, filename, ignore_missing):
1209             from_path = os.path.join(from_dir, filename)
1210             if ignore_missing and not os.path.exists(from_path):
1211                 return
1212             to_path = os.path.join(to_dir, filename)
1213             shutil.move(from_path, to_path)
1214
1215         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1216                       .format(appid, len(apks), keepversions, len(archapks)))
1217
1218         if len(apks) > keepversions:
1219             apklist = filter_apk_list_sorted(apks)
1220             # Move back the ones we don't want.
1221             for apk in apklist[keepversions:]:
1222                 logging.info("Moving " + apk['apkname'] + " to archive")
1223                 move_file(repodir, archivedir, apk['apkname'], False)
1224                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1225                 for density in all_screen_densities:
1226                     repo_icon_dir = get_icon_dir(repodir, density)
1227                     archive_icon_dir = get_icon_dir(archivedir, density)
1228                     if density not in apk['icons']:
1229                         continue
1230                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1231                 if 'srcname' in apk:
1232                     move_file(repodir, archivedir, apk['srcname'], False)
1233                 archapks.append(apk)
1234                 apks.remove(apk)
1235         elif len(apks) < keepversions and len(archapks) > 0:
1236             required = keepversions - len(apks)
1237             archapklist = filter_apk_list_sorted(archapks)
1238             # Move forward the ones we want again.
1239             for apk in archapklist[:required]:
1240                 logging.info("Moving " + apk['apkname'] + " from archive")
1241                 move_file(archivedir, repodir, apk['apkname'], False)
1242                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1243                 for density in all_screen_densities:
1244                     repo_icon_dir = get_icon_dir(repodir, density)
1245                     archive_icon_dir = get_icon_dir(archivedir, density)
1246                     if density not in apk['icons']:
1247                         continue
1248                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1249                 if 'srcname' in apk:
1250                     move_file(archivedir, repodir, apk['srcname'], False)
1251                 archapks.remove(apk)
1252                 apks.append(apk)
1253
1254
1255 def add_apks_to_per_app_repos(repodir, apks):
1256     apks_per_app = dict()
1257     for apk in apks:
1258         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1259         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1260         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1261         apks_per_app[apk['id']] = apk
1262
1263         if not os.path.exists(apk['per_app_icons']):
1264             logging.info('Adding new repo for only ' + apk['id'])
1265             os.makedirs(apk['per_app_icons'])
1266
1267         apkpath = os.path.join(repodir, apk['apkname'])
1268         shutil.copy(apkpath, apk['per_app_repo'])
1269         apksigpath = apkpath + '.sig'
1270         if os.path.exists(apksigpath):
1271             shutil.copy(apksigpath, apk['per_app_repo'])
1272         apkascpath = apkpath + '.asc'
1273         if os.path.exists(apkascpath):
1274             shutil.copy(apkascpath, apk['per_app_repo'])
1275
1276
1277 config = None
1278 options = None
1279
1280
1281 def main():
1282
1283     global config, options
1284
1285     # Parse command line...
1286     parser = ArgumentParser()
1287     common.setup_global_opts(parser)
1288     parser.add_argument("--create-key", action="store_true", default=False,
1289                         help="Create a repo signing key in a keystore")
1290     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1291                         help="Create skeleton metadata files that are missing")
1292     parser.add_argument("--delete-unknown", action="store_true", default=False,
1293                         help="Delete APKs and/or OBBs without metadata from the repo")
1294     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1295                         help="Report on build data status")
1296     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1297                         help="Interactively ask about things that need updating.")
1298     parser.add_argument("-I", "--icons", action="store_true", default=False,
1299                         help="Resize all the icons exceeding the max pixel size and exit")
1300     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1301                         help="Specify editor to use in interactive mode. Default " +
1302                         "is /etc/alternatives/editor")
1303     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1304                         help="Update the wiki")
1305     parser.add_argument("--pretty", action="store_true", default=False,
1306                         help="Produce human-readable index.xml")
1307     parser.add_argument("--clean", action="store_true", default=False,
1308                         help="Clean update - don't uses caches, reprocess all apks")
1309     parser.add_argument("--nosign", action="store_true", default=False,
1310                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1311     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1312                         help="Use date from apk instead of current time for newly added apks")
1313     options = parser.parse_args()
1314
1315     config = common.read_config(options)
1316
1317     if not ('jarsigner' in config and 'keytool' in config):
1318         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1319         sys.exit(1)
1320
1321     repodirs = ['repo']
1322     if config['archive_older'] != 0:
1323         repodirs.append('archive')
1324         if not os.path.exists('archive'):
1325             os.mkdir('archive')
1326
1327     if options.icons:
1328         resize_all_icons(repodirs)
1329         sys.exit(0)
1330
1331     # check that icons exist now, rather than fail at the end of `fdroid update`
1332     for k in ['repo_icon', 'archive_icon']:
1333         if k in config:
1334             if not os.path.exists(config[k]):
1335                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1336                 sys.exit(1)
1337
1338     # if the user asks to create a keystore, do it now, reusing whatever it can
1339     if options.create_key:
1340         if os.path.exists(config['keystore']):
1341             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1342             logging.critical("\t'" + config['keystore'] + "'")
1343             sys.exit(1)
1344
1345         if 'repo_keyalias' not in config:
1346             config['repo_keyalias'] = socket.getfqdn()
1347             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1348         if 'keydname' not in config:
1349             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1350             common.write_to_config(config, 'keydname', config['keydname'])
1351         if 'keystore' not in config:
1352             config['keystore'] = common.default_config.keystore
1353             common.write_to_config(config, 'keystore', config['keystore'])
1354
1355         password = common.genpassword()
1356         if 'keystorepass' not in config:
1357             config['keystorepass'] = password
1358             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1359         if 'keypass' not in config:
1360             config['keypass'] = password
1361             common.write_to_config(config, 'keypass', config['keypass'])
1362         common.genkeystore(config)
1363
1364     # Get all apps...
1365     apps = metadata.read_metadata()
1366
1367     # Generate a list of categories...
1368     categories = set()
1369     for app in apps.values():
1370         categories.update(app.Categories)
1371
1372     # Read known apks data (will be updated and written back when we've finished)
1373     knownapks = common.KnownApks()
1374
1375     # Gather information about all the apk files in the repo directory, using
1376     # cached data if possible.
1377     apkcachefile = os.path.join('tmp', 'apkcache')
1378     if not options.clean and os.path.exists(apkcachefile):
1379         with open(apkcachefile, 'rb') as cf:
1380             apkcache = pickle.load(cf, encoding='utf-8')
1381         if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
1382             apkcache = {}
1383     else:
1384         apkcache = {}
1385
1386     delete_disabled_builds(apps, apkcache, repodirs)
1387
1388     # Scan all apks in the main repo
1389     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1390
1391     # Generate warnings for apk's with no metadata (or create skeleton
1392     # metadata files, if requested on the command line)
1393     newmetadata = False
1394     for apk in apks:
1395         if apk['id'] not in apps:
1396             if options.create_metadata:
1397                 if 'name' not in apk:
1398                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1399                     continue
1400                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
1401                 f.write("License:Unknown\n")
1402                 f.write("Web Site:\n")
1403                 f.write("Source Code:\n")
1404                 f.write("Issue Tracker:\n")
1405                 f.write("Changelog:\n")
1406                 f.write("Summary:" + apk['name'] + "\n")
1407                 f.write("Description:\n")
1408                 f.write(apk['name'] + "\n")
1409                 f.write(".\n")
1410                 f.close()
1411                 logging.info("Generated skeleton metadata for " + apk['id'])
1412                 newmetadata = True
1413             else:
1414                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1415                 if options.delete_unknown:
1416                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1417                     rmf = os.path.join(repodirs[0], apk['apkname'])
1418                     if not os.path.exists(rmf):
1419                         logging.error("Could not find {0} to remove it".format(rmf))
1420                     else:
1421                         os.remove(rmf)
1422                 else:
1423                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1424
1425     # update the metadata with the newly created ones included
1426     if newmetadata:
1427         apps = metadata.read_metadata()
1428
1429     insert_obbs(repodirs[0], apps, apks)
1430
1431     # Scan the archive repo for apks as well
1432     if len(repodirs) > 1:
1433         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1434         if cc:
1435             cachechanged = True
1436     else:
1437         archapks = []
1438
1439     # Some information from the apks needs to be applied up to the application
1440     # level. When doing this, we use the info from the most recent version's apk.
1441     # We deal with figuring out when the app was added and last updated at the
1442     # same time.
1443     for appid, app in apps.items():
1444         bestver = 0
1445         for apk in apks + archapks:
1446             if apk['id'] == appid:
1447                 if apk['versioncode'] > bestver:
1448                     bestver = apk['versioncode']
1449                     bestapk = apk
1450
1451                 if 'added' in apk:
1452                     if not app.added or apk['added'] < app.added:
1453                         app.added = apk['added']
1454                     if not app.lastupdated or apk['added'] > app.lastupdated:
1455                         app.lastupdated = apk['added']
1456
1457         if not app.added:
1458             logging.debug("Don't know when " + appid + " was added")
1459         if not app.lastupdated:
1460             logging.debug("Don't know when " + appid + " was last updated")
1461
1462         if bestver == 0:
1463             if app.Name is None:
1464                 app.Name = app.AutoName or appid
1465             app.icon = None
1466             logging.debug("Application " + appid + " has no packages")
1467         else:
1468             if app.Name is None:
1469                 app.Name = bestapk['name']
1470             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1471             if app.CurrentVersionCode is None:
1472                 app.CurrentVersionCode = str(bestver)
1473
1474     # Sort the app list by name, then the web site doesn't have to by default.
1475     # (we had to wait until we'd scanned the apks to do this, because mostly the
1476     # name comes from there!)
1477     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1478
1479     # APKs are placed into multiple repos based on the app package, providing
1480     # per-app subscription feeds for nightly builds and things like it
1481     if config['per_app_repos']:
1482         add_apks_to_per_app_repos(repodirs[0], apks)
1483         for appid, app in apps.items():
1484             repodir = os.path.join(appid, 'fdroid', 'repo')
1485             appdict = dict()
1486             appdict[appid] = app
1487             if os.path.isdir(repodir):
1488                 make_index(appdict, [appid], apks, repodir, False, categories)
1489             else:
1490                 logging.info('Skipping index generation for ' + appid)
1491         return
1492
1493     if len(repodirs) > 1:
1494         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1495
1496     # Make the index for the main repo...
1497     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1498
1499     # If there's an archive repo,  make the index for it. We already scanned it
1500     # earlier on.
1501     if len(repodirs) > 1:
1502         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1503
1504     if config['update_stats']:
1505
1506         # Update known apks info...
1507         knownapks.writeifchanged()
1508
1509         # Generate latest apps data for widget
1510         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1511             data = ''
1512             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1513                 for line in f:
1514                     appid = line.rstrip()
1515                     data += appid + "\t"
1516                     app = apps[appid]
1517                     data += app.Name + "\t"
1518                     if app.icon is not None:
1519                         data += app.icon + "\t"
1520                     data += app.License + "\n"
1521             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1522                 f.write(data)
1523
1524     if cachechanged:
1525         apkcache["METADATA_VERSION"] = METADATA_VERSION
1526         with open(apkcachefile, 'wb') as cf:
1527             pickle.dump(apkcache, cf)
1528
1529     # Update the wiki...
1530     if options.wiki:
1531         update_wiki(apps, sortedids, apks + archapks)
1532
1533     logging.info("Finished.")
1534
1535 if __name__ == "__main__":
1536     main()