fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   5 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU Affero General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Affero General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Affero General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 import sys
  21 import os
  22 import shutil
  23 import glob
  24 import re
  25 import socket
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 import urllib.parse
  30 from datetime import datetime, timedelta
  31 from xml.dom.minidom import Document
  32 from argparse import ArgumentParser
  33 import time
  34 from pyasn1.error import PyAsn1Error
  35 from pyasn1.codec.der import decoder, encoder
  36 from pyasn1_modules import rfc2315
  37 from binascii import hexlify, unhexlify
  38
  39 from PIL import Image
  40 import logging
  41
  42 from . import common
  43 from . import metadata
  44 from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
  45 from .metadata import MetaDataException
  46
  47 METADATA_VERSION = 16
  48
  49 screen_densities = ['640', '480', '320', '240', '160', '120']
  50
  51 all_screen_densities = ['0'] + screen_densities
  52
  53
  54 def dpi_to_px(density):
  55     return (int(density) * 48) / 160
  56
  57
  58 def px_to_dpi(px):
  59     return (int(px) * 160) / 48
  60
  61
  62 def get_icon_dir(repodir, density):
  63     if density == '0':
  64         return os.path.join(repodir, "icons")
  65     return os.path.join(repodir, "icons-%s" % density)
  66
  67
  68 def get_icon_dirs(repodir):
  69     for density in screen_densities:
  70         yield get_icon_dir(repodir, density)
  71
  72
  73 def get_all_icon_dirs(repodir):
  74     for density in all_screen_densities:
  75         yield get_icon_dir(repodir, density)
  76
  77
  78 def update_wiki(apps, sortedids, apks):
  79     """Update the wiki
  80
  81     :param apps: fully populated list of all applications
  82     :param apks: all apks, except...
  83     """
  84     logging.info("Updating wiki")
  85     wikicat = 'Apps'
  86     wikiredircat = 'App Redirects'
  87     import mwclient
  88     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  89                          path=config['wiki_path'])
  90     site.login(config['wiki_user'], config['wiki_password'])
  91     generated_pages = {}
  92     generated_redirects = {}
  93
  94     for appid in sortedids:
  95         app = apps[appid]
  96
  97         wikidata = ''
  98         if app.Disabled:
  99             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 100         if app.AntiFeatures:
 101             for af in app.AntiFeatures:
 102                 wikidata += '{{AntiFeature|' + af + '}}\n'
 103         if app.RequiresRoot:
 104             requiresroot = 'Yes'
 105         else:
 106             requiresroot = 'No'
 107         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 108             appid,
 109             app.Name,
 110             time.strftime('%Y-%m-%d', app.added) if app.added else '',
 111             time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
 112             app.SourceCode,
 113             app.IssueTracker,
 114             app.WebSite,
 115             app.Changelog,
 116             app.Donate,
 117             app.FlattrID,
 118             app.Bitcoin,
 119             app.Litecoin,
 120             app.License,
 121             requiresroot,
 122             app.AuthorName,
 123             app.AuthorEmail)
 124
 125         if app.Provides:
 126             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 127
 128         wikidata += app.Summary
 129         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 130
 131         wikidata += "=Description=\n"
 132         wikidata += metadata.description_wiki(app.Description) + "\n"
 133
 134         wikidata += "=Maintainer Notes=\n"
 135         if app.MaintainerNotes:
 136             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 137         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 138
 139         # Get a list of all packages for this application...
 140         apklist = []
 141         gotcurrentver = False
 142         cantupdate = False
 143         buildfails = False
 144         for apk in apks:
 145             if apk['id'] == appid:
 146                 if str(apk['versioncode']) == app.CurrentVersionCode:
 147                     gotcurrentver = True
 148                 apklist.append(apk)
 149         # Include ones we can't build, as a special case...
 150         for build in app.builds:
 151             if build.disable:
 152                 if build.vercode == app.CurrentVersionCode:
 153                     cantupdate = True
 154                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 155                 apklist.append({'versioncode': int(build.vercode),
 156                                 'version': build.version,
 157                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 158                                 })
 159             else:
 160                 builtit = False
 161                 for apk in apklist:
 162                     if apk['versioncode'] == int(build.vercode):
 163                         builtit = True
 164                         break
 165                 if not builtit:
 166                     buildfails = True
 167                     apklist.append({'versioncode': int(build.vercode),
 168                                     'version': build.version,
 169                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
 170                                     })
 171         if app.CurrentVersionCode == '0':
 172             cantupdate = True
 173         # Sort with most recent first...
 174         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 175
 176         wikidata += "=Versions=\n"
 177         if len(apklist) == 0:
 178             wikidata += "We currently have no versions of this app available."
 179         elif not gotcurrentver:
 180             wikidata += "We don't have the current version of this app."
 181         else:
 182             wikidata += "We have the current version of this app."
 183         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 184         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 185         if len(app.NoSourceSince) > 0:
 186             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 187         if len(app.CurrentVersion) > 0:
 188             wikidata += "The current (recommended) version is " + app.CurrentVersion
 189             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 190         validapks = 0
 191         for apk in apklist:
 192             wikidata += "==" + apk['version'] + "==\n"
 193
 194             if 'buildproblem' in apk:
 195                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 196             else:
 197                 validapks += 1
 198                 wikidata += "This version is built and signed by "
 199                 if 'srcname' in apk:
 200                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 201                 else:
 202                     wikidata += "the original developer.\n\n"
 203             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 204
 205         wikidata += '\n[[Category:' + wikicat + ']]\n'
 206         if len(app.NoSourceSince) > 0:
 207             wikidata += '\n[[Category:Apps missing source code]]\n'
 208         if validapks == 0 and not app.Disabled:
 209             wikidata += '\n[[Category:Apps with no packages]]\n'
 210         if cantupdate and not app.Disabled:
 211             wikidata += "\n[[Category:Apps we cannot update]]\n"
 212         if buildfails and not app.Disabled:
 213             wikidata += "\n[[Category:Apps with failing builds]]\n"
 214         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 215             wikidata += '\n[[Category:Apps to Update]]\n'
 216         if app.Disabled:
 217             wikidata += '\n[[Category:Apps that are disabled]]\n'
 218         if app.UpdateCheckMode == 'None' and not app.Disabled:
 219             wikidata += '\n[[Category:Apps with no update check]]\n'
 220         for appcat in app.Categories:
 221             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 222
 223         # We can't have underscores in the page name, even if they're in
 224         # the package ID, because MediaWiki messes with them...
 225         pagename = appid.replace('_', ' ')
 226
 227         # Drop a trailing newline, because mediawiki is going to drop it anyway
 228         # and it we don't we'll think the page has changed when it hasn't...
 229         if wikidata.endswith('\n'):
 230             wikidata = wikidata[:-1]
 231
 232         generated_pages[pagename] = wikidata
 233
 234         # Make a redirect from the name to the ID too, unless there's
 235         # already an existing page with the name and it isn't a redirect.
 236         noclobber = False
 237         apppagename = app.Name.replace('_', ' ')
 238         apppagename = apppagename.replace('{', '')
 239         apppagename = apppagename.replace('}', ' ')
 240         apppagename = apppagename.replace(':', ' ')
 241         # Drop double spaces caused mostly by replacing ':' above
 242         apppagename = apppagename.replace('  ', ' ')
 243         for expagename in site.allpages(prefix=apppagename,
 244                                         filterredir='nonredirects',
 245                                         generator=False):
 246             if expagename == apppagename:
 247                 noclobber = True
 248         # Another reason not to make the redirect page is if the app name
 249         # is the same as it's ID, because that will overwrite the real page
 250         # with an redirect to itself! (Although it seems like an odd
 251         # scenario this happens a lot, e.g. where there is metadata but no
 252         # builds or binaries to extract a name from.
 253         if apppagename == pagename:
 254             noclobber = True
 255         if not noclobber:
 256             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 257
 258     for tcat, genp in [(wikicat, generated_pages),
 259                        (wikiredircat, generated_redirects)]:
 260         catpages = site.Pages['Category:' + tcat]
 261         existingpages = []
 262         for page in catpages:
 263             existingpages.append(page.name)
 264             if page.name in genp:
 265                 pagetxt = page.edit()
 266                 if pagetxt != genp[page.name]:
 267                     logging.debug("Updating modified page " + page.name)
 268                     page.save(genp[page.name], summary='Auto-updated')
 269                 else:
 270                     logging.debug("Page " + page.name + " is unchanged")
 271             else:
 272                 logging.warn("Deleting page " + page.name)
 273                 page.delete('No longer published')
 274         for pagename, text in genp.items():
 275             logging.debug("Checking " + pagename)
 276             if pagename not in existingpages:
 277                 logging.debug("Creating page " + pagename)
 278                 try:
 279                     newpage = site.Pages[pagename]
 280                     newpage.save(text, summary='Auto-created')
 281                 except:
 282                     logging.error("...FAILED to create page '{0}'".format(pagename))
 283
 284     # Purge server cache to ensure counts are up to date
 285     site.pages['Repository Maintenance'].purge()
 286
 287
 288 def delete_disabled_builds(apps, apkcache, repodirs):
 289     """Delete disabled build outputs.
 290
 291     :param apps: list of all applications, as per metadata.read_metadata
 292     :param apkcache: current apk cache information
 293     :param repodirs: the repo directories to process
 294     """
 295     for appid, app in apps.items():
 296         for build in app.builds:
 297             if not build.disable:
 298                 continue
 299             apkfilename = appid + '_' + str(build.vercode) + '.apk'
 300             iconfilename = "%s.%s.png" % (
 301                 appid,
 302                 build.vercode)
 303             for repodir in repodirs:
 304                 files = [
 305                     os.path.join(repodir, apkfilename),
 306                     os.path.join(repodir, apkfilename + '.asc'),
 307                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 308                 ]
 309                 for density in all_screen_densities:
 310                     repo_dir = get_icon_dir(repodir, density)
 311                     files.append(os.path.join(repo_dir, iconfilename))
 312
 313                 for f in files:
 314                     if os.path.exists(f):
 315                         logging.info("Deleting disabled build output " + f)
 316                         os.remove(f)
 317             if apkfilename in apkcache:
 318                 del apkcache[apkfilename]
 319
 320
 321 def resize_icon(iconpath, density):
 322
 323     if not os.path.isfile(iconpath):
 324         return
 325
 326     fp = None
 327     try:
 328         fp = open(iconpath, 'rb')
 329         im = Image.open(fp)
 330         size = dpi_to_px(density)
 331
 332         if any(length > size for length in im.size):
 333             oldsize = im.size
 334             im.thumbnail((size, size), Image.ANTIALIAS)
 335             logging.debug("%s was too large at %s - new size is %s" % (
 336                 iconpath, oldsize, im.size))
 337             im.save(iconpath, "PNG")
 338
 339     except Exception as e:
 340         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 341
 342     finally:
 343         if fp:
 344             fp.close()
 345
 346
 347 def resize_all_icons(repodirs):
 348     """Resize all icons that exceed the max size
 349
 350     :param repodirs: the repo directories to process
 351     """
 352     for repodir in repodirs:
 353         for density in screen_densities:
 354             icon_dir = get_icon_dir(repodir, density)
 355             icon_glob = os.path.join(icon_dir, '*.png')
 356             for iconpath in glob.glob(icon_glob):
 357                 resize_icon(iconpath, density)
 358
 359
 360 # A signature block file with a .DSA, .RSA, or .EC extension
 361 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 362
 363
 364 def getsig(apkpath):
 365     """ Get the signing certificate of an apk. To get the same md5 has that
 366     Android gets, we encode the .RSA certificate in a specific format and pass
 367     it hex-encoded to the md5 digest algorithm.
 368
 369     :param apkpath: path to the apk
 370     :returns: A string containing the md5 of the signature of the apk or None
 371               if an error occurred.
 372     """
 373
 374     cert = None
 375
 376     # verify the jar signature is correct
 377     args = [config['jarsigner'], '-verify', apkpath]
 378     p = FDroidPopen(args)
 379     if p.returncode != 0:
 380         logging.critical(apkpath + " has a bad signature!")
 381         return None
 382
 383     with zipfile.ZipFile(apkpath, 'r') as apk:
 384
 385         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 386
 387         if len(certs) < 1:
 388             logging.error("Found no signing certificates on %s" % apkpath)
 389             return None
 390         if len(certs) > 1:
 391             logging.error("Found multiple signing certificates on %s" % apkpath)
 392             return None
 393
 394         cert = apk.read(certs[0])
 395
 396     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 397     if content.getComponentByName('contentType') != rfc2315.signedData:
 398         logging.error("Unexpected format.")
 399         return None
 400
 401     content = decoder.decode(content.getComponentByName('content'),
 402                              asn1Spec=rfc2315.SignedData())[0]
 403     try:
 404         certificates = content.getComponentByName('certificates')
 405     except PyAsn1Error:
 406         logging.error("Certificates not found.")
 407         return None
 408
 409     cert_encoded = encoder.encode(certificates)[4:]
 410
 411     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 412
 413
 414 def get_icon_bytes(apkzip, iconsrc):
 415     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 416     try:
 417         return apkzip.read(iconsrc)
 418     except KeyError:
 419         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 420
 421
 422 def sha256sum(filename):
 423     '''Calculate the sha256 of the given file'''
 424     sha = hashlib.sha256()
 425     with open(filename, 'rb') as f:
 426         while True:
 427             t = f.read(16384)
 428             if len(t) == 0:
 429                 break
 430             sha.update(t)
 431     return sha.hexdigest()
 432
 433
 434 def insert_obbs(repodir, apps, apks):
 435     """Scans the .obb files in a given repo directory and adds them to the
 436     relevant APK instances.  OBB files have versionCodes like APK
 437     files, and they are loosely associated.  If there is an OBB file
 438     present, then any APK with the same or higher versionCode will use
 439     that OBB file.  There are two OBB types: main and patch, each APK
 440     can only have only have one of each.
 441
 442     https://developer.android.com/google/play/expansion-files.html
 443
 444     :param repodir: repo directory to scan
 445     :param apps: list of current, valid apps
 446     :param apks: current information on all APKs
 447
 448     """
 449
 450     def obbWarnDelete(f, msg):
 451         logging.warning(msg + f)
 452         if options.delete_unknown:
 453             logging.error("Deleting unknown file: " + f)
 454             os.remove(f)
 455
 456     obbs = []
 457     java_Integer_MIN_VALUE = -pow(2, 31)
 458     for f in glob.glob(os.path.join(repodir, '*.obb')):
 459         obbfile = os.path.basename(f)
 460         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 461         chunks = obbfile.split('.')
 462         if chunks[0] != 'main' and chunks[0] != 'patch':
 463             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 464             continue
 465         if not re.match(r'^-?[0-9]+$', chunks[1]):
 466             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 467             continue
 468         versioncode = int(chunks[1])
 469         packagename = ".".join(chunks[2:-1])
 470
 471         highestVersionCode = java_Integer_MIN_VALUE
 472         if packagename not in apps.keys():
 473             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 474             continue
 475         for apk in apks:
 476             if packagename == apk['id'] and apk['versioncode'] > highestVersionCode:
 477                 highestVersionCode = apk['versioncode']
 478         if versioncode > highestVersionCode:
 479             obbWarnDelete(f, 'OBB file has newer versioncode(' + str(versioncode)
 480                           + ') than any APK: ')
 481             continue
 482         obbsha256 = sha256sum(f)
 483         obbs.append((packagename, versioncode, obbfile, obbsha256))
 484
 485     for apk in apks:
 486         for (packagename, versioncode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 487             if versioncode <= apk['versioncode'] and packagename == apk['id']:
 488                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 489                     apk['obbMainFile'] = obbfile
 490                     apk['obbMainFileSha256'] = obbsha256
 491                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 492                     apk['obbPatchFile'] = obbfile
 493                     apk['obbPatchFileSha256'] = obbsha256
 494             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 495                 break
 496
 497
 498 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
 499     """Scan the apks in the given repo directory.
 500
 501     This also extracts the icons.
 502
 503     :param apps: list of all applications, as per metadata.read_metadata
 504     :param apkcache: current apk cache information
 505     :param repodir: repo directory to scan
 506     :param knownapks: known apks info
 507     :param use_date_from_apk: use date from APK (instead of current date)
 508                               for newly added APKs
 509     :returns: (apks, cachechanged) where apks is a list of apk information,
 510               and cachechanged is True if the apkcache got changed.
 511     """
 512
 513     cachechanged = False
 514
 515     for icon_dir in get_all_icon_dirs(repodir):
 516         if os.path.exists(icon_dir):
 517             if options.clean:
 518                 shutil.rmtree(icon_dir)
 519                 os.makedirs(icon_dir)
 520         else:
 521             os.makedirs(icon_dir)
 522
 523     apks = []
 524     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 525     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 526     vername_pat = re.compile(".*versionName='([^']*)'.*")
 527     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 528     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 529     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 530     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 531     string_pat = re.compile(".* name='([^']*)'.*")
 532     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 533
 534         apkfilename = apkfile[len(repodir) + 1:]
 535         if ' ' in apkfilename:
 536             logging.critical("Spaces in filenames are not allowed.")
 537             sys.exit(1)
 538
 539         shasum = sha256sum(apkfile)
 540
 541         usecache = False
 542         if apkfilename in apkcache:
 543             apk = apkcache[apkfilename]
 544             if apk['sha256'] == shasum:
 545                 logging.debug("Reading " + apkfilename + " from cache")
 546                 usecache = True
 547             else:
 548                 logging.debug("Ignoring stale cache data for " + apkfilename)
 549
 550         if not usecache:
 551             logging.debug("Processing " + apkfilename)
 552             apk = {}
 553             apk['apkname'] = apkfilename
 554             apk['sha256'] = shasum
 555             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 556             if os.path.exists(os.path.join(repodir, srcfilename)):
 557                 apk['srcname'] = srcfilename
 558             apk['size'] = os.path.getsize(apkfile)
 559             apk['permissions'] = set()
 560             apk['features'] = set()
 561             apk['icons_src'] = {}
 562             apk['icons'] = {}
 563             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 564             if p.returncode != 0:
 565                 if options.delete_unknown:
 566                     if os.path.exists(apkfile):
 567                         logging.error("Failed to get apk information, deleting " + apkfile)
 568                         os.remove(apkfile)
 569                     else:
 570                         logging.error("Could not find {0} to remove it".format(apkfile))
 571                 else:
 572                     logging.error("Failed to get apk information, skipping " + apkfile)
 573                 continue
 574             for line in p.output.splitlines():
 575                 if line.startswith("package:"):
 576                     try:
 577                         apk['id'] = re.match(name_pat, line).group(1)
 578                         apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
 579                         apk['version'] = re.match(vername_pat, line).group(1)
 580                     except Exception as e:
 581                         logging.error("Package matching failed: " + str(e))
 582                         logging.info("Line was: " + line)
 583                         sys.exit(1)
 584                 elif line.startswith("application:"):
 585                     apk['name'] = re.match(label_pat, line).group(1)
 586                     # Keep path to non-dpi icon in case we need it
 587                     match = re.match(icon_pat_nodpi, line)
 588                     if match:
 589                         apk['icons_src']['-1'] = match.group(1)
 590                 elif line.startswith("launchable-activity:"):
 591                     # Only use launchable-activity as fallback to application
 592                     if not apk['name']:
 593                         apk['name'] = re.match(label_pat, line).group(1)
 594                     if '-1' not in apk['icons_src']:
 595                         match = re.match(icon_pat_nodpi, line)
 596                         if match:
 597                             apk['icons_src']['-1'] = match.group(1)
 598                 elif line.startswith("application-icon-"):
 599                     match = re.match(icon_pat, line)
 600                     if match:
 601                         density = match.group(1)
 602                         path = match.group(2)
 603                         apk['icons_src'][density] = path
 604                 elif line.startswith("sdkVersion:"):
 605                     m = re.match(sdkversion_pat, line)
 606                     if m is None:
 607                         logging.error(line.replace('sdkVersion:', '')
 608                                       + ' is not a valid minSdkVersion!')
 609                     else:
 610                         apk['minSdkVersion'] = m.group(1)
 611                         # if target not set, default to min
 612                         if 'targetSdkVersion' not in apk:
 613                             apk['targetSdkVersion'] = m.group(1)
 614                 elif line.startswith("targetSdkVersion:"):
 615                     m = re.match(sdkversion_pat, line)
 616                     if m is None:
 617                         logging.error(line.replace('targetSdkVersion:', '')
 618                                       + ' is not a valid targetSdkVersion!')
 619                     else:
 620                         apk['targetSdkVersion'] = m.group(1)
 621                 elif line.startswith("maxSdkVersion:"):
 622                     apk['maxSdkVersion'] = re.match(sdkversion_pat, line).group(1)
 623                 elif line.startswith("native-code:"):
 624                     apk['nativecode'] = []
 625                     for arch in line[13:].split(' '):
 626                         apk['nativecode'].append(arch[1:-1])
 627                 elif line.startswith("uses-permission:"):
 628                     perm = re.match(string_pat, line).group(1)
 629                     if perm.startswith("android.permission."):
 630                         perm = perm[19:]
 631                     apk['permissions'].add(perm)
 632                 elif line.startswith("uses-feature:"):
 633                     perm = re.match(string_pat, line).group(1)
 634                     # Filter out this, it's only added with the latest SDK tools and
 635                     # causes problems for lots of apps.
 636                     if perm != "android.hardware.screen.portrait" \
 637                             and perm != "android.hardware.screen.landscape":
 638                         if perm.startswith("android.feature."):
 639                             perm = perm[16:]
 640                         apk['features'].add(perm)
 641
 642             if 'minSdkVersion' not in apk:
 643                 logging.warn("No SDK version information found in {0}".format(apkfile))
 644                 apk['minSdkVersion'] = 1
 645
 646             # Check for debuggable apks...
 647             if common.isApkDebuggable(apkfile, config):
 648                 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
 649
 650             # Get the signature (or md5 of, to be precise)...
 651             logging.debug('Getting signature of {0}'.format(apkfile))
 652             apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 653             if not apk['sig']:
 654                 logging.critical("Failed to get apk signature")
 655                 sys.exit(1)
 656
 657             apkzip = zipfile.ZipFile(apkfile, 'r')
 658
 659             # if an APK has files newer than the system time, suggest updating
 660             # the system clock.  This is useful for offline systems, used for
 661             # signing, which do not have another source of clock sync info. It
 662             # has to be more than 24 hours newer because ZIP/APK files do not
 663             # store timezone info
 664             manifest = apkzip.getinfo('AndroidManifest.xml')
 665             if manifest.date_time[1] == 0:  # month can't be zero
 666                 logging.debug('AndroidManifest.xml has no date')
 667             else:
 668                 dt_obj = datetime(*manifest.date_time)
 669                 checkdt = dt_obj - timedelta(1)
 670                 if datetime.today() < checkdt:
 671                     logging.warn('System clock is older than manifest in: '
 672                                  + apkfilename
 673                                  + '\nSet clock to that time using:\n'
 674                                  + 'sudo date -s "' + str(dt_obj) + '"')
 675
 676             iconfilename = "%s.%s.png" % (
 677                 apk['id'],
 678                 apk['versioncode'])
 679
 680             # Extract the icon file...
 681             empty_densities = []
 682             for density in screen_densities:
 683                 if density not in apk['icons_src']:
 684                     empty_densities.append(density)
 685                     continue
 686                 iconsrc = apk['icons_src'][density]
 687                 icon_dir = get_icon_dir(repodir, density)
 688                 icondest = os.path.join(icon_dir, iconfilename)
 689
 690                 try:
 691                     with open(icondest, 'wb') as f:
 692                         f.write(get_icon_bytes(apkzip, iconsrc))
 693                     apk['icons'][density] = iconfilename
 694
 695                 except:
 696                     logging.warn("Error retrieving icon file")
 697                     del apk['icons'][density]
 698                     del apk['icons_src'][density]
 699                     empty_densities.append(density)
 700
 701             if '-1' in apk['icons_src']:
 702                 iconsrc = apk['icons_src']['-1']
 703                 iconpath = os.path.join(
 704                     get_icon_dir(repodir, '0'), iconfilename)
 705                 with open(iconpath, 'wb') as f:
 706                     f.write(get_icon_bytes(apkzip, iconsrc))
 707                 try:
 708                     im = Image.open(iconpath)
 709                     dpi = px_to_dpi(im.size[0])
 710                     for density in screen_densities:
 711                         if density in apk['icons']:
 712                             break
 713                         if density == screen_densities[-1] or dpi >= int(density):
 714                             apk['icons'][density] = iconfilename
 715                             shutil.move(iconpath,
 716                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 717                             empty_densities.remove(density)
 718                             break
 719                 except Exception as e:
 720                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 721
 722             if apk['icons']:
 723                 apk['icon'] = iconfilename
 724
 725             apkzip.close()
 726
 727             # First try resizing down to not lose quality
 728             last_density = None
 729             for density in screen_densities:
 730                 if density not in empty_densities:
 731                     last_density = density
 732                     continue
 733                 if last_density is None:
 734                     continue
 735                 logging.debug("Density %s not available, resizing down from %s"
 736                               % (density, last_density))
 737
 738                 last_iconpath = os.path.join(
 739                     get_icon_dir(repodir, last_density), iconfilename)
 740                 iconpath = os.path.join(
 741                     get_icon_dir(repodir, density), iconfilename)
 742                 fp = None
 743                 try:
 744                     fp = open(last_iconpath, 'rb')
 745                     im = Image.open(fp)
 746
 747                     size = dpi_to_px(density)
 748
 749                     im.thumbnail((size, size), Image.ANTIALIAS)
 750                     im.save(iconpath, "PNG")
 751                     empty_densities.remove(density)
 752                 except:
 753                     logging.warning("Invalid image file at %s" % last_iconpath)
 754                 finally:
 755                     if fp:
 756                         fp.close()
 757
 758             # Then just copy from the highest resolution available
 759             last_density = None
 760             for density in reversed(screen_densities):
 761                 if density not in empty_densities:
 762                     last_density = density
 763                     continue
 764                 if last_density is None:
 765                     continue
 766                 logging.debug("Density %s not available, copying from lower density %s"
 767                               % (density, last_density))
 768
 769                 shutil.copyfile(
 770                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 771                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 772
 773                 empty_densities.remove(density)
 774
 775             for density in screen_densities:
 776                 icon_dir = get_icon_dir(repodir, density)
 777                 icondest = os.path.join(icon_dir, iconfilename)
 778                 resize_icon(icondest, density)
 779
 780             # Copy from icons-mdpi to icons since mdpi is the baseline density
 781             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 782             if os.path.isfile(baseline):
 783                 apk['icons']['0'] = iconfilename
 784                 shutil.copyfile(baseline,
 785                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 786
 787             # Record in known apks, getting the added date at the same time..
 788             added = knownapks.recordapk(apk['apkname'], apk['id'])
 789             if added:
 790                 if use_date_from_apk and manifest.date_time[1] != 0:
 791                     added = datetime(*manifest.date_time).timetuple()
 792                     logging.debug("Using date from APK")
 793
 794                 apk['added'] = added
 795
 796             apkcache[apkfilename] = apk
 797             cachechanged = True
 798
 799         apks.append(apk)
 800
 801     return apks, cachechanged
 802
 803
 804 repo_pubkey_fingerprint = None
 805
 806
 807 # Generate a certificate fingerprint the same way keytool does it
 808 # (but with slightly different formatting)
 809 def cert_fingerprint(data):
 810     digest = hashlib.sha256(data).digest()
 811     ret = []
 812     ret.append(' '.join("%02X" % b for b in bytearray(digest)))
 813     return " ".join(ret)
 814
 815
 816 def extract_pubkey():
 817     global repo_pubkey_fingerprint
 818     if 'repo_pubkey' in config:
 819         pubkey = unhexlify(config['repo_pubkey'])
 820     else:
 821         p = FDroidPopenBytes([config['keytool'], '-exportcert',
 822                               '-alias', config['repo_keyalias'],
 823                               '-keystore', config['keystore'],
 824                               '-storepass:file', config['keystorepassfile']]
 825                              + config['smartcardoptions'],
 826                              output=False, stderr_to_stdout=False)
 827         if p.returncode != 0 or len(p.output) < 20:
 828             msg = "Failed to get repo pubkey!"
 829             if config['keystore'] == 'NONE':
 830                 msg += ' Is your crypto smartcard plugged in?'
 831             logging.critical(msg)
 832             sys.exit(1)
 833         pubkey = p.output
 834     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 835     return hexlify(pubkey)
 836
 837
 838 def make_index(apps, sortedids, apks, repodir, archive, categories):
 839     """Make a repo index.
 840
 841     :param apps: fully populated apps list
 842     :param apks: full populated apks list
 843     :param repodir: the repo directory
 844     :param archive: True if this is the archive repo, False if it's the
 845                     main one.
 846     :param categories: list of categories
 847     """
 848
 849     doc = Document()
 850
 851     def addElement(name, value, doc, parent):
 852         el = doc.createElement(name)
 853         el.appendChild(doc.createTextNode(value))
 854         parent.appendChild(el)
 855
 856     def addElementNonEmpty(name, value, doc, parent):
 857         if not value:
 858             return
 859         addElement(name, value, doc, parent)
 860
 861     def addElementCDATA(name, value, doc, parent):
 862         el = doc.createElement(name)
 863         el.appendChild(doc.createCDATASection(value))
 864         parent.appendChild(el)
 865
 866     root = doc.createElement("fdroid")
 867     doc.appendChild(root)
 868
 869     repoel = doc.createElement("repo")
 870
 871     mirrorcheckfailed = False
 872     for mirror in config.get('mirrors', []):
 873         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
 874         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
 875             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 876             mirrorcheckfailed = True
 877     if mirrorcheckfailed:
 878         sys.exit(1)
 879
 880     if archive:
 881         repoel.setAttribute("name", config['archive_name'])
 882         if config['repo_maxage'] != 0:
 883             repoel.setAttribute("maxage", str(config['repo_maxage']))
 884         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 885         repoel.setAttribute("url", config['archive_url'])
 886         addElement('description', config['archive_description'], doc, repoel)
 887         urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
 888         for mirror in config.get('mirrors', []):
 889             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 890
 891     else:
 892         repoel.setAttribute("name", config['repo_name'])
 893         if config['repo_maxage'] != 0:
 894             repoel.setAttribute("maxage", str(config['repo_maxage']))
 895         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 896         repoel.setAttribute("url", config['repo_url'])
 897         addElement('description', config['repo_description'], doc, repoel)
 898         urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
 899         for mirror in config.get('mirrors', []):
 900             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 901
 902     repoel.setAttribute("version", str(METADATA_VERSION))
 903     repoel.setAttribute("timestamp", str(int(time.time())))
 904
 905     nosigningkey = False
 906     if not options.nosign:
 907         if 'repo_keyalias' not in config:
 908             nosigningkey = True
 909             logging.critical("'repo_keyalias' not found in config.py!")
 910         if 'keystore' not in config:
 911             nosigningkey = True
 912             logging.critical("'keystore' not found in config.py!")
 913         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 914             nosigningkey = True
 915             logging.critical("'keystorepass' not found in config.py!")
 916         if 'keypass' not in config and 'keypassfile' not in config:
 917             nosigningkey = True
 918             logging.critical("'keypass' not found in config.py!")
 919         if not os.path.exists(config['keystore']):
 920             nosigningkey = True
 921             logging.critical("'" + config['keystore'] + "' does not exist!")
 922         if nosigningkey:
 923             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 924             logging.warning("\tfdroid update --create-key")
 925             sys.exit(1)
 926
 927     repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
 928     root.appendChild(repoel)
 929
 930     for appid in sortedids:
 931         app = apps[appid]
 932
 933         if app.Disabled is not None:
 934             continue
 935
 936         # Get a list of the apks for this app...
 937         apklist = []
 938         for apk in apks:
 939             if apk['id'] == appid:
 940                 apklist.append(apk)
 941
 942         if len(apklist) == 0:
 943             continue
 944
 945         apel = doc.createElement("application")
 946         apel.setAttribute("id", app.id)
 947         root.appendChild(apel)
 948
 949         addElement('id', app.id, doc, apel)
 950         if app.added:
 951             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
 952         if app.lastupdated:
 953             addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
 954         addElement('name', app.Name, doc, apel)
 955         addElement('summary', app.Summary, doc, apel)
 956         if app.icon:
 957             addElement('icon', app.icon, doc, apel)
 958
 959         def linkres(appid):
 960             if appid in apps:
 961                 return ("fdroid.app:" + appid, apps[appid].Name)
 962             raise MetaDataException("Cannot resolve app id " + appid)
 963
 964         addElement('desc',
 965                    metadata.description_html(app.Description, linkres),
 966                    doc, apel)
 967         addElement('license', app.License, doc, apel)
 968         if app.Categories:
 969             addElement('categories', ','.join(app.Categories), doc, apel)
 970             # We put the first (primary) category in LAST, which will have
 971             # the desired effect of making clients that only understand one
 972             # category see that one.
 973             addElement('category', app.Categories[0], doc, apel)
 974         addElement('web', app.WebSite, doc, apel)
 975         addElement('source', app.SourceCode, doc, apel)
 976         addElement('tracker', app.IssueTracker, doc, apel)
 977         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 978         addElementNonEmpty('author', app.AuthorName, doc, apel)
 979         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 980         addElementNonEmpty('donate', app.Donate, doc, apel)
 981         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 982         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 983         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 984
 985         # These elements actually refer to the current version (i.e. which
 986         # one is recommended. They are historically mis-named, and need
 987         # changing, but stay like this for now to support existing clients.
 988         addElement('marketversion', app.CurrentVersion, doc, apel)
 989         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 990
 991         if app.AntiFeatures:
 992             af = app.AntiFeatures
 993             if af:
 994                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 995         if app.Provides:
 996             pv = app.Provides.split(',')
 997             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 998         if app.RequiresRoot:
 999             addElement('requirements', 'root', doc, apel)
1000
1001         # Sort the apk list into version order, just so the web site
1002         # doesn't have to do any work by default...
1003         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
1004
1005         # Check for duplicates - they will make the client unhappy...
1006         for i in range(len(apklist) - 1):
1007             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
1008                 logging.critical("duplicate versions: '%s' - '%s'" % (
1009                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
1010                 sys.exit(1)
1011
1012         current_version_code = 0
1013         current_version_file = None
1014         for apk in apklist:
1015             # find the APK for the "Current Version"
1016             if current_version_code < apk['versioncode']:
1017                 current_version_code = apk['versioncode']
1018             if current_version_code < int(app.CurrentVersionCode):
1019                 current_version_file = apk['apkname']
1020
1021             apkel = doc.createElement("package")
1022             apel.appendChild(apkel)
1023             addElement('version', apk['version'], doc, apkel)
1024             addElement('versioncode', str(apk['versioncode']), doc, apkel)
1025             addElement('apkname', apk['apkname'], doc, apkel)
1026             if 'srcname' in apk:
1027                 addElement('srcname', apk['srcname'], doc, apkel)
1028             for hash_type in ['sha256']:
1029                 if hash_type not in apk:
1030                     continue
1031                 hashel = doc.createElement("hash")
1032                 hashel.setAttribute("type", hash_type)
1033                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
1034                 apkel.appendChild(hashel)
1035             addElement('sig', apk['sig'], doc, apkel)
1036             addElement('size', str(apk['size']), doc, apkel)
1037             addElement('sdkver', str(apk['minSdkVersion']), doc, apkel)
1038             if 'targetSdkVersion' in apk:
1039                 addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
1040             if 'maxSdkVersion' in apk:
1041                 addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
1042             addElementNonEmpty('obbMainFile', apk.get('obbMainFile'), doc, apkel)
1043             addElementNonEmpty('obbMainFileSha256', apk.get('obbMainFileSha256'), doc, apkel)
1044             addElementNonEmpty('obbPatchFile', apk.get('obbPatchFile'), doc, apkel)
1045             addElementNonEmpty('obbPatchFileSha256', apk.get('obbPatchFileSha256'), doc, apkel)
1046             if 'added' in apk:
1047                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
1048             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
1049             if 'nativecode' in apk:
1050                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
1051             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
1052
1053         if current_version_file is not None \
1054                 and config['make_current_version_link'] \
1055                 and repodir == 'repo':  # only create these
1056             namefield = config['current_version_name_source']
1057             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
1058             apklinkname = sanitized_name + '.apk'
1059             current_version_path = os.path.join(repodir, current_version_file)
1060             if os.path.islink(apklinkname):
1061                 os.remove(apklinkname)
1062             os.symlink(current_version_path, apklinkname)
1063             # also symlink gpg signature, if it exists
1064             for extension in ('.asc', '.sig'):
1065                 sigfile_path = current_version_path + extension
1066                 if os.path.exists(sigfile_path):
1067                     siglinkname = apklinkname + extension
1068                     if os.path.islink(siglinkname):
1069                         os.remove(siglinkname)
1070                     os.symlink(sigfile_path, siglinkname)
1071
1072     if options.pretty:
1073         output = doc.toprettyxml(encoding='utf-8')
1074     else:
1075         output = doc.toxml(encoding='utf-8')
1076
1077     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
1078         f.write(output)
1079
1080     if 'repo_keyalias' in config:
1081
1082         if options.nosign:
1083             logging.info("Creating unsigned index in preparation for signing")
1084         else:
1085             logging.info("Creating signed index with this key (SHA256):")
1086             logging.info("%s" % repo_pubkey_fingerprint)
1087
1088         # Create a jar of the index...
1089         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
1090         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
1091         if p.returncode != 0:
1092             logging.critical("Failed to create {0}".format(jar_output))
1093             sys.exit(1)
1094
1095         # Sign the index...
1096         signed = os.path.join(repodir, 'index.jar')
1097         if options.nosign:
1098             # Remove old signed index if not signing
1099             if os.path.exists(signed):
1100                 os.remove(signed)
1101         else:
1102             args = [config['jarsigner'], '-keystore', config['keystore'],
1103                     '-storepass:file', config['keystorepassfile'],
1104                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1105                     signed, config['repo_keyalias']]
1106             if config['keystore'] == 'NONE':
1107                 args += config['smartcardoptions']
1108             else:  # smardcards never use -keypass
1109                 args += ['-keypass:file', config['keypassfile']]
1110             p = FDroidPopen(args)
1111             if p.returncode != 0:
1112                 logging.critical("Failed to sign index")
1113                 sys.exit(1)
1114
1115     # Copy the repo icon into the repo directory...
1116     icon_dir = os.path.join(repodir, 'icons')
1117     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1118     shutil.copyfile(config['repo_icon'], iconfilename)
1119
1120     # Write a category list in the repo to allow quick access...
1121     catdata = ''
1122     for cat in categories:
1123         catdata += cat + '\n'
1124     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1125         f.write(catdata)
1126
1127
1128 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1129
1130     for appid, app in apps.items():
1131
1132         if app.ArchivePolicy:
1133             keepversions = int(app.ArchivePolicy[:-9])
1134         else:
1135             keepversions = defaultkeepversions
1136
1137         def filter_apk_list_sorted(apk_list):
1138             res = []
1139             for apk in apk_list:
1140                 if apk['id'] == appid:
1141                     res.append(apk)
1142
1143             # Sort the apk list by version code. First is highest/newest.
1144             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1145
1146         def move_file(from_dir, to_dir, filename, ignore_missing):
1147             from_path = os.path.join(from_dir, filename)
1148             if ignore_missing and not os.path.exists(from_path):
1149                 return
1150             to_path = os.path.join(to_dir, filename)
1151             shutil.move(from_path, to_path)
1152
1153         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1154                       .format(appid, len(apks), keepversions, len(archapks)))
1155
1156         if len(apks) > keepversions:
1157             apklist = filter_apk_list_sorted(apks)
1158             # Move back the ones we don't want.
1159             for apk in apklist[keepversions:]:
1160                 logging.info("Moving " + apk['apkname'] + " to archive")
1161                 move_file(repodir, archivedir, apk['apkname'], False)
1162                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1163                 for density in all_screen_densities:
1164                     repo_icon_dir = get_icon_dir(repodir, density)
1165                     archive_icon_dir = get_icon_dir(archivedir, density)
1166                     if density not in apk['icons']:
1167                         continue
1168                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1169                 if 'srcname' in apk:
1170                     move_file(repodir, archivedir, apk['srcname'], False)
1171                 archapks.append(apk)
1172                 apks.remove(apk)
1173         elif len(apks) < keepversions and len(archapks) > 0:
1174             required = keepversions - len(apks)
1175             archapklist = filter_apk_list_sorted(archapks)
1176             # Move forward the ones we want again.
1177             for apk in archapklist[:required]:
1178                 logging.info("Moving " + apk['apkname'] + " from archive")
1179                 move_file(archivedir, repodir, apk['apkname'], False)
1180                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1181                 for density in all_screen_densities:
1182                     repo_icon_dir = get_icon_dir(repodir, density)
1183                     archive_icon_dir = get_icon_dir(archivedir, density)
1184                     if density not in apk['icons']:
1185                         continue
1186                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1187                 if 'srcname' in apk:
1188                     move_file(archivedir, repodir, apk['srcname'], False)
1189                 archapks.remove(apk)
1190                 apks.append(apk)
1191
1192
1193 def add_apks_to_per_app_repos(repodir, apks):
1194     apks_per_app = dict()
1195     for apk in apks:
1196         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1197         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1198         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1199         apks_per_app[apk['id']] = apk
1200
1201         if not os.path.exists(apk['per_app_icons']):
1202             logging.info('Adding new repo for only ' + apk['id'])
1203             os.makedirs(apk['per_app_icons'])
1204
1205         apkpath = os.path.join(repodir, apk['apkname'])
1206         shutil.copy(apkpath, apk['per_app_repo'])
1207         apksigpath = apkpath + '.sig'
1208         if os.path.exists(apksigpath):
1209             shutil.copy(apksigpath, apk['per_app_repo'])
1210         apkascpath = apkpath + '.asc'
1211         if os.path.exists(apkascpath):
1212             shutil.copy(apkascpath, apk['per_app_repo'])
1213
1214
1215 config = None
1216 options = None
1217
1218
1219 def main():
1220
1221     global config, options
1222
1223     # Parse command line...
1224     parser = ArgumentParser()
1225     common.setup_global_opts(parser)
1226     parser.add_argument("--create-key", action="store_true", default=False,
1227                         help="Create a repo signing key in a keystore")
1228     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1229                         help="Create skeleton metadata files that are missing")
1230     parser.add_argument("--delete-unknown", action="store_true", default=False,
1231                         help="Delete APKs and/or OBBs without metadata from the repo")
1232     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1233                         help="Report on build data status")
1234     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1235                         help="Interactively ask about things that need updating.")
1236     parser.add_argument("-I", "--icons", action="store_true", default=False,
1237                         help="Resize all the icons exceeding the max pixel size and exit")
1238     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1239                         help="Specify editor to use in interactive mode. Default " +
1240                         "is /etc/alternatives/editor")
1241     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1242                         help="Update the wiki")
1243     parser.add_argument("--pretty", action="store_true", default=False,
1244                         help="Produce human-readable index.xml")
1245     parser.add_argument("--clean", action="store_true", default=False,
1246                         help="Clean update - don't uses caches, reprocess all apks")
1247     parser.add_argument("--nosign", action="store_true", default=False,
1248                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1249     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1250                         help="Use date from apk instead of current time for newly added apks")
1251     options = parser.parse_args()
1252
1253     config = common.read_config(options)
1254
1255     if not ('jarsigner' in config and 'keytool' in config):
1256         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1257         sys.exit(1)
1258
1259     repodirs = ['repo']
1260     if config['archive_older'] != 0:
1261         repodirs.append('archive')
1262         if not os.path.exists('archive'):
1263             os.mkdir('archive')
1264
1265     if options.icons:
1266         resize_all_icons(repodirs)
1267         sys.exit(0)
1268
1269     # check that icons exist now, rather than fail at the end of `fdroid update`
1270     for k in ['repo_icon', 'archive_icon']:
1271         if k in config:
1272             if not os.path.exists(config[k]):
1273                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1274                 sys.exit(1)
1275
1276     # if the user asks to create a keystore, do it now, reusing whatever it can
1277     if options.create_key:
1278         if os.path.exists(config['keystore']):
1279             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1280             logging.critical("\t'" + config['keystore'] + "'")
1281             sys.exit(1)
1282
1283         if 'repo_keyalias' not in config:
1284             config['repo_keyalias'] = socket.getfqdn()
1285             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1286         if 'keydname' not in config:
1287             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1288             common.write_to_config(config, 'keydname', config['keydname'])
1289         if 'keystore' not in config:
1290             config['keystore'] = common.default_config.keystore
1291             common.write_to_config(config, 'keystore', config['keystore'])
1292
1293         password = common.genpassword()
1294         if 'keystorepass' not in config:
1295             config['keystorepass'] = password
1296             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1297         if 'keypass' not in config:
1298             config['keypass'] = password
1299             common.write_to_config(config, 'keypass', config['keypass'])
1300         common.genkeystore(config)
1301
1302     # Get all apps...
1303     apps = metadata.read_metadata()
1304
1305     # Generate a list of categories...
1306     categories = set()
1307     for app in apps.values():
1308         categories.update(app.Categories)
1309
1310     # Read known apks data (will be updated and written back when we've finished)
1311     knownapks = common.KnownApks()
1312
1313     # Gather information about all the apk files in the repo directory, using
1314     # cached data if possible.
1315     apkcachefile = os.path.join('tmp', 'apkcache')
1316     if not options.clean and os.path.exists(apkcachefile):
1317         with open(apkcachefile, 'rb') as cf:
1318             apkcache = pickle.load(cf, encoding='utf-8')
1319         if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
1320             apkcache = {}
1321     else:
1322         apkcache = {}
1323
1324     delete_disabled_builds(apps, apkcache, repodirs)
1325
1326     # Scan all apks in the main repo
1327     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1328
1329     # Generate warnings for apk's with no metadata (or create skeleton
1330     # metadata files, if requested on the command line)
1331     newmetadata = False
1332     for apk in apks:
1333         if apk['id'] not in apps:
1334             if options.create_metadata:
1335                 if 'name' not in apk:
1336                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1337                     continue
1338                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
1339                 f.write("License:Unknown\n")
1340                 f.write("Web Site:\n")
1341                 f.write("Source Code:\n")
1342                 f.write("Issue Tracker:\n")
1343                 f.write("Changelog:\n")
1344                 f.write("Summary:" + apk['name'] + "\n")
1345                 f.write("Description:\n")
1346                 f.write(apk['name'] + "\n")
1347                 f.write(".\n")
1348                 f.close()
1349                 logging.info("Generated skeleton metadata for " + apk['id'])
1350                 newmetadata = True
1351             else:
1352                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1353                 if options.delete_unknown:
1354                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1355                     rmf = os.path.join(repodirs[0], apk['apkname'])
1356                     if not os.path.exists(rmf):
1357                         logging.error("Could not find {0} to remove it".format(rmf))
1358                     else:
1359                         os.remove(rmf)
1360                 else:
1361                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1362
1363     # update the metadata with the newly created ones included
1364     if newmetadata:
1365         apps = metadata.read_metadata()
1366
1367     insert_obbs(repodirs[0], apps, apks)
1368
1369     # Scan the archive repo for apks as well
1370     if len(repodirs) > 1:
1371         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1372         if cc:
1373             cachechanged = True
1374     else:
1375         archapks = []
1376
1377     # Some information from the apks needs to be applied up to the application
1378     # level. When doing this, we use the info from the most recent version's apk.
1379     # We deal with figuring out when the app was added and last updated at the
1380     # same time.
1381     for appid, app in apps.items():
1382         bestver = 0
1383         for apk in apks + archapks:
1384             if apk['id'] == appid:
1385                 if apk['versioncode'] > bestver:
1386                     bestver = apk['versioncode']
1387                     bestapk = apk
1388
1389                 if 'added' in apk:
1390                     if not app.added or apk['added'] < app.added:
1391                         app.added = apk['added']
1392                     if not app.lastupdated or apk['added'] > app.lastupdated:
1393                         app.lastupdated = apk['added']
1394
1395         if not app.added:
1396             logging.debug("Don't know when " + appid + " was added")
1397         if not app.lastupdated:
1398             logging.debug("Don't know when " + appid + " was last updated")
1399
1400         if bestver == 0:
1401             if app.Name is None:
1402                 app.Name = app.AutoName or appid
1403             app.icon = None
1404             logging.debug("Application " + appid + " has no packages")
1405         else:
1406             if app.Name is None:
1407                 app.Name = bestapk['name']
1408             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1409             if app.CurrentVersionCode is None:
1410                 app.CurrentVersionCode = str(bestver)
1411
1412     # Sort the app list by name, then the web site doesn't have to by default.
1413     # (we had to wait until we'd scanned the apks to do this, because mostly the
1414     # name comes from there!)
1415     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1416
1417     # APKs are placed into multiple repos based on the app package, providing
1418     # per-app subscription feeds for nightly builds and things like it
1419     if config['per_app_repos']:
1420         add_apks_to_per_app_repos(repodirs[0], apks)
1421         for appid, app in apps.items():
1422             repodir = os.path.join(appid, 'fdroid', 'repo')
1423             appdict = dict()
1424             appdict[appid] = app
1425             if os.path.isdir(repodir):
1426                 make_index(appdict, [appid], apks, repodir, False, categories)
1427             else:
1428                 logging.info('Skipping index generation for ' + appid)
1429         return
1430
1431     if len(repodirs) > 1:
1432         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1433
1434     # Make the index for the main repo...
1435     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1436
1437     # If there's an archive repo,  make the index for it. We already scanned it
1438     # earlier on.
1439     if len(repodirs) > 1:
1440         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1441
1442     if config['update_stats']:
1443
1444         # Update known apks info...
1445         knownapks.writeifchanged()
1446
1447         # Generate latest apps data for widget
1448         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1449             data = ''
1450             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1451                 for line in f:
1452                     appid = line.rstrip()
1453                     data += appid + "\t"
1454                     app = apps[appid]
1455                     data += app.Name + "\t"
1456                     if app.icon is not None:
1457                         data += app.icon + "\t"
1458                     data += app.License + "\n"
1459             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1460                 f.write(data)
1461
1462     if cachechanged:
1463         apkcache["METADATA_VERSION"] = METADATA_VERSION
1464         with open(apkcachefile, 'wb') as cf:
1465             pickle.dump(apkcache, cf)
1466
1467     # Update the wiki...
1468     if options.wiki:
1469         update_wiki(apps, sortedids, apks + archapks)
1470
1471     logging.info("Finished.")
1472
1473 if __name__ == "__main__":
1474     main()