fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   5 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU Affero General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Affero General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Affero General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 import sys
  21 import os
  22 import shutil
  23 import glob
  24 import re
  25 import socket
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 import urllib.parse
  30 from datetime import datetime, timedelta
  31 from xml.dom.minidom import Document
  32 from argparse import ArgumentParser
  33 import time
  34 from pyasn1.error import PyAsn1Error
  35 from pyasn1.codec.der import decoder, encoder
  36 from pyasn1_modules import rfc2315
  37 from binascii import hexlify, unhexlify
  38
  39 from PIL import Image
  40 import logging
  41
  42 from . import common
  43 from . import metadata
  44 from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
  45 from .metadata import MetaDataException
  46
  47 METADATA_VERSION = 16
  48
  49 screen_densities = ['640', '480', '320', '240', '160', '120']
  50
  51 all_screen_densities = ['0'] + screen_densities
  52
  53
  54 def dpi_to_px(density):
  55     return (int(density) * 48) / 160
  56
  57
  58 def px_to_dpi(px):
  59     return (int(px) * 160) / 48
  60
  61
  62 def get_icon_dir(repodir, density):
  63     if density == '0':
  64         return os.path.join(repodir, "icons")
  65     return os.path.join(repodir, "icons-%s" % density)
  66
  67
  68 def get_icon_dirs(repodir):
  69     for density in screen_densities:
  70         yield get_icon_dir(repodir, density)
  71
  72
  73 def get_all_icon_dirs(repodir):
  74     for density in all_screen_densities:
  75         yield get_icon_dir(repodir, density)
  76
  77
  78 def update_wiki(apps, sortedids, apks):
  79     """Update the wiki
  80
  81     :param apps: fully populated list of all applications
  82     :param apks: all apks, except...
  83     """
  84     logging.info("Updating wiki")
  85     wikicat = 'Apps'
  86     wikiredircat = 'App Redirects'
  87     import mwclient
  88     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  89                          path=config['wiki_path'])
  90     site.login(config['wiki_user'], config['wiki_password'])
  91     generated_pages = {}
  92     generated_redirects = {}
  93
  94     for appid in sortedids:
  95         app = apps[appid]
  96
  97         wikidata = ''
  98         if app.Disabled:
  99             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 100         if app.AntiFeatures:
 101             for af in app.AntiFeatures:
 102                 wikidata += '{{AntiFeature|' + af + '}}\n'
 103         if app.RequiresRoot:
 104             requiresroot = 'Yes'
 105         else:
 106             requiresroot = 'No'
 107         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 108             appid,
 109             app.Name,
 110             time.strftime('%Y-%m-%d', app.added) if app.added else '',
 111             time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
 112             app.SourceCode,
 113             app.IssueTracker,
 114             app.WebSite,
 115             app.Changelog,
 116             app.Donate,
 117             app.FlattrID,
 118             app.Bitcoin,
 119             app.Litecoin,
 120             app.License,
 121             requiresroot,
 122             app.AuthorName,
 123             app.AuthorEmail)
 124
 125         if app.Provides:
 126             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 127
 128         wikidata += app.Summary
 129         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 130
 131         wikidata += "=Description=\n"
 132         wikidata += metadata.description_wiki(app.Description) + "\n"
 133
 134         wikidata += "=Maintainer Notes=\n"
 135         if app.MaintainerNotes:
 136             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 137         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 138
 139         # Get a list of all packages for this application...
 140         apklist = []
 141         gotcurrentver = False
 142         cantupdate = False
 143         buildfails = False
 144         for apk in apks:
 145             if apk['id'] == appid:
 146                 if str(apk['versioncode']) == app.CurrentVersionCode:
 147                     gotcurrentver = True
 148                 apklist.append(apk)
 149         # Include ones we can't build, as a special case...
 150         for build in app.builds:
 151             if build.disable:
 152                 if build.vercode == app.CurrentVersionCode:
 153                     cantupdate = True
 154                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 155                 apklist.append({'versioncode': int(build.vercode),
 156                                 'version': build.version,
 157                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 158                                 })
 159             else:
 160                 builtit = False
 161                 for apk in apklist:
 162                     if apk['versioncode'] == int(build.vercode):
 163                         builtit = True
 164                         break
 165                 if not builtit:
 166                     buildfails = True
 167                     apklist.append({'versioncode': int(build.vercode),
 168                                     'version': build.version,
 169                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
 170                                     })
 171         if app.CurrentVersionCode == '0':
 172             cantupdate = True
 173         # Sort with most recent first...
 174         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 175
 176         wikidata += "=Versions=\n"
 177         if len(apklist) == 0:
 178             wikidata += "We currently have no versions of this app available."
 179         elif not gotcurrentver:
 180             wikidata += "We don't have the current version of this app."
 181         else:
 182             wikidata += "We have the current version of this app."
 183         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 184         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 185         if len(app.NoSourceSince) > 0:
 186             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 187         if len(app.CurrentVersion) > 0:
 188             wikidata += "The current (recommended) version is " + app.CurrentVersion
 189             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 190         validapks = 0
 191         for apk in apklist:
 192             wikidata += "==" + apk['version'] + "==\n"
 193
 194             if 'buildproblem' in apk:
 195                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 196             else:
 197                 validapks += 1
 198                 wikidata += "This version is built and signed by "
 199                 if 'srcname' in apk:
 200                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 201                 else:
 202                     wikidata += "the original developer.\n\n"
 203             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 204
 205         wikidata += '\n[[Category:' + wikicat + ']]\n'
 206         if len(app.NoSourceSince) > 0:
 207             wikidata += '\n[[Category:Apps missing source code]]\n'
 208         if validapks == 0 and not app.Disabled:
 209             wikidata += '\n[[Category:Apps with no packages]]\n'
 210         if cantupdate and not app.Disabled:
 211             wikidata += "\n[[Category:Apps we cannot update]]\n"
 212         if buildfails and not app.Disabled:
 213             wikidata += "\n[[Category:Apps with failing builds]]\n"
 214         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 215             wikidata += '\n[[Category:Apps to Update]]\n'
 216         if app.Disabled:
 217             wikidata += '\n[[Category:Apps that are disabled]]\n'
 218         if app.UpdateCheckMode == 'None' and not app.Disabled:
 219             wikidata += '\n[[Category:Apps with no update check]]\n'
 220         for appcat in app.Categories:
 221             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 222
 223         # We can't have underscores in the page name, even if they're in
 224         # the package ID, because MediaWiki messes with them...
 225         pagename = appid.replace('_', ' ')
 226
 227         # Drop a trailing newline, because mediawiki is going to drop it anyway
 228         # and it we don't we'll think the page has changed when it hasn't...
 229         if wikidata.endswith('\n'):
 230             wikidata = wikidata[:-1]
 231
 232         generated_pages[pagename] = wikidata
 233
 234         # Make a redirect from the name to the ID too, unless there's
 235         # already an existing page with the name and it isn't a redirect.
 236         noclobber = False
 237         apppagename = app.Name.replace('_', ' ')
 238         apppagename = apppagename.replace('{', '')
 239         apppagename = apppagename.replace('}', ' ')
 240         apppagename = apppagename.replace(':', ' ')
 241         # Drop double spaces caused mostly by replacing ':' above
 242         apppagename = apppagename.replace('  ', ' ')
 243         for expagename in site.allpages(prefix=apppagename,
 244                                         filterredir='nonredirects',
 245                                         generator=False):
 246             if expagename == apppagename:
 247                 noclobber = True
 248         # Another reason not to make the redirect page is if the app name
 249         # is the same as it's ID, because that will overwrite the real page
 250         # with an redirect to itself! (Although it seems like an odd
 251         # scenario this happens a lot, e.g. where there is metadata but no
 252         # builds or binaries to extract a name from.
 253         if apppagename == pagename:
 254             noclobber = True
 255         if not noclobber:
 256             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 257
 258     for tcat, genp in [(wikicat, generated_pages),
 259                        (wikiredircat, generated_redirects)]:
 260         catpages = site.Pages['Category:' + tcat]
 261         existingpages = []
 262         for page in catpages:
 263             existingpages.append(page.name)
 264             if page.name in genp:
 265                 pagetxt = page.edit()
 266                 if pagetxt != genp[page.name]:
 267                     logging.debug("Updating modified page " + page.name)
 268                     page.save(genp[page.name], summary='Auto-updated')
 269                 else:
 270                     logging.debug("Page " + page.name + " is unchanged")
 271             else:
 272                 logging.warn("Deleting page " + page.name)
 273                 page.delete('No longer published')
 274         for pagename, text in genp.items():
 275             logging.debug("Checking " + pagename)
 276             if pagename not in existingpages:
 277                 logging.debug("Creating page " + pagename)
 278                 try:
 279                     newpage = site.Pages[pagename]
 280                     newpage.save(text, summary='Auto-created')
 281                 except:
 282                     logging.error("...FAILED to create page '{0}'".format(pagename))
 283
 284     # Purge server cache to ensure counts are up to date
 285     site.pages['Repository Maintenance'].purge()
 286
 287
 288 def delete_disabled_builds(apps, apkcache, repodirs):
 289     """Delete disabled build outputs.
 290
 291     :param apps: list of all applications, as per metadata.read_metadata
 292     :param apkcache: current apk cache information
 293     :param repodirs: the repo directories to process
 294     """
 295     for appid, app in apps.items():
 296         for build in app.builds:
 297             if not build.disable:
 298                 continue
 299             apkfilename = appid + '_' + str(build.vercode) + '.apk'
 300             iconfilename = "%s.%s.png" % (
 301                 appid,
 302                 build.vercode)
 303             for repodir in repodirs:
 304                 files = [
 305                     os.path.join(repodir, apkfilename),
 306                     os.path.join(repodir, apkfilename + '.asc'),
 307                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 308                 ]
 309                 for density in all_screen_densities:
 310                     repo_dir = get_icon_dir(repodir, density)
 311                     files.append(os.path.join(repo_dir, iconfilename))
 312
 313                 for f in files:
 314                     if os.path.exists(f):
 315                         logging.info("Deleting disabled build output " + f)
 316                         os.remove(f)
 317             if apkfilename in apkcache:
 318                 del apkcache[apkfilename]
 319
 320
 321 def resize_icon(iconpath, density):
 322
 323     if not os.path.isfile(iconpath):
 324         return
 325
 326     fp = None
 327     try:
 328         fp = open(iconpath, 'rb')
 329         im = Image.open(fp)
 330         size = dpi_to_px(density)
 331
 332         if any(length > size for length in im.size):
 333             oldsize = im.size
 334             im.thumbnail((size, size), Image.ANTIALIAS)
 335             logging.debug("%s was too large at %s - new size is %s" % (
 336                 iconpath, oldsize, im.size))
 337             im.save(iconpath, "PNG")
 338
 339     except Exception as e:
 340         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 341
 342     finally:
 343         if fp:
 344             fp.close()
 345
 346
 347 def resize_all_icons(repodirs):
 348     """Resize all icons that exceed the max size
 349
 350     :param repodirs: the repo directories to process
 351     """
 352     for repodir in repodirs:
 353         for density in screen_densities:
 354             icon_dir = get_icon_dir(repodir, density)
 355             icon_glob = os.path.join(icon_dir, '*.png')
 356             for iconpath in glob.glob(icon_glob):
 357                 resize_icon(iconpath, density)
 358
 359
 360 # A signature block file with a .DSA, .RSA, or .EC extension
 361 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 362
 363
 364 def getsig(apkpath):
 365     """ Get the signing certificate of an apk. To get the same md5 has that
 366     Android gets, we encode the .RSA certificate in a specific format and pass
 367     it hex-encoded to the md5 digest algorithm.
 368
 369     :param apkpath: path to the apk
 370     :returns: A string containing the md5 of the signature of the apk or None
 371               if an error occurred.
 372     """
 373
 374     cert = None
 375
 376     # verify the jar signature is correct
 377     args = [config['jarsigner'], '-verify', apkpath]
 378     p = FDroidPopen(args)
 379     if p.returncode != 0:
 380         logging.critical(apkpath + " has a bad signature!")
 381         return None
 382
 383     with zipfile.ZipFile(apkpath, 'r') as apk:
 384
 385         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 386
 387         if len(certs) < 1:
 388             logging.error("Found no signing certificates on %s" % apkpath)
 389             return None
 390         if len(certs) > 1:
 391             logging.error("Found multiple signing certificates on %s" % apkpath)
 392             return None
 393
 394         cert = apk.read(certs[0])
 395
 396     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 397     if content.getComponentByName('contentType') != rfc2315.signedData:
 398         logging.error("Unexpected format.")
 399         return None
 400
 401     content = decoder.decode(content.getComponentByName('content'),
 402                              asn1Spec=rfc2315.SignedData())[0]
 403     try:
 404         certificates = content.getComponentByName('certificates')
 405     except PyAsn1Error:
 406         logging.error("Certificates not found.")
 407         return None
 408
 409     cert_encoded = encoder.encode(certificates)[4:]
 410
 411     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 412
 413
 414 def get_icon_bytes(apkzip, iconsrc):
 415     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 416     try:
 417         return apkzip.read(iconsrc)
 418     except KeyError:
 419         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 420
 421
 422 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
 423     """Scan the apks in the given repo directory.
 424
 425     This also extracts the icons.
 426
 427     :param apps: list of all applications, as per metadata.read_metadata
 428     :param apkcache: current apk cache information
 429     :param repodir: repo directory to scan
 430     :param knownapks: known apks info
 431     :param use_date_from_apk: use date from APK (instead of current date)
 432                               for newly added APKs
 433     :returns: (apks, cachechanged) where apks is a list of apk information,
 434               and cachechanged is True if the apkcache got changed.
 435     """
 436
 437     cachechanged = False
 438
 439     for icon_dir in get_all_icon_dirs(repodir):
 440         if os.path.exists(icon_dir):
 441             if options.clean:
 442                 shutil.rmtree(icon_dir)
 443                 os.makedirs(icon_dir)
 444         else:
 445             os.makedirs(icon_dir)
 446
 447     apks = []
 448     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 449     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 450     vername_pat = re.compile(".*versionName='([^']*)'.*")
 451     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 452     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 453     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 454     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 455     string_pat = re.compile(".* name='([^']*)'.*")
 456     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 457
 458         apkfilename = apkfile[len(repodir) + 1:]
 459         if ' ' in apkfilename:
 460             logging.critical("Spaces in filenames are not allowed.")
 461             sys.exit(1)
 462
 463         # Calculate the sha256...
 464         sha = hashlib.sha256()
 465         with open(apkfile, 'rb') as f:
 466             while True:
 467                 t = f.read(16384)
 468                 if len(t) == 0:
 469                     break
 470                 sha.update(t)
 471             shasum = sha.hexdigest()
 472
 473         usecache = False
 474         if apkfilename in apkcache:
 475             apk = apkcache[apkfilename]
 476             if apk['sha256'] == shasum:
 477                 logging.debug("Reading " + apkfilename + " from cache")
 478                 usecache = True
 479             else:
 480                 logging.debug("Ignoring stale cache data for " + apkfilename)
 481
 482         if not usecache:
 483             logging.debug("Processing " + apkfilename)
 484             apk = {}
 485             apk['apkname'] = apkfilename
 486             apk['sha256'] = shasum
 487             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 488             if os.path.exists(os.path.join(repodir, srcfilename)):
 489                 apk['srcname'] = srcfilename
 490             apk['size'] = os.path.getsize(apkfile)
 491             apk['permissions'] = set()
 492             apk['features'] = set()
 493             apk['icons_src'] = {}
 494             apk['icons'] = {}
 495             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 496             if p.returncode != 0:
 497                 if options.delete_unknown:
 498                     if os.path.exists(apkfile):
 499                         logging.error("Failed to get apk information, deleting " + apkfile)
 500                         os.remove(apkfile)
 501                     else:
 502                         logging.error("Could not find {0} to remove it".format(apkfile))
 503                 else:
 504                     logging.error("Failed to get apk information, skipping " + apkfile)
 505                 continue
 506             for line in p.output.splitlines():
 507                 if line.startswith("package:"):
 508                     try:
 509                         apk['id'] = re.match(name_pat, line).group(1)
 510                         apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
 511                         apk['version'] = re.match(vername_pat, line).group(1)
 512                     except Exception as e:
 513                         logging.error("Package matching failed: " + str(e))
 514                         logging.info("Line was: " + line)
 515                         sys.exit(1)
 516                 elif line.startswith("application:"):
 517                     apk['name'] = re.match(label_pat, line).group(1)
 518                     # Keep path to non-dpi icon in case we need it
 519                     match = re.match(icon_pat_nodpi, line)
 520                     if match:
 521                         apk['icons_src']['-1'] = match.group(1)
 522                 elif line.startswith("launchable-activity:"):
 523                     # Only use launchable-activity as fallback to application
 524                     if not apk['name']:
 525                         apk['name'] = re.match(label_pat, line).group(1)
 526                     if '-1' not in apk['icons_src']:
 527                         match = re.match(icon_pat_nodpi, line)
 528                         if match:
 529                             apk['icons_src']['-1'] = match.group(1)
 530                 elif line.startswith("application-icon-"):
 531                     match = re.match(icon_pat, line)
 532                     if match:
 533                         density = match.group(1)
 534                         path = match.group(2)
 535                         apk['icons_src'][density] = path
 536                 elif line.startswith("sdkVersion:"):
 537                     m = re.match(sdkversion_pat, line)
 538                     if m is None:
 539                         logging.error(line.replace('sdkVersion:', '')
 540                                       + ' is not a valid minSdkVersion!')
 541                     else:
 542                         apk['minSdkVersion'] = m.group(1)
 543                         # if target not set, default to min
 544                         if 'targetSdkVersion' not in apk:
 545                             apk['targetSdkVersion'] = m.group(1)
 546                 elif line.startswith("targetSdkVersion:"):
 547                     m = re.match(sdkversion_pat, line)
 548                     if m is None:
 549                         logging.error(line.replace('targetSdkVersion:', '')
 550                                       + ' is not a valid targetSdkVersion!')
 551                     else:
 552                         apk['targetSdkVersion'] = m.group(1)
 553                 elif line.startswith("maxSdkVersion:"):
 554                     apk['maxSdkVersion'] = re.match(sdkversion_pat, line).group(1)
 555                 elif line.startswith("native-code:"):
 556                     apk['nativecode'] = []
 557                     for arch in line[13:].split(' '):
 558                         apk['nativecode'].append(arch[1:-1])
 559                 elif line.startswith("uses-permission:"):
 560                     perm = re.match(string_pat, line).group(1)
 561                     if perm.startswith("android.permission."):
 562                         perm = perm[19:]
 563                     apk['permissions'].add(perm)
 564                 elif line.startswith("uses-feature:"):
 565                     perm = re.match(string_pat, line).group(1)
 566                     # Filter out this, it's only added with the latest SDK tools and
 567                     # causes problems for lots of apps.
 568                     if perm != "android.hardware.screen.portrait" \
 569                             and perm != "android.hardware.screen.landscape":
 570                         if perm.startswith("android.feature."):
 571                             perm = perm[16:]
 572                         apk['features'].add(perm)
 573
 574             if 'minSdkVersion' not in apk:
 575                 logging.warn("No SDK version information found in {0}".format(apkfile))
 576                 apk['minSdkVersion'] = 1
 577
 578             # Check for debuggable apks...
 579             if common.isApkDebuggable(apkfile, config):
 580                 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
 581
 582             # Get the signature (or md5 of, to be precise)...
 583             logging.debug('Getting signature of {0}'.format(apkfile))
 584             apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 585             if not apk['sig']:
 586                 logging.critical("Failed to get apk signature")
 587                 sys.exit(1)
 588
 589             apkzip = zipfile.ZipFile(apkfile, 'r')
 590
 591             # if an APK has files newer than the system time, suggest updating
 592             # the system clock.  This is useful for offline systems, used for
 593             # signing, which do not have another source of clock sync info. It
 594             # has to be more than 24 hours newer because ZIP/APK files do not
 595             # store timezone info
 596             manifest = apkzip.getinfo('AndroidManifest.xml')
 597             if manifest.date_time[1] == 0:  # month can't be zero
 598                 logging.debug('AndroidManifest.xml has no date')
 599             else:
 600                 dt_obj = datetime(*manifest.date_time)
 601                 checkdt = dt_obj - timedelta(1)
 602                 if datetime.today() < checkdt:
 603                     logging.warn('System clock is older than manifest in: '
 604                                  + apkfilename
 605                                  + '\nSet clock to that time using:\n'
 606                                  + 'sudo date -s "' + str(dt_obj) + '"')
 607
 608             iconfilename = "%s.%s.png" % (
 609                 apk['id'],
 610                 apk['versioncode'])
 611
 612             # Extract the icon file...
 613             empty_densities = []
 614             for density in screen_densities:
 615                 if density not in apk['icons_src']:
 616                     empty_densities.append(density)
 617                     continue
 618                 iconsrc = apk['icons_src'][density]
 619                 icon_dir = get_icon_dir(repodir, density)
 620                 icondest = os.path.join(icon_dir, iconfilename)
 621
 622                 try:
 623                     with open(icondest, 'wb') as f:
 624                         f.write(get_icon_bytes(apkzip, iconsrc))
 625                     apk['icons'][density] = iconfilename
 626
 627                 except:
 628                     logging.warn("Error retrieving icon file")
 629                     del apk['icons'][density]
 630                     del apk['icons_src'][density]
 631                     empty_densities.append(density)
 632
 633             if '-1' in apk['icons_src']:
 634                 iconsrc = apk['icons_src']['-1']
 635                 iconpath = os.path.join(
 636                     get_icon_dir(repodir, '0'), iconfilename)
 637                 with open(iconpath, 'wb') as f:
 638                     f.write(get_icon_bytes(apkzip, iconsrc))
 639                 try:
 640                     im = Image.open(iconpath)
 641                     dpi = px_to_dpi(im.size[0])
 642                     for density in screen_densities:
 643                         if density in apk['icons']:
 644                             break
 645                         if density == screen_densities[-1] or dpi >= int(density):
 646                             apk['icons'][density] = iconfilename
 647                             shutil.move(iconpath,
 648                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 649                             empty_densities.remove(density)
 650                             break
 651                 except Exception as e:
 652                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 653
 654             if apk['icons']:
 655                 apk['icon'] = iconfilename
 656
 657             apkzip.close()
 658
 659             # First try resizing down to not lose quality
 660             last_density = None
 661             for density in screen_densities:
 662                 if density not in empty_densities:
 663                     last_density = density
 664                     continue
 665                 if last_density is None:
 666                     continue
 667                 logging.debug("Density %s not available, resizing down from %s"
 668                               % (density, last_density))
 669
 670                 last_iconpath = os.path.join(
 671                     get_icon_dir(repodir, last_density), iconfilename)
 672                 iconpath = os.path.join(
 673                     get_icon_dir(repodir, density), iconfilename)
 674                 fp = None
 675                 try:
 676                     fp = open(last_iconpath, 'rb')
 677                     im = Image.open(fp)
 678
 679                     size = dpi_to_px(density)
 680
 681                     im.thumbnail((size, size), Image.ANTIALIAS)
 682                     im.save(iconpath, "PNG")
 683                     empty_densities.remove(density)
 684                 except:
 685                     logging.warning("Invalid image file at %s" % last_iconpath)
 686                 finally:
 687                     if fp:
 688                         fp.close()
 689
 690             # Then just copy from the highest resolution available
 691             last_density = None
 692             for density in reversed(screen_densities):
 693                 if density not in empty_densities:
 694                     last_density = density
 695                     continue
 696                 if last_density is None:
 697                     continue
 698                 logging.debug("Density %s not available, copying from lower density %s"
 699                               % (density, last_density))
 700
 701                 shutil.copyfile(
 702                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 703                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 704
 705                 empty_densities.remove(density)
 706
 707             for density in screen_densities:
 708                 icon_dir = get_icon_dir(repodir, density)
 709                 icondest = os.path.join(icon_dir, iconfilename)
 710                 resize_icon(icondest, density)
 711
 712             # Copy from icons-mdpi to icons since mdpi is the baseline density
 713             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 714             if os.path.isfile(baseline):
 715                 apk['icons']['0'] = iconfilename
 716                 shutil.copyfile(baseline,
 717                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 718
 719             # Record in known apks, getting the added date at the same time..
 720             added = knownapks.recordapk(apk['apkname'], apk['id'])
 721             if added:
 722                 if use_date_from_apk and manifest.date_time[1] != 0:
 723                     added = datetime(*manifest.date_time).timetuple()
 724                     logging.debug("Using date from APK")
 725
 726                 apk['added'] = added
 727
 728             apkcache[apkfilename] = apk
 729             cachechanged = True
 730
 731         apks.append(apk)
 732
 733     return apks, cachechanged
 734
 735
 736 repo_pubkey_fingerprint = None
 737
 738
 739 # Generate a certificate fingerprint the same way keytool does it
 740 # (but with slightly different formatting)
 741 def cert_fingerprint(data):
 742     digest = hashlib.sha256(data).digest()
 743     ret = []
 744     ret.append(' '.join("%02X" % b for b in bytearray(digest)))
 745     return " ".join(ret)
 746
 747
 748 def extract_pubkey():
 749     global repo_pubkey_fingerprint
 750     if 'repo_pubkey' in config:
 751         pubkey = unhexlify(config['repo_pubkey'])
 752     else:
 753         p = FDroidPopenBytes([config['keytool'], '-exportcert',
 754                               '-alias', config['repo_keyalias'],
 755                               '-keystore', config['keystore'],
 756                               '-storepass:file', config['keystorepassfile']]
 757                              + config['smartcardoptions'],
 758                              output=False, stderr_to_stdout=False)
 759         if p.returncode != 0 or len(p.output) < 20:
 760             msg = "Failed to get repo pubkey!"
 761             if config['keystore'] == 'NONE':
 762                 msg += ' Is your crypto smartcard plugged in?'
 763             logging.critical(msg)
 764             sys.exit(1)
 765         pubkey = p.output
 766     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 767     return hexlify(pubkey)
 768
 769
 770 def make_index(apps, sortedids, apks, repodir, archive, categories):
 771     """Make a repo index.
 772
 773     :param apps: fully populated apps list
 774     :param apks: full populated apks list
 775     :param repodir: the repo directory
 776     :param archive: True if this is the archive repo, False if it's the
 777                     main one.
 778     :param categories: list of categories
 779     """
 780
 781     doc = Document()
 782
 783     def addElement(name, value, doc, parent):
 784         el = doc.createElement(name)
 785         el.appendChild(doc.createTextNode(value))
 786         parent.appendChild(el)
 787
 788     def addElementNonEmpty(name, value, doc, parent):
 789         if not value:
 790             return
 791         addElement(name, value, doc, parent)
 792
 793     def addElementCDATA(name, value, doc, parent):
 794         el = doc.createElement(name)
 795         el.appendChild(doc.createCDATASection(value))
 796         parent.appendChild(el)
 797
 798     root = doc.createElement("fdroid")
 799     doc.appendChild(root)
 800
 801     repoel = doc.createElement("repo")
 802
 803     mirrorcheckfailed = False
 804     for mirror in config.get('mirrors', []):
 805         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
 806         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
 807             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 808             mirrorcheckfailed = True
 809     if mirrorcheckfailed:
 810         sys.exit(1)
 811
 812     if archive:
 813         repoel.setAttribute("name", config['archive_name'])
 814         if config['repo_maxage'] != 0:
 815             repoel.setAttribute("maxage", str(config['repo_maxage']))
 816         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 817         repoel.setAttribute("url", config['archive_url'])
 818         addElement('description', config['archive_description'], doc, repoel)
 819         urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
 820         for mirror in config.get('mirrors', []):
 821             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 822
 823     else:
 824         repoel.setAttribute("name", config['repo_name'])
 825         if config['repo_maxage'] != 0:
 826             repoel.setAttribute("maxage", str(config['repo_maxage']))
 827         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 828         repoel.setAttribute("url", config['repo_url'])
 829         addElement('description', config['repo_description'], doc, repoel)
 830         urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
 831         for mirror in config.get('mirrors', []):
 832             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 833
 834     repoel.setAttribute("version", str(METADATA_VERSION))
 835     repoel.setAttribute("timestamp", str(int(time.time())))
 836
 837     nosigningkey = False
 838     if not options.nosign:
 839         if 'repo_keyalias' not in config:
 840             nosigningkey = True
 841             logging.critical("'repo_keyalias' not found in config.py!")
 842         if 'keystore' not in config:
 843             nosigningkey = True
 844             logging.critical("'keystore' not found in config.py!")
 845         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 846             nosigningkey = True
 847             logging.critical("'keystorepass' not found in config.py!")
 848         if 'keypass' not in config and 'keypassfile' not in config:
 849             nosigningkey = True
 850             logging.critical("'keypass' not found in config.py!")
 851         if not os.path.exists(config['keystore']):
 852             nosigningkey = True
 853             logging.critical("'" + config['keystore'] + "' does not exist!")
 854         if nosigningkey:
 855             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 856             logging.warning("\tfdroid update --create-key")
 857             sys.exit(1)
 858
 859     repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
 860     root.appendChild(repoel)
 861
 862     for appid in sortedids:
 863         app = apps[appid]
 864
 865         if app.Disabled is not None:
 866             continue
 867
 868         # Get a list of the apks for this app...
 869         apklist = []
 870         for apk in apks:
 871             if apk['id'] == appid:
 872                 apklist.append(apk)
 873
 874         if len(apklist) == 0:
 875             continue
 876
 877         apel = doc.createElement("application")
 878         apel.setAttribute("id", app.id)
 879         root.appendChild(apel)
 880
 881         addElement('id', app.id, doc, apel)
 882         if app.added:
 883             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
 884         if app.lastupdated:
 885             addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
 886         addElement('name', app.Name, doc, apel)
 887         addElement('summary', app.Summary, doc, apel)
 888         if app.icon:
 889             addElement('icon', app.icon, doc, apel)
 890
 891         def linkres(appid):
 892             if appid in apps:
 893                 return ("fdroid.app:" + appid, apps[appid].Name)
 894             raise MetaDataException("Cannot resolve app id " + appid)
 895
 896         addElement('desc',
 897                    metadata.description_html(app.Description, linkres),
 898                    doc, apel)
 899         addElement('license', app.License, doc, apel)
 900         if app.Categories:
 901             addElement('categories', ','.join(app.Categories), doc, apel)
 902             # We put the first (primary) category in LAST, which will have
 903             # the desired effect of making clients that only understand one
 904             # category see that one.
 905             addElement('category', app.Categories[0], doc, apel)
 906         addElement('web', app.WebSite, doc, apel)
 907         addElement('source', app.SourceCode, doc, apel)
 908         addElement('tracker', app.IssueTracker, doc, apel)
 909         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 910         addElementNonEmpty('author', app.AuthorName, doc, apel)
 911         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 912         addElementNonEmpty('donate', app.Donate, doc, apel)
 913         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 914         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 915         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 916
 917         # These elements actually refer to the current version (i.e. which
 918         # one is recommended. They are historically mis-named, and need
 919         # changing, but stay like this for now to support existing clients.
 920         addElement('marketversion', app.CurrentVersion, doc, apel)
 921         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 922
 923         if app.AntiFeatures:
 924             af = app.AntiFeatures
 925             if af:
 926                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 927         if app.Provides:
 928             pv = app.Provides.split(',')
 929             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 930         if app.RequiresRoot:
 931             addElement('requirements', 'root', doc, apel)
 932
 933         # Sort the apk list into version order, just so the web site
 934         # doesn't have to do any work by default...
 935         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 936
 937         # Check for duplicates - they will make the client unhappy...
 938         for i in range(len(apklist) - 1):
 939             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 940                 logging.critical("duplicate versions: '%s' - '%s'" % (
 941                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 942                 sys.exit(1)
 943
 944         current_version_code = 0
 945         current_version_file = None
 946         for apk in apklist:
 947             # find the APK for the "Current Version"
 948             if current_version_code < apk['versioncode']:
 949                 current_version_code = apk['versioncode']
 950             if current_version_code < int(app.CurrentVersionCode):
 951                 current_version_file = apk['apkname']
 952
 953             apkel = doc.createElement("package")
 954             apel.appendChild(apkel)
 955             addElement('version', apk['version'], doc, apkel)
 956             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 957             addElement('apkname', apk['apkname'], doc, apkel)
 958             if 'srcname' in apk:
 959                 addElement('srcname', apk['srcname'], doc, apkel)
 960             for hash_type in ['sha256']:
 961                 if hash_type not in apk:
 962                     continue
 963                 hashel = doc.createElement("hash")
 964                 hashel.setAttribute("type", hash_type)
 965                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 966                 apkel.appendChild(hashel)
 967             addElement('sig', apk['sig'], doc, apkel)
 968             addElement('size', str(apk['size']), doc, apkel)
 969             addElement('sdkver', str(apk['minSdkVersion']), doc, apkel)
 970             if 'targetSdkVersion' in apk:
 971                 addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
 972             if 'maxSdkVersion' in apk:
 973                 addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
 974             if 'added' in apk:
 975                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 976             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
 977             if 'nativecode' in apk:
 978                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 979             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
 980
 981         if current_version_file is not None \
 982                 and config['make_current_version_link'] \
 983                 and repodir == 'repo':  # only create these
 984             namefield = config['current_version_name_source']
 985             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
 986             apklinkname = sanitized_name + '.apk'
 987             current_version_path = os.path.join(repodir, current_version_file)
 988             if os.path.islink(apklinkname):
 989                 os.remove(apklinkname)
 990             os.symlink(current_version_path, apklinkname)
 991             # also symlink gpg signature, if it exists
 992             for extension in ('.asc', '.sig'):
 993                 sigfile_path = current_version_path + extension
 994                 if os.path.exists(sigfile_path):
 995                     siglinkname = apklinkname + extension
 996                     if os.path.islink(siglinkname):
 997                         os.remove(siglinkname)
 998                     os.symlink(sigfile_path, siglinkname)
 999
1000     if options.pretty:
1001         output = doc.toprettyxml(encoding='utf-8')
1002     else:
1003         output = doc.toxml(encoding='utf-8')
1004
1005     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
1006         f.write(output)
1007
1008     if 'repo_keyalias' in config:
1009
1010         if options.nosign:
1011             logging.info("Creating unsigned index in preparation for signing")
1012         else:
1013             logging.info("Creating signed index with this key (SHA256):")
1014             logging.info("%s" % repo_pubkey_fingerprint)
1015
1016         # Create a jar of the index...
1017         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
1018         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
1019         if p.returncode != 0:
1020             logging.critical("Failed to create {0}".format(jar_output))
1021             sys.exit(1)
1022
1023         # Sign the index...
1024         signed = os.path.join(repodir, 'index.jar')
1025         if options.nosign:
1026             # Remove old signed index if not signing
1027             if os.path.exists(signed):
1028                 os.remove(signed)
1029         else:
1030             args = [config['jarsigner'], '-keystore', config['keystore'],
1031                     '-storepass:file', config['keystorepassfile'],
1032                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1033                     signed, config['repo_keyalias']]
1034             if config['keystore'] == 'NONE':
1035                 args += config['smartcardoptions']
1036             else:  # smardcards never use -keypass
1037                 args += ['-keypass:file', config['keypassfile']]
1038             p = FDroidPopen(args)
1039             if p.returncode != 0:
1040                 logging.critical("Failed to sign index")
1041                 sys.exit(1)
1042
1043     # Copy the repo icon into the repo directory...
1044     icon_dir = os.path.join(repodir, 'icons')
1045     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1046     shutil.copyfile(config['repo_icon'], iconfilename)
1047
1048     # Write a category list in the repo to allow quick access...
1049     catdata = ''
1050     for cat in categories:
1051         catdata += cat + '\n'
1052     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1053         f.write(catdata)
1054
1055
1056 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1057
1058     for appid, app in apps.items():
1059
1060         if app.ArchivePolicy:
1061             keepversions = int(app.ArchivePolicy[:-9])
1062         else:
1063             keepversions = defaultkeepversions
1064
1065         def filter_apk_list_sorted(apk_list):
1066             res = []
1067             for apk in apk_list:
1068                 if apk['id'] == appid:
1069                     res.append(apk)
1070
1071             # Sort the apk list by version code. First is highest/newest.
1072             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1073
1074         def move_file(from_dir, to_dir, filename, ignore_missing):
1075             from_path = os.path.join(from_dir, filename)
1076             if ignore_missing and not os.path.exists(from_path):
1077                 return
1078             to_path = os.path.join(to_dir, filename)
1079             shutil.move(from_path, to_path)
1080
1081         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1082                       .format(appid, len(apks), keepversions, len(archapks)))
1083
1084         if len(apks) > keepversions:
1085             apklist = filter_apk_list_sorted(apks)
1086             # Move back the ones we don't want.
1087             for apk in apklist[keepversions:]:
1088                 logging.info("Moving " + apk['apkname'] + " to archive")
1089                 move_file(repodir, archivedir, apk['apkname'], False)
1090                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1091                 for density in all_screen_densities:
1092                     repo_icon_dir = get_icon_dir(repodir, density)
1093                     archive_icon_dir = get_icon_dir(archivedir, density)
1094                     if density not in apk['icons']:
1095                         continue
1096                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1097                 if 'srcname' in apk:
1098                     move_file(repodir, archivedir, apk['srcname'], False)
1099                 archapks.append(apk)
1100                 apks.remove(apk)
1101         elif len(apks) < keepversions and len(archapks) > 0:
1102             required = keepversions - len(apks)
1103             archapklist = filter_apk_list_sorted(archapks)
1104             # Move forward the ones we want again.
1105             for apk in archapklist[:required]:
1106                 logging.info("Moving " + apk['apkname'] + " from archive")
1107                 move_file(archivedir, repodir, apk['apkname'], False)
1108                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1109                 for density in all_screen_densities:
1110                     repo_icon_dir = get_icon_dir(repodir, density)
1111                     archive_icon_dir = get_icon_dir(archivedir, density)
1112                     if density not in apk['icons']:
1113                         continue
1114                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1115                 if 'srcname' in apk:
1116                     move_file(archivedir, repodir, apk['srcname'], False)
1117                 archapks.remove(apk)
1118                 apks.append(apk)
1119
1120
1121 def add_apks_to_per_app_repos(repodir, apks):
1122     apks_per_app = dict()
1123     for apk in apks:
1124         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1125         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1126         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1127         apks_per_app[apk['id']] = apk
1128
1129         if not os.path.exists(apk['per_app_icons']):
1130             logging.info('Adding new repo for only ' + apk['id'])
1131             os.makedirs(apk['per_app_icons'])
1132
1133         apkpath = os.path.join(repodir, apk['apkname'])
1134         shutil.copy(apkpath, apk['per_app_repo'])
1135         apksigpath = apkpath + '.sig'
1136         if os.path.exists(apksigpath):
1137             shutil.copy(apksigpath, apk['per_app_repo'])
1138         apkascpath = apkpath + '.asc'
1139         if os.path.exists(apkascpath):
1140             shutil.copy(apkascpath, apk['per_app_repo'])
1141
1142
1143 config = None
1144 options = None
1145
1146
1147 def main():
1148
1149     global config, options
1150
1151     # Parse command line...
1152     parser = ArgumentParser()
1153     common.setup_global_opts(parser)
1154     parser.add_argument("--create-key", action="store_true", default=False,
1155                         help="Create a repo signing key in a keystore")
1156     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1157                         help="Create skeleton metadata files that are missing")
1158     parser.add_argument("--delete-unknown", action="store_true", default=False,
1159                         help="Delete APKs without metadata from the repo")
1160     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1161                         help="Report on build data status")
1162     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1163                         help="Interactively ask about things that need updating.")
1164     parser.add_argument("-I", "--icons", action="store_true", default=False,
1165                         help="Resize all the icons exceeding the max pixel size and exit")
1166     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1167                         help="Specify editor to use in interactive mode. Default " +
1168                         "is /etc/alternatives/editor")
1169     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1170                         help="Update the wiki")
1171     parser.add_argument("--pretty", action="store_true", default=False,
1172                         help="Produce human-readable index.xml")
1173     parser.add_argument("--clean", action="store_true", default=False,
1174                         help="Clean update - don't uses caches, reprocess all apks")
1175     parser.add_argument("--nosign", action="store_true", default=False,
1176                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1177     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1178                         help="Use date from apk instead of current time for newly added apks")
1179     options = parser.parse_args()
1180
1181     config = common.read_config(options)
1182
1183     if not ('jarsigner' in config and 'keytool' in config):
1184         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1185         sys.exit(1)
1186
1187     repodirs = ['repo']
1188     if config['archive_older'] != 0:
1189         repodirs.append('archive')
1190         if not os.path.exists('archive'):
1191             os.mkdir('archive')
1192
1193     if options.icons:
1194         resize_all_icons(repodirs)
1195         sys.exit(0)
1196
1197     # check that icons exist now, rather than fail at the end of `fdroid update`
1198     for k in ['repo_icon', 'archive_icon']:
1199         if k in config:
1200             if not os.path.exists(config[k]):
1201                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1202                 sys.exit(1)
1203
1204     # if the user asks to create a keystore, do it now, reusing whatever it can
1205     if options.create_key:
1206         if os.path.exists(config['keystore']):
1207             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1208             logging.critical("\t'" + config['keystore'] + "'")
1209             sys.exit(1)
1210
1211         if 'repo_keyalias' not in config:
1212             config['repo_keyalias'] = socket.getfqdn()
1213             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1214         if 'keydname' not in config:
1215             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1216             common.write_to_config(config, 'keydname', config['keydname'])
1217         if 'keystore' not in config:
1218             config['keystore'] = common.default_config.keystore
1219             common.write_to_config(config, 'keystore', config['keystore'])
1220
1221         password = common.genpassword()
1222         if 'keystorepass' not in config:
1223             config['keystorepass'] = password
1224             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1225         if 'keypass' not in config:
1226             config['keypass'] = password
1227             common.write_to_config(config, 'keypass', config['keypass'])
1228         common.genkeystore(config)
1229
1230     # Get all apps...
1231     apps = metadata.read_metadata()
1232
1233     # Generate a list of categories...
1234     categories = set()
1235     for app in apps.values():
1236         categories.update(app.Categories)
1237
1238     # Read known apks data (will be updated and written back when we've finished)
1239     knownapks = common.KnownApks()
1240
1241     # Gather information about all the apk files in the repo directory, using
1242     # cached data if possible.
1243     apkcachefile = os.path.join('tmp', 'apkcache')
1244     if not options.clean and os.path.exists(apkcachefile):
1245         with open(apkcachefile, 'rb') as cf:
1246             apkcache = pickle.load(cf, encoding='utf-8')
1247         if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
1248             apkcache = {}
1249     else:
1250         apkcache = {}
1251
1252     delete_disabled_builds(apps, apkcache, repodirs)
1253
1254     # Scan all apks in the main repo
1255     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1256
1257     # Generate warnings for apk's with no metadata (or create skeleton
1258     # metadata files, if requested on the command line)
1259     newmetadata = False
1260     for apk in apks:
1261         if apk['id'] not in apps:
1262             if options.create_metadata:
1263                 if 'name' not in apk:
1264                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1265                     continue
1266                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
1267                 f.write("License:Unknown\n")
1268                 f.write("Web Site:\n")
1269                 f.write("Source Code:\n")
1270                 f.write("Issue Tracker:\n")
1271                 f.write("Changelog:\n")
1272                 f.write("Summary:" + apk['name'] + "\n")
1273                 f.write("Description:\n")
1274                 f.write(apk['name'] + "\n")
1275                 f.write(".\n")
1276                 f.close()
1277                 logging.info("Generated skeleton metadata for " + apk['id'])
1278                 newmetadata = True
1279             else:
1280                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1281                 if options.delete_unknown:
1282                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1283                     rmf = os.path.join(repodirs[0], apk['apkname'])
1284                     if not os.path.exists(rmf):
1285                         logging.error("Could not find {0} to remove it".format(rmf))
1286                     else:
1287                         os.remove(rmf)
1288                 else:
1289                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1290
1291     # update the metadata with the newly created ones included
1292     if newmetadata:
1293         apps = metadata.read_metadata()
1294
1295     # Scan the archive repo for apks as well
1296     if len(repodirs) > 1:
1297         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1298         if cc:
1299             cachechanged = True
1300     else:
1301         archapks = []
1302
1303     # Some information from the apks needs to be applied up to the application
1304     # level. When doing this, we use the info from the most recent version's apk.
1305     # We deal with figuring out when the app was added and last updated at the
1306     # same time.
1307     for appid, app in apps.items():
1308         bestver = 0
1309         for apk in apks + archapks:
1310             if apk['id'] == appid:
1311                 if apk['versioncode'] > bestver:
1312                     bestver = apk['versioncode']
1313                     bestapk = apk
1314
1315                 if 'added' in apk:
1316                     if not app.added or apk['added'] < app.added:
1317                         app.added = apk['added']
1318                     if not app.lastupdated or apk['added'] > app.lastupdated:
1319                         app.lastupdated = apk['added']
1320
1321         if not app.added:
1322             logging.debug("Don't know when " + appid + " was added")
1323         if not app.lastupdated:
1324             logging.debug("Don't know when " + appid + " was last updated")
1325
1326         if bestver == 0:
1327             if app.Name is None:
1328                 app.Name = app.AutoName or appid
1329             app.icon = None
1330             logging.debug("Application " + appid + " has no packages")
1331         else:
1332             if app.Name is None:
1333                 app.Name = bestapk['name']
1334             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1335             if app.CurrentVersionCode is None:
1336                 app.CurrentVersionCode = str(bestver)
1337
1338     # Sort the app list by name, then the web site doesn't have to by default.
1339     # (we had to wait until we'd scanned the apks to do this, because mostly the
1340     # name comes from there!)
1341     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1342
1343     # APKs are placed into multiple repos based on the app package, providing
1344     # per-app subscription feeds for nightly builds and things like it
1345     if config['per_app_repos']:
1346         add_apks_to_per_app_repos(repodirs[0], apks)
1347         for appid, app in apps.items():
1348             repodir = os.path.join(appid, 'fdroid', 'repo')
1349             appdict = dict()
1350             appdict[appid] = app
1351             if os.path.isdir(repodir):
1352                 make_index(appdict, [appid], apks, repodir, False, categories)
1353             else:
1354                 logging.info('Skipping index generation for ' + appid)
1355         return
1356
1357     if len(repodirs) > 1:
1358         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1359
1360     # Make the index for the main repo...
1361     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1362
1363     # If there's an archive repo,  make the index for it. We already scanned it
1364     # earlier on.
1365     if len(repodirs) > 1:
1366         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1367
1368     if config['update_stats']:
1369
1370         # Update known apks info...
1371         knownapks.writeifchanged()
1372
1373         # Generate latest apps data for widget
1374         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1375             data = ''
1376             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1377                 for line in f:
1378                     appid = line.rstrip()
1379                     data += appid + "\t"
1380                     app = apps[appid]
1381                     data += app.Name + "\t"
1382                     if app.icon is not None:
1383                         data += app.icon + "\t"
1384                     data += app.License + "\n"
1385             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1386                 f.write(data)
1387
1388     if cachechanged:
1389         apkcache["METADATA_VERSION"] = METADATA_VERSION
1390         with open(apkcachefile, 'wb') as cf:
1391             pickle.dump(apkcache, cf)
1392
1393     # Update the wiki...
1394     if options.wiki:
1395         update_wiki(apps, sortedids, apks + archapks)
1396
1397     logging.info("Finished.")
1398
1399 if __name__ == "__main__":
1400     main()