fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import socket
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 import urlparse
  31 from datetime import datetime, timedelta
  32 from xml.dom.minidom import Document
  33 from argparse import ArgumentParser
  34 import time
  35 from pyasn1.error import PyAsn1Error
  36 from pyasn1.codec.der import decoder, encoder
  37 from pyasn1_modules import rfc2315
  38 from hashlib import md5
  39 from binascii import hexlify, unhexlify
  40
  41 from PIL import Image
  42 import logging
  43
  44 import common
  45 import metadata
  46 from common import FDroidPopen, SdkToolsPopen
  47 from metadata import MetaDataException
  48
  49 screen_densities = ['640', '480', '320', '240', '160', '120']
  50
  51 all_screen_densities = ['0'] + screen_densities
  52
  53
  54 def dpi_to_px(density):
  55     return (int(density) * 48) / 160
  56
  57
  58 def px_to_dpi(px):
  59     return (int(px) * 160) / 48
  60
  61
  62 def get_icon_dir(repodir, density):
  63     if density == '0':
  64         return os.path.join(repodir, "icons")
  65     return os.path.join(repodir, "icons-%s" % density)
  66
  67
  68 def get_icon_dirs(repodir):
  69     for density in screen_densities:
  70         yield get_icon_dir(repodir, density)
  71
  72
  73 def get_all_icon_dirs(repodir):
  74     for density in all_screen_densities:
  75         yield get_icon_dir(repodir, density)
  76
  77
  78 def update_wiki(apps, sortedids, apks):
  79     """Update the wiki
  80
  81     :param apps: fully populated list of all applications
  82     :param apks: all apks, except...
  83     """
  84     logging.info("Updating wiki")
  85     wikicat = 'Apps'
  86     wikiredircat = 'App Redirects'
  87     import mwclient
  88     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  89                          path=config['wiki_path'])
  90     site.login(config['wiki_user'], config['wiki_password'])
  91     generated_pages = {}
  92     generated_redirects = {}
  93
  94     for appid in sortedids:
  95         app = apps[appid]
  96
  97         wikidata = ''
  98         if app.Disabled:
  99             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 100         if app.AntiFeatures:
 101             for af in app.AntiFeatures:
 102                 wikidata += '{{AntiFeature|' + af + '}}\n'
 103         if app.RequiresRoot:
 104             requiresroot = 'Yes'
 105         else:
 106             requiresroot = 'No'
 107         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 108             appid,
 109             app.Name,
 110             time.strftime('%Y-%m-%d', app.added) if app.added else '',
 111             time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
 112             app.SourceCode,
 113             app.IssueTracker,
 114             app.WebSite,
 115             app.Changelog,
 116             app.Donate,
 117             app.FlattrID,
 118             app.Bitcoin,
 119             app.Litecoin,
 120             app.License,
 121             requiresroot,
 122             app.AuthorName,
 123             app.AuthorEmail)
 124
 125         if app.Provides:
 126             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 127
 128         wikidata += app.Summary
 129         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 130
 131         wikidata += "=Description=\n"
 132         wikidata += metadata.description_wiki(app.Description) + "\n"
 133
 134         wikidata += "=Maintainer Notes=\n"
 135         if app.MaintainerNotes:
 136             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 137         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 138
 139         # Get a list of all packages for this application...
 140         apklist = []
 141         gotcurrentver = False
 142         cantupdate = False
 143         buildfails = False
 144         for apk in apks:
 145             if apk['id'] == appid:
 146                 if str(apk['versioncode']) == app.CurrentVersionCode:
 147                     gotcurrentver = True
 148                 apklist.append(apk)
 149         # Include ones we can't build, as a special case...
 150         for build in app.builds:
 151             if build.disable:
 152                 if build.vercode == app.CurrentVersionCode:
 153                     cantupdate = True
 154                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 155                 apklist.append({'versioncode': int(build.vercode),
 156                                 'version': build.version,
 157                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 158                                 })
 159             else:
 160                 builtit = False
 161                 for apk in apklist:
 162                     if apk['versioncode'] == int(build.vercode):
 163                         builtit = True
 164                         break
 165                 if not builtit:
 166                     buildfails = True
 167                     apklist.append({'versioncode': int(build.vercode),
 168                                     'version': build.version,
 169                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
 170                                     })
 171         if app.CurrentVersionCode == '0':
 172             cantupdate = True
 173         # Sort with most recent first...
 174         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 175
 176         wikidata += "=Versions=\n"
 177         if len(apklist) == 0:
 178             wikidata += "We currently have no versions of this app available."
 179         elif not gotcurrentver:
 180             wikidata += "We don't have the current version of this app."
 181         else:
 182             wikidata += "We have the current version of this app."
 183         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 184         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 185         if len(app.NoSourceSince) > 0:
 186             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 187         if len(app.CurrentVersion) > 0:
 188             wikidata += "The current (recommended) version is " + app.CurrentVersion
 189             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 190         validapks = 0
 191         for apk in apklist:
 192             wikidata += "==" + apk['version'] + "==\n"
 193
 194             if 'buildproblem' in apk:
 195                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 196             else:
 197                 validapks += 1
 198                 wikidata += "This version is built and signed by "
 199                 if 'srcname' in apk:
 200                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 201                 else:
 202                     wikidata += "the original developer.\n\n"
 203             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 204
 205         wikidata += '\n[[Category:' + wikicat + ']]\n'
 206         if len(app.NoSourceSince) > 0:
 207             wikidata += '\n[[Category:Apps missing source code]]\n'
 208         if validapks == 0 and not app.Disabled:
 209             wikidata += '\n[[Category:Apps with no packages]]\n'
 210         if cantupdate and not app.Disabled:
 211             wikidata += "\n[[Category:Apps we cannot update]]\n"
 212         if buildfails and not app.Disabled:
 213             wikidata += "\n[[Category:Apps with failing builds]]\n"
 214         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 215             wikidata += '\n[[Category:Apps to Update]]\n'
 216         if app.Disabled:
 217             wikidata += '\n[[Category:Apps that are disabled]]\n'
 218         if app.UpdateCheckMode == 'None' and not app.Disabled:
 219             wikidata += '\n[[Category:Apps with no update check]]\n'
 220         for appcat in app.Categories:
 221             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 222
 223         # We can't have underscores in the page name, even if they're in
 224         # the package ID, because MediaWiki messes with them...
 225         pagename = appid.replace('_', ' ')
 226
 227         # Drop a trailing newline, because mediawiki is going to drop it anyway
 228         # and it we don't we'll think the page has changed when it hasn't...
 229         if wikidata.endswith('\n'):
 230             wikidata = wikidata[:-1]
 231
 232         generated_pages[pagename] = wikidata
 233
 234         # Make a redirect from the name to the ID too, unless there's
 235         # already an existing page with the name and it isn't a redirect.
 236         noclobber = False
 237         apppagename = app.Name.replace('_', ' ')
 238         apppagename = apppagename.replace('{', '')
 239         apppagename = apppagename.replace('}', ' ')
 240         apppagename = apppagename.replace(':', ' ')
 241         # Drop double spaces caused mostly by replacing ':' above
 242         apppagename = apppagename.replace('  ', ' ')
 243         for expagename in site.allpages(prefix=apppagename,
 244                                         filterredir='nonredirects',
 245                                         generator=False):
 246             if expagename == apppagename:
 247                 noclobber = True
 248         # Another reason not to make the redirect page is if the app name
 249         # is the same as it's ID, because that will overwrite the real page
 250         # with an redirect to itself! (Although it seems like an odd
 251         # scenario this happens a lot, e.g. where there is metadata but no
 252         # builds or binaries to extract a name from.
 253         if apppagename == pagename:
 254             noclobber = True
 255         if not noclobber:
 256             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 257
 258     for tcat, genp in [(wikicat, generated_pages),
 259                        (wikiredircat, generated_redirects)]:
 260         catpages = site.Pages['Category:' + tcat]
 261         existingpages = []
 262         for page in catpages:
 263             existingpages.append(page.name)
 264             if page.name in genp:
 265                 pagetxt = page.edit()
 266                 if pagetxt != genp[page.name]:
 267                     logging.debug("Updating modified page " + page.name)
 268                     page.save(genp[page.name], summary='Auto-updated')
 269                 else:
 270                     logging.debug("Page " + page.name + " is unchanged")
 271             else:
 272                 logging.warn("Deleting page " + page.name)
 273                 page.delete('No longer published')
 274         for pagename, text in genp.items():
 275             logging.debug("Checking " + pagename)
 276             if pagename not in existingpages:
 277                 logging.debug("Creating page " + pagename)
 278                 try:
 279                     newpage = site.Pages[pagename]
 280                     newpage.save(text, summary='Auto-created')
 281                 except:
 282                     logging.error("...FAILED to create page '{0}'".format(pagename))
 283
 284     # Purge server cache to ensure counts are up to date
 285     site.pages['Repository Maintenance'].purge()
 286
 287
 288 def delete_disabled_builds(apps, apkcache, repodirs):
 289     """Delete disabled build outputs.
 290
 291     :param apps: list of all applications, as per metadata.read_metadata
 292     :param apkcache: current apk cache information
 293     :param repodirs: the repo directories to process
 294     """
 295     for appid, app in apps.iteritems():
 296         for build in app.builds:
 297             if not build.disable:
 298                 continue
 299             apkfilename = appid + '_' + str(build.vercode) + '.apk'
 300             iconfilename = "%s.%s.png" % (
 301                 appid,
 302                 build.vercode)
 303             for repodir in repodirs:
 304                 files = [
 305                     os.path.join(repodir, apkfilename),
 306                     os.path.join(repodir, apkfilename + '.asc'),
 307                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 308                 ]
 309                 for density in all_screen_densities:
 310                     repo_dir = get_icon_dir(repodir, density)
 311                     files.append(os.path.join(repo_dir, iconfilename))
 312
 313                 for f in files:
 314                     if os.path.exists(f):
 315                         logging.info("Deleting disabled build output " + f)
 316                         os.remove(f)
 317             if apkfilename in apkcache:
 318                 del apkcache[apkfilename]
 319
 320
 321 def resize_icon(iconpath, density):
 322
 323     if not os.path.isfile(iconpath):
 324         return
 325
 326     try:
 327         im = Image.open(iconpath)
 328         size = dpi_to_px(density)
 329
 330         if any(length > size for length in im.size):
 331             oldsize = im.size
 332             im.thumbnail((size, size), Image.ANTIALIAS)
 333             logging.debug("%s was too large at %s - new size is %s" % (
 334                 iconpath, oldsize, im.size))
 335             im.save(iconpath, "PNG")
 336
 337     except Exception as e:
 338         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 339
 340
 341 def resize_all_icons(repodirs):
 342     """Resize all icons that exceed the max size
 343
 344     :param repodirs: the repo directories to process
 345     """
 346     for repodir in repodirs:
 347         for density in screen_densities:
 348             icon_dir = get_icon_dir(repodir, density)
 349             icon_glob = os.path.join(icon_dir, '*.png')
 350             for iconpath in glob.glob(icon_glob):
 351                 resize_icon(iconpath, density)
 352
 353
 354 # A signature block file with a .DSA, .RSA, or .EC extension
 355 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 356
 357
 358 def getsig(apkpath):
 359     """ Get the signing certificate of an apk. To get the same md5 has that
 360     Android gets, we encode the .RSA certificate in a specific format and pass
 361     it hex-encoded to the md5 digest algorithm.
 362
 363     :param apkpath: path to the apk
 364     :returns: A string containing the md5 of the signature of the apk or None
 365               if an error occurred.
 366     """
 367
 368     cert = None
 369
 370     # verify the jar signature is correct
 371     args = [config['jarsigner'], '-verify', apkpath]
 372     p = FDroidPopen(args)
 373     if p.returncode != 0:
 374         logging.critical(apkpath + " has a bad signature!")
 375         return None
 376
 377     with zipfile.ZipFile(apkpath, 'r') as apk:
 378
 379         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 380
 381         if len(certs) < 1:
 382             logging.error("Found no signing certificates on %s" % apkpath)
 383             return None
 384         if len(certs) > 1:
 385             logging.error("Found multiple signing certificates on %s" % apkpath)
 386             return None
 387
 388         cert = apk.read(certs[0])
 389
 390     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 391     if content.getComponentByName('contentType') != rfc2315.signedData:
 392         logging.error("Unexpected format.")
 393         return None
 394
 395     content = decoder.decode(content.getComponentByName('content'),
 396                              asn1Spec=rfc2315.SignedData())[0]
 397     try:
 398         certificates = content.getComponentByName('certificates')
 399     except PyAsn1Error:
 400         logging.error("Certificates not found.")
 401         return None
 402
 403     cert_encoded = encoder.encode(certificates)[4:]
 404
 405     return md5(cert_encoded.encode('hex')).hexdigest()
 406
 407
 408 def scan_apks(apps, apkcache, repodir, knownapks):
 409     """Scan the apks in the given repo directory.
 410
 411     This also extracts the icons.
 412
 413     :param apps: list of all applications, as per metadata.read_metadata
 414     :param apkcache: current apk cache information
 415     :param repodir: repo directory to scan
 416     :param knownapks: known apks info
 417     :returns: (apks, cachechanged) where apks is a list of apk information,
 418               and cachechanged is True if the apkcache got changed.
 419     """
 420
 421     cachechanged = False
 422
 423     for icon_dir in get_all_icon_dirs(repodir):
 424         if os.path.exists(icon_dir):
 425             if options.clean:
 426                 shutil.rmtree(icon_dir)
 427                 os.makedirs(icon_dir)
 428         else:
 429             os.makedirs(icon_dir)
 430
 431     apks = []
 432     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 433     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 434     vername_pat = re.compile(".*versionName='([^']*)'.*")
 435     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 436     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 437     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 438     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 439     string_pat = re.compile(".*'([^']*)'.*")
 440     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 441
 442         apkfilename = apkfile[len(repodir) + 1:]
 443         if ' ' in apkfilename:
 444             logging.critical("Spaces in filenames are not allowed.")
 445             sys.exit(1)
 446
 447         # Calculate the sha256...
 448         sha = hashlib.sha256()
 449         with open(apkfile, 'rb') as f:
 450             while True:
 451                 t = f.read(16384)
 452                 if len(t) == 0:
 453                     break
 454                 sha.update(t)
 455             shasum = sha.hexdigest()
 456
 457         usecache = False
 458         if apkfilename in apkcache:
 459             apk = apkcache[apkfilename]
 460             if apk['sha256'] == shasum:
 461                 logging.debug("Reading " + apkfilename + " from cache")
 462                 usecache = True
 463             else:
 464                 logging.debug("Ignoring stale cache data for " + apkfilename)
 465
 466         if not usecache:
 467             logging.debug("Processing " + apkfilename)
 468             apk = {}
 469             apk['apkname'] = apkfilename
 470             apk['sha256'] = shasum
 471             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 472             if os.path.exists(os.path.join(repodir, srcfilename)):
 473                 apk['srcname'] = srcfilename
 474             apk['size'] = os.path.getsize(apkfile)
 475             apk['permissions'] = set()
 476             apk['features'] = set()
 477             apk['icons_src'] = {}
 478             apk['icons'] = {}
 479             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 480             if p.returncode != 0:
 481                 if options.delete_unknown:
 482                     if os.path.exists(apkfile):
 483                         logging.error("Failed to get apk information, deleting " + apkfile)
 484                         os.remove(apkfile)
 485                     else:
 486                         logging.error("Could not find {0} to remove it".format(apkfile))
 487                 else:
 488                     logging.error("Failed to get apk information, skipping " + apkfile)
 489                 continue
 490             for line in p.output.splitlines():
 491                 if line.startswith("package:"):
 492                     try:
 493                         apk['id'] = re.match(name_pat, line).group(1)
 494                         apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
 495                         apk['version'] = re.match(vername_pat, line).group(1)
 496                     except Exception as e:
 497                         logging.error("Package matching failed: " + str(e))
 498                         logging.info("Line was: " + line)
 499                         sys.exit(1)
 500                 elif line.startswith("application:"):
 501                     apk['name'] = re.match(label_pat, line).group(1)
 502                     # Keep path to non-dpi icon in case we need it
 503                     match = re.match(icon_pat_nodpi, line)
 504                     if match:
 505                         apk['icons_src']['-1'] = match.group(1)
 506                 elif line.startswith("launchable-activity:"):
 507                     # Only use launchable-activity as fallback to application
 508                     if not apk['name']:
 509                         apk['name'] = re.match(label_pat, line).group(1)
 510                     if '-1' not in apk['icons_src']:
 511                         match = re.match(icon_pat_nodpi, line)
 512                         if match:
 513                             apk['icons_src']['-1'] = match.group(1)
 514                 elif line.startswith("application-icon-"):
 515                     match = re.match(icon_pat, line)
 516                     if match:
 517                         density = match.group(1)
 518                         path = match.group(2)
 519                         apk['icons_src'][density] = path
 520                 elif line.startswith("sdkVersion:"):
 521                     m = re.match(sdkversion_pat, line)
 522                     if m is None:
 523                         logging.error(line.replace('sdkVersion:', '')
 524                                       + ' is not a valid minSdkVersion!')
 525                     else:
 526                         apk['sdkversion'] = m.group(1)
 527                 elif line.startswith("maxSdkVersion:"):
 528                     apk['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 529                 elif line.startswith("native-code:"):
 530                     apk['nativecode'] = []
 531                     for arch in line[13:].split(' '):
 532                         apk['nativecode'].append(arch[1:-1])
 533                 elif line.startswith("uses-permission:"):
 534                     perm = re.match(string_pat, line).group(1)
 535                     if perm.startswith("android.permission."):
 536                         perm = perm[19:]
 537                     apk['permissions'].add(perm)
 538                 elif line.startswith("uses-feature:"):
 539                     perm = re.match(string_pat, line).group(1)
 540                     # Filter out this, it's only added with the latest SDK tools and
 541                     # causes problems for lots of apps.
 542                     if perm != "android.hardware.screen.portrait" \
 543                             and perm != "android.hardware.screen.landscape":
 544                         if perm.startswith("android.feature."):
 545                             perm = perm[16:]
 546                         apk['features'].add(perm)
 547
 548             if 'sdkversion' not in apk:
 549                 logging.warn("No SDK version information found in {0}".format(apkfile))
 550                 apk['sdkversion'] = 0
 551
 552             # Check for debuggable apks...
 553             if common.isApkDebuggable(apkfile, config):
 554                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 555
 556             # Get the signature (or md5 of, to be precise)...
 557             logging.debug('Getting signature of {0}'.format(apkfile))
 558             apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 559             if not apk['sig']:
 560                 logging.critical("Failed to get apk signature")
 561                 sys.exit(1)
 562
 563             apkzip = zipfile.ZipFile(apkfile, 'r')
 564
 565             # if an APK has files newer than the system time, suggest updating
 566             # the system clock.  This is useful for offline systems, used for
 567             # signing, which do not have another source of clock sync info. It
 568             # has to be more than 24 hours newer because ZIP/APK files do not
 569             # store timezone info
 570             manifest = apkzip.getinfo('AndroidManifest.xml')
 571             dt_obj = datetime(*manifest.date_time)
 572             checkdt = dt_obj - timedelta(1)
 573             if datetime.today() < checkdt:
 574                 logging.warn('System clock is older than manifest in: '
 575                              + apkfilename + '\nSet clock to that time using:\n'
 576                              + 'sudo date -s "' + str(dt_obj) + '"')
 577
 578             iconfilename = "%s.%s.png" % (
 579                 apk['id'],
 580                 apk['versioncode'])
 581
 582             # Extract the icon file...
 583             empty_densities = []
 584             for density in screen_densities:
 585                 if density not in apk['icons_src']:
 586                     empty_densities.append(density)
 587                     continue
 588                 iconsrc = apk['icons_src'][density]
 589                 icon_dir = get_icon_dir(repodir, density)
 590                 icondest = os.path.join(icon_dir, iconfilename)
 591
 592                 try:
 593                     with open(icondest, 'wb') as f:
 594                         f.write(apkzip.read(iconsrc))
 595                     apk['icons'][density] = iconfilename
 596
 597                 except:
 598                     logging.warn("Error retrieving icon file")
 599                     del apk['icons'][density]
 600                     del apk['icons_src'][density]
 601                     empty_densities.append(density)
 602
 603             if '-1' in apk['icons_src']:
 604                 iconsrc = apk['icons_src']['-1']
 605                 iconpath = os.path.join(
 606                     get_icon_dir(repodir, '0'), iconfilename)
 607                 with open(iconpath, 'wb') as f:
 608                     f.write(apkzip.read(iconsrc))
 609                 try:
 610                     im = Image.open(iconpath)
 611                     dpi = px_to_dpi(im.size[0])
 612                     for density in screen_densities:
 613                         if density in apk['icons']:
 614                             break
 615                         if density == screen_densities[-1] or dpi >= int(density):
 616                             apk['icons'][density] = iconfilename
 617                             shutil.move(iconpath,
 618                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 619                             empty_densities.remove(density)
 620                             break
 621                 except Exception as e:
 622                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 623
 624             if apk['icons']:
 625                 apk['icon'] = iconfilename
 626
 627             apkzip.close()
 628
 629             # First try resizing down to not lose quality
 630             last_density = None
 631             for density in screen_densities:
 632                 if density not in empty_densities:
 633                     last_density = density
 634                     continue
 635                 if last_density is None:
 636                     continue
 637                 logging.debug("Density %s not available, resizing down from %s"
 638                               % (density, last_density))
 639
 640                 last_iconpath = os.path.join(
 641                     get_icon_dir(repodir, last_density), iconfilename)
 642                 iconpath = os.path.join(
 643                     get_icon_dir(repodir, density), iconfilename)
 644                 try:
 645                     im = Image.open(last_iconpath)
 646                 except:
 647                     logging.warn("Invalid image file at %s" % last_iconpath)
 648                     continue
 649
 650                 size = dpi_to_px(density)
 651
 652                 im.thumbnail((size, size), Image.ANTIALIAS)
 653                 im.save(iconpath, "PNG")
 654                 empty_densities.remove(density)
 655
 656             # Then just copy from the highest resolution available
 657             last_density = None
 658             for density in reversed(screen_densities):
 659                 if density not in empty_densities:
 660                     last_density = density
 661                     continue
 662                 if last_density is None:
 663                     continue
 664                 logging.debug("Density %s not available, copying from lower density %s"
 665                               % (density, last_density))
 666
 667                 shutil.copyfile(
 668                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 669                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 670
 671                 empty_densities.remove(density)
 672
 673             for density in screen_densities:
 674                 icon_dir = get_icon_dir(repodir, density)
 675                 icondest = os.path.join(icon_dir, iconfilename)
 676                 resize_icon(icondest, density)
 677
 678             # Copy from icons-mdpi to icons since mdpi is the baseline density
 679             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 680             if os.path.isfile(baseline):
 681                 apk['icons']['0'] = iconfilename
 682                 shutil.copyfile(baseline,
 683                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 684
 685             # Record in known apks, getting the added date at the same time..
 686             added = knownapks.recordapk(apk['apkname'], apk['id'])
 687             if added:
 688                 apk['added'] = added
 689
 690             apkcache[apkfilename] = apk
 691             cachechanged = True
 692
 693         apks.append(apk)
 694
 695     return apks, cachechanged
 696
 697
 698 repo_pubkey_fingerprint = None
 699
 700
 701 # Generate a certificate fingerprint the same way keytool does it
 702 # (but with slightly different formatting)
 703 def cert_fingerprint(data):
 704     digest = hashlib.sha256(data).digest()
 705     ret = []
 706     ret.append(' '.join("%02X" % ord(b) for b in digest))
 707     return " ".join(ret)
 708
 709
 710 def extract_pubkey():
 711     global repo_pubkey_fingerprint
 712     if 'repo_pubkey' in config:
 713         pubkey = unhexlify(config['repo_pubkey'])
 714     else:
 715         p = FDroidPopen([config['keytool'], '-exportcert',
 716                          '-alias', config['repo_keyalias'],
 717                          '-keystore', config['keystore'],
 718                          '-storepass:file', config['keystorepassfile']]
 719                         + config['smartcardoptions'],
 720                         output=False, stderr_to_stdout=False)
 721         if p.returncode != 0 or len(p.output) < 20:
 722             msg = "Failed to get repo pubkey!"
 723             if config['keystore'] == 'NONE':
 724                 msg += ' Is your crypto smartcard plugged in?'
 725             logging.critical(msg)
 726             sys.exit(1)
 727         pubkey = p.output
 728     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 729     return hexlify(pubkey)
 730
 731
 732 def make_index(apps, sortedids, apks, repodir, archive, categories):
 733     """Make a repo index.
 734
 735     :param apps: fully populated apps list
 736     :param apks: full populated apks list
 737     :param repodir: the repo directory
 738     :param archive: True if this is the archive repo, False if it's the
 739                     main one.
 740     :param categories: list of categories
 741     """
 742
 743     doc = Document()
 744
 745     def addElement(name, value, doc, parent):
 746         el = doc.createElement(name)
 747         el.appendChild(doc.createTextNode(value))
 748         parent.appendChild(el)
 749
 750     def addElementNonEmpty(name, value, doc, parent):
 751         if not value:
 752             return
 753         addElement(name, value, doc, parent)
 754
 755     def addElementCDATA(name, value, doc, parent):
 756         el = doc.createElement(name)
 757         el.appendChild(doc.createCDATASection(value))
 758         parent.appendChild(el)
 759
 760     root = doc.createElement("fdroid")
 761     doc.appendChild(root)
 762
 763     repoel = doc.createElement("repo")
 764
 765     mirrorcheckfailed = False
 766     for mirror in config.get('mirrors', []):
 767         base = os.path.basename(urlparse.urlparse(mirror).path.rstrip('/'))
 768         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
 769             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 770             mirrorcheckfailed = True
 771     if mirrorcheckfailed:
 772         sys.exit(1)
 773
 774     if archive:
 775         repoel.setAttribute("name", config['archive_name'])
 776         if config['repo_maxage'] != 0:
 777             repoel.setAttribute("maxage", str(config['repo_maxage']))
 778         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 779         repoel.setAttribute("url", config['archive_url'])
 780         addElement('description', config['archive_description'], doc, repoel)
 781         urlbasepath = os.path.basename(urlparse.urlparse(config['archive_url']).path)
 782         for mirror in config.get('mirrors', []):
 783             addElement('mirror', urlparse.urljoin(mirror, urlbasepath), doc, repoel)
 784
 785     else:
 786         repoel.setAttribute("name", config['repo_name'])
 787         if config['repo_maxage'] != 0:
 788             repoel.setAttribute("maxage", str(config['repo_maxage']))
 789         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 790         repoel.setAttribute("url", config['repo_url'])
 791         addElement('description', config['repo_description'], doc, repoel)
 792         urlbasepath = os.path.basename(urlparse.urlparse(config['repo_url']).path)
 793         for mirror in config.get('mirrors', []):
 794             addElement('mirror', urlparse.urljoin(mirror, urlbasepath), doc, repoel)
 795
 796     repoel.setAttribute("version", "15")
 797     repoel.setAttribute("timestamp", str(int(time.time())))
 798
 799     nosigningkey = False
 800     if not options.nosign:
 801         if 'repo_keyalias' not in config:
 802             nosigningkey = True
 803             logging.critical("'repo_keyalias' not found in config.py!")
 804         if 'keystore' not in config:
 805             nosigningkey = True
 806             logging.critical("'keystore' not found in config.py!")
 807         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 808             nosigningkey = True
 809             logging.critical("'keystorepass' not found in config.py!")
 810         if 'keypass' not in config and 'keypassfile' not in config:
 811             nosigningkey = True
 812             logging.critical("'keypass' not found in config.py!")
 813         if not os.path.exists(config['keystore']):
 814             nosigningkey = True
 815             logging.critical("'" + config['keystore'] + "' does not exist!")
 816         if nosigningkey:
 817             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 818             logging.warning("\tfdroid update --create-key")
 819             sys.exit(1)
 820
 821     repoel.setAttribute("pubkey", extract_pubkey())
 822     root.appendChild(repoel)
 823
 824     for appid in sortedids:
 825         app = apps[appid]
 826
 827         if app.Disabled is not None:
 828             continue
 829
 830         # Get a list of the apks for this app...
 831         apklist = []
 832         for apk in apks:
 833             if apk['id'] == appid:
 834                 apklist.append(apk)
 835
 836         if len(apklist) == 0:
 837             continue
 838
 839         apel = doc.createElement("application")
 840         apel.setAttribute("id", app.id)
 841         root.appendChild(apel)
 842
 843         addElement('id', app.id, doc, apel)
 844         if app.added:
 845             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
 846         if app.lastupdated:
 847             addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
 848         addElement('name', app.Name, doc, apel)
 849         addElement('summary', app.Summary, doc, apel)
 850         if app.icon:
 851             addElement('icon', app.icon, doc, apel)
 852
 853         def linkres(appid):
 854             if appid in apps:
 855                 return ("fdroid.app:" + appid, apps[appid].Name)
 856             raise MetaDataException("Cannot resolve app id " + appid)
 857
 858         addElement('desc',
 859                    metadata.description_html(app.Description, linkres),
 860                    doc, apel)
 861         addElement('license', app.License, doc, apel)
 862         if app.Categories:
 863             addElement('categories', ','.join(app.Categories), doc, apel)
 864             # We put the first (primary) category in LAST, which will have
 865             # the desired effect of making clients that only understand one
 866             # category see that one.
 867             addElement('category', app.Categories[0], doc, apel)
 868         addElement('web', app.WebSite, doc, apel)
 869         addElement('source', app.SourceCode, doc, apel)
 870         addElement('tracker', app.IssueTracker, doc, apel)
 871         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 872         addElementNonEmpty('author', app.AuthorName, doc, apel)
 873         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 874         addElementNonEmpty('donate', app.Donate, doc, apel)
 875         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 876         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 877         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 878
 879         # These elements actually refer to the current version (i.e. which
 880         # one is recommended. They are historically mis-named, and need
 881         # changing, but stay like this for now to support existing clients.
 882         addElement('marketversion', app.CurrentVersion, doc, apel)
 883         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 884
 885         if app.AntiFeatures:
 886             af = app.AntiFeatures
 887             if af:
 888                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 889         if app.Provides:
 890             pv = app.Provides.split(',')
 891             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 892         if app.RequiresRoot:
 893             addElement('requirements', 'root', doc, apel)
 894
 895         # Sort the apk list into version order, just so the web site
 896         # doesn't have to do any work by default...
 897         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 898
 899         # Check for duplicates - they will make the client unhappy...
 900         for i in range(len(apklist) - 1):
 901             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 902                 logging.critical("duplicate versions: '%s' - '%s'" % (
 903                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 904                 sys.exit(1)
 905
 906         current_version_code = 0
 907         current_version_file = None
 908         for apk in apklist:
 909             # find the APK for the "Current Version"
 910             if current_version_code < apk['versioncode']:
 911                 current_version_code = apk['versioncode']
 912             if current_version_code < int(app.CurrentVersionCode):
 913                 current_version_file = apk['apkname']
 914
 915             apkel = doc.createElement("package")
 916             apel.appendChild(apkel)
 917             addElement('version', apk['version'], doc, apkel)
 918             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 919             addElement('apkname', apk['apkname'], doc, apkel)
 920             if 'srcname' in apk:
 921                 addElement('srcname', apk['srcname'], doc, apkel)
 922             for hash_type in ['sha256']:
 923                 if hash_type not in apk:
 924                     continue
 925                 hashel = doc.createElement("hash")
 926                 hashel.setAttribute("type", hash_type)
 927                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 928                 apkel.appendChild(hashel)
 929             addElement('sig', apk['sig'], doc, apkel)
 930             addElement('size', str(apk['size']), doc, apkel)
 931             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 932             if 'maxsdkversion' in apk:
 933                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 934             if 'added' in apk:
 935                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 936             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
 937             if 'nativecode' in apk:
 938                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 939             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
 940
 941         if current_version_file is not None \
 942                 and config['make_current_version_link'] \
 943                 and repodir == 'repo':  # only create these
 944             namefield = config['current_version_name_source']
 945             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
 946             apklinkname = sanitized_name + '.apk'
 947             current_version_path = os.path.join(repodir, current_version_file)
 948             if os.path.islink(apklinkname):
 949                 os.remove(apklinkname)
 950             os.symlink(current_version_path, apklinkname)
 951             # also symlink gpg signature, if it exists
 952             for extension in ('.asc', '.sig'):
 953                 sigfile_path = current_version_path + extension
 954                 if os.path.exists(sigfile_path):
 955                     siglinkname = apklinkname + extension
 956                     if os.path.islink(siglinkname):
 957                         os.remove(siglinkname)
 958                     os.symlink(sigfile_path, siglinkname)
 959
 960     if options.pretty:
 961         output = doc.toprettyxml()
 962     else:
 963         output = doc.toxml()
 964
 965     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 966         f.write(output)
 967
 968     if 'repo_keyalias' in config:
 969
 970         if options.nosign:
 971             logging.info("Creating unsigned index in preparation for signing")
 972         else:
 973             logging.info("Creating signed index with this key (SHA256):")
 974             logging.info("%s" % repo_pubkey_fingerprint)
 975
 976         # Create a jar of the index...
 977         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
 978         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 979         if p.returncode != 0:
 980             logging.critical("Failed to create {0}".format(jar_output))
 981             sys.exit(1)
 982
 983         # Sign the index...
 984         signed = os.path.join(repodir, 'index.jar')
 985         if options.nosign:
 986             # Remove old signed index if not signing
 987             if os.path.exists(signed):
 988                 os.remove(signed)
 989         else:
 990             args = [config['jarsigner'], '-keystore', config['keystore'],
 991                     '-storepass:file', config['keystorepassfile'],
 992                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
 993                     signed, config['repo_keyalias']]
 994             if config['keystore'] == 'NONE':
 995                 args += config['smartcardoptions']
 996             else:  # smardcards never use -keypass
 997                 args += ['-keypass:file', config['keypassfile']]
 998             p = FDroidPopen(args)
 999             if p.returncode != 0:
1000                 logging.critical("Failed to sign index")
1001                 sys.exit(1)
1002
1003     # Copy the repo icon into the repo directory...
1004     icon_dir = os.path.join(repodir, 'icons')
1005     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1006     shutil.copyfile(config['repo_icon'], iconfilename)
1007
1008     # Write a category list in the repo to allow quick access...
1009     catdata = ''
1010     for cat in categories:
1011         catdata += cat + '\n'
1012     with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
1013         f.write(catdata)
1014
1015
1016 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1017
1018     for appid, app in apps.iteritems():
1019
1020         if app.ArchivePolicy:
1021             keepversions = int(app.ArchivePolicy[:-9])
1022         else:
1023             keepversions = defaultkeepversions
1024
1025         def filter_apk_list_sorted(apk_list):
1026             res = []
1027             for apk in apk_list:
1028                 if apk['id'] == appid:
1029                     res.append(apk)
1030
1031             # Sort the apk list by version code. First is highest/newest.
1032             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1033
1034         def move_file(from_dir, to_dir, filename, ignore_missing):
1035             from_path = os.path.join(from_dir, filename)
1036             if ignore_missing and not os.path.exists(from_path):
1037                 return
1038             to_path = os.path.join(to_dir, filename)
1039             shutil.move(from_path, to_path)
1040
1041         if len(apks) > keepversions:
1042             apklist = filter_apk_list_sorted(apks)
1043             # Move back the ones we don't want.
1044             for apk in apklist[keepversions:]:
1045                 logging.info("Moving " + apk['apkname'] + " to archive")
1046                 move_file(repodir, archivedir, apk['apkname'], False)
1047                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1048                 for density in all_screen_densities:
1049                     repo_icon_dir = get_icon_dir(repodir, density)
1050                     archive_icon_dir = get_icon_dir(archivedir, density)
1051                     if density not in apk['icons']:
1052                         continue
1053                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1054                 if 'srcname' in apk:
1055                     move_file(repodir, archivedir, apk['srcname'], False)
1056                 archapks.append(apk)
1057                 apks.remove(apk)
1058         elif len(apks) < keepversions and len(archapks) > 0:
1059             required = keepversions - len(apks)
1060             archapklist = filter_apk_list_sorted(archapks)
1061             # Move forward the ones we want again.
1062             for apk in archapklist[:required]:
1063                 logging.info("Moving " + apk['apkname'] + " from archive")
1064                 move_file(archivedir, repodir, apk['apkname'], False)
1065                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1066                 for density in all_screen_densities:
1067                     repo_icon_dir = get_icon_dir(repodir, density)
1068                     archive_icon_dir = get_icon_dir(archivedir, density)
1069                     if density not in apk['icons']:
1070                         continue
1071                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1072                 if 'srcname' in apk:
1073                     move_file(archivedir, repodir, apk['srcname'], False)
1074                 archapks.remove(apk)
1075                 apks.append(apk)
1076
1077
1078 def add_apks_to_per_app_repos(repodir, apks):
1079     apks_per_app = dict()
1080     for apk in apks:
1081         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1082         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1083         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1084         apks_per_app[apk['id']] = apk
1085
1086         if not os.path.exists(apk['per_app_icons']):
1087             logging.info('Adding new repo for only ' + apk['id'])
1088             os.makedirs(apk['per_app_icons'])
1089
1090         apkpath = os.path.join(repodir, apk['apkname'])
1091         shutil.copy(apkpath, apk['per_app_repo'])
1092         apksigpath = apkpath + '.sig'
1093         if os.path.exists(apksigpath):
1094             shutil.copy(apksigpath, apk['per_app_repo'])
1095         apkascpath = apkpath + '.asc'
1096         if os.path.exists(apkascpath):
1097             shutil.copy(apkascpath, apk['per_app_repo'])
1098
1099
1100 config = None
1101 options = None
1102
1103
1104 def main():
1105
1106     global config, options
1107
1108     # Parse command line...
1109     parser = ArgumentParser()
1110     common.setup_global_opts(parser)
1111     parser.add_argument("--create-key", action="store_true", default=False,
1112                         help="Create a repo signing key in a keystore")
1113     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1114                         help="Create skeleton metadata files that are missing")
1115     parser.add_argument("--delete-unknown", action="store_true", default=False,
1116                         help="Delete APKs without metadata from the repo")
1117     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1118                         help="Report on build data status")
1119     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1120                         help="Interactively ask about things that need updating.")
1121     parser.add_argument("-I", "--icons", action="store_true", default=False,
1122                         help="Resize all the icons exceeding the max pixel size and exit")
1123     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1124                         help="Specify editor to use in interactive mode. Default " +
1125                         "is /etc/alternatives/editor")
1126     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1127                         help="Update the wiki")
1128     parser.add_argument("--pretty", action="store_true", default=False,
1129                         help="Produce human-readable index.xml")
1130     parser.add_argument("--clean", action="store_true", default=False,
1131                         help="Clean update - don't uses caches, reprocess all apks")
1132     parser.add_argument("--nosign", action="store_true", default=False,
1133                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1134     options = parser.parse_args()
1135
1136     config = common.read_config(options)
1137
1138     if not ('jarsigner' in config and 'keytool' in config):
1139         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1140         sys.exit(1)
1141
1142     repodirs = ['repo']
1143     if config['archive_older'] != 0:
1144         repodirs.append('archive')
1145         if not os.path.exists('archive'):
1146             os.mkdir('archive')
1147
1148     if options.icons:
1149         resize_all_icons(repodirs)
1150         sys.exit(0)
1151
1152     # check that icons exist now, rather than fail at the end of `fdroid update`
1153     for k in ['repo_icon', 'archive_icon']:
1154         if k in config:
1155             if not os.path.exists(config[k]):
1156                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1157                 sys.exit(1)
1158
1159     # if the user asks to create a keystore, do it now, reusing whatever it can
1160     if options.create_key:
1161         if os.path.exists(config['keystore']):
1162             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1163             logging.critical("\t'" + config['keystore'] + "'")
1164             sys.exit(1)
1165
1166         if 'repo_keyalias' not in config:
1167             config['repo_keyalias'] = socket.getfqdn()
1168             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1169         if 'keydname' not in config:
1170             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1171             common.write_to_config(config, 'keydname', config['keydname'])
1172         if 'keystore' not in config:
1173             config['keystore'] = common.default_config.keystore
1174             common.write_to_config(config, 'keystore', config['keystore'])
1175
1176         password = common.genpassword()
1177         if 'keystorepass' not in config:
1178             config['keystorepass'] = password
1179             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1180         if 'keypass' not in config:
1181             config['keypass'] = password
1182             common.write_to_config(config, 'keypass', config['keypass'])
1183         common.genkeystore(config)
1184
1185     # Get all apps...
1186     apps = metadata.read_metadata()
1187
1188     # Generate a list of categories...
1189     categories = set()
1190     for app in apps.itervalues():
1191         categories.update(app.Categories)
1192
1193     # Read known apks data (will be updated and written back when we've finished)
1194     knownapks = common.KnownApks()
1195
1196     # Gather information about all the apk files in the repo directory, using
1197     # cached data if possible.
1198     apkcachefile = os.path.join('tmp', 'apkcache')
1199     if not options.clean and os.path.exists(apkcachefile):
1200         with open(apkcachefile, 'rb') as cf:
1201             apkcache = pickle.load(cf)
1202     else:
1203         apkcache = {}
1204
1205     delete_disabled_builds(apps, apkcache, repodirs)
1206
1207     # Scan all apks in the main repo
1208     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks)
1209
1210     # Generate warnings for apk's with no metadata (or create skeleton
1211     # metadata files, if requested on the command line)
1212     newmetadata = False
1213     for apk in apks:
1214         if apk['id'] not in apps:
1215             if options.create_metadata:
1216                 if 'name' not in apk:
1217                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1218                     continue
1219                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1220                 f.write("License:Unknown\n")
1221                 f.write("Web Site:\n")
1222                 f.write("Source Code:\n")
1223                 f.write("Issue Tracker:\n")
1224                 f.write("Changelog:\n")
1225                 f.write("Summary:" + apk['name'] + "\n")
1226                 f.write("Description:\n")
1227                 f.write(apk['name'] + "\n")
1228                 f.write(".\n")
1229                 f.close()
1230                 logging.info("Generated skeleton metadata for " + apk['id'])
1231                 newmetadata = True
1232             else:
1233                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1234                 if options.delete_unknown:
1235                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1236                     rmf = os.path.join(repodirs[0], apk['apkname'])
1237                     if not os.path.exists(rmf):
1238                         logging.error("Could not find {0} to remove it".format(rmf))
1239                     else:
1240                         os.remove(rmf)
1241                 else:
1242                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1243
1244     # update the metadata with the newly created ones included
1245     if newmetadata:
1246         apps = metadata.read_metadata()
1247
1248     # Scan the archive repo for apks as well
1249     if len(repodirs) > 1:
1250         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1251         if cc:
1252             cachechanged = True
1253     else:
1254         archapks = []
1255
1256     # Some information from the apks needs to be applied up to the application
1257     # level. When doing this, we use the info from the most recent version's apk.
1258     # We deal with figuring out when the app was added and last updated at the
1259     # same time.
1260     for appid, app in apps.iteritems():
1261         bestver = 0
1262         for apk in apks + archapks:
1263             if apk['id'] == appid:
1264                 if apk['versioncode'] > bestver:
1265                     bestver = apk['versioncode']
1266                     bestapk = apk
1267
1268                 if 'added' in apk:
1269                     if not app.added or apk['added'] < app.added:
1270                         app.added = apk['added']
1271                     if not app.lastupdated or apk['added'] > app.lastupdated:
1272                         app.lastupdated = apk['added']
1273
1274         if not app.added:
1275             logging.debug("Don't know when " + appid + " was added")
1276         if not app.lastupdated:
1277             logging.debug("Don't know when " + appid + " was last updated")
1278
1279         if bestver == 0:
1280             if app.Name is None:
1281                 app.Name = app.AutoName or appid
1282             app.icon = None
1283             logging.debug("Application " + appid + " has no packages")
1284         else:
1285             if app.Name is None:
1286                 app.Name = bestapk['name']
1287             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1288             if app.CurrentVersionCode is None:
1289                 app.CurrentVersionCode = str(bestver)
1290
1291     # Sort the app list by name, then the web site doesn't have to by default.
1292     # (we had to wait until we'd scanned the apks to do this, because mostly the
1293     # name comes from there!)
1294     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid].Name.upper())
1295
1296     # APKs are placed into multiple repos based on the app package, providing
1297     # per-app subscription feeds for nightly builds and things like it
1298     if config['per_app_repos']:
1299         add_apks_to_per_app_repos(repodirs[0], apks)
1300         for appid, app in apps.iteritems():
1301             repodir = os.path.join(appid, 'fdroid', 'repo')
1302             appdict = dict()
1303             appdict[appid] = app
1304             if os.path.isdir(repodir):
1305                 make_index(appdict, [appid], apks, repodir, False, categories)
1306             else:
1307                 logging.info('Skipping index generation for ' + appid)
1308         return
1309
1310     if len(repodirs) > 1:
1311         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1312
1313     # Make the index for the main repo...
1314     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1315
1316     # If there's an archive repo,  make the index for it. We already scanned it
1317     # earlier on.
1318     if len(repodirs) > 1:
1319         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1320
1321     if config['update_stats']:
1322
1323         # Update known apks info...
1324         knownapks.writeifchanged()
1325
1326         # Generate latest apps data for widget
1327         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1328             data = ''
1329             for line in file(os.path.join('stats', 'latestapps.txt')):
1330                 appid = line.rstrip()
1331                 data += appid + "\t"
1332                 app = apps[appid]
1333                 data += app.Name + "\t"
1334                 if app.icon is not None:
1335                     data += app.icon + "\t"
1336                 data += app.License + "\n"
1337             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
1338                 f.write(data)
1339
1340     if cachechanged:
1341         with open(apkcachefile, 'wb') as cf:
1342             pickle.dump(apkcache, cf)
1343
1344     # Update the wiki...
1345     if options.wiki:
1346         update_wiki(apps, sortedids, apks + archapks)
1347
1348     logging.info("Finished.")
1349
1350 if __name__ == "__main__":
1351     main()