fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import socket
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from datetime import datetime, timedelta
  31 from xml.dom.minidom import Document
  32 from argparse import ArgumentParser
  33 import time
  34 from pyasn1.error import PyAsn1Error
  35 from pyasn1.codec.der import decoder, encoder
  36 from pyasn1_modules import rfc2315
  37 from hashlib import md5
  38 from binascii import hexlify, unhexlify
  39
  40 from PIL import Image
  41 import logging
  42
  43 import common
  44 import metadata
  45 from common import FDroidPopen, SdkToolsPopen
  46 from metadata import MetaDataException
  47
  48 screen_densities = ['640', '480', '320', '240', '160', '120']
  49
  50 all_screen_densities = ['0'] + screen_densities
  51
  52
  53 def dpi_to_px(density):
  54     return (int(density) * 48) / 160
  55
  56
  57 def px_to_dpi(px):
  58     return (int(px) * 160) / 48
  59
  60
  61 def get_icon_dir(repodir, density):
  62     if density == '0':
  63         return os.path.join(repodir, "icons")
  64     return os.path.join(repodir, "icons-%s" % density)
  65
  66
  67 def get_icon_dirs(repodir):
  68     for density in screen_densities:
  69         yield get_icon_dir(repodir, density)
  70
  71
  72 def get_all_icon_dirs(repodir):
  73     for density in all_screen_densities:
  74         yield get_icon_dir(repodir, density)
  75
  76
  77 def update_wiki(apps, sortedids, apks):
  78     """Update the wiki
  79
  80     :param apps: fully populated list of all applications
  81     :param apks: all apks, except...
  82     """
  83     logging.info("Updating wiki")
  84     wikicat = 'Apps'
  85     wikiredircat = 'App Redirects'
  86     import mwclient
  87     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  88                          path=config['wiki_path'])
  89     site.login(config['wiki_user'], config['wiki_password'])
  90     generated_pages = {}
  91     generated_redirects = {}
  92
  93     for appid in sortedids:
  94         app = apps[appid]
  95
  96         wikidata = ''
  97         if app['Disabled']:
  98             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  99         if 'AntiFeatures' in app:
 100             for af in app['AntiFeatures']:
 101                 wikidata += '{{AntiFeature|' + af + '}}\n'
 102         if app['Requires Root']:
 103             requiresroot = 'Yes'
 104         else:
 105             requiresroot = 'No'
 106         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s}}\n' % (
 107             appid,
 108             app['Name'],
 109             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
 110             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
 111             app['Source Code'],
 112             app['Issue Tracker'],
 113             app['Web Site'],
 114             app['Changelog'],
 115             app['Donate'],
 116             app['FlattrID'],
 117             app['Bitcoin'],
 118             app['Litecoin'],
 119             app['License'],
 120             requiresroot)
 121
 122         if app['Provides']:
 123             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 124
 125         wikidata += app['Summary']
 126         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 127
 128         wikidata += "=Description=\n"
 129         wikidata += metadata.description_wiki(app['Description']) + "\n"
 130
 131         wikidata += "=Maintainer Notes=\n"
 132         if 'Maintainer Notes' in app:
 133             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 134         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 135
 136         # Get a list of all packages for this application...
 137         apklist = []
 138         gotcurrentver = False
 139         cantupdate = False
 140         buildfails = False
 141         for apk in apks:
 142             if apk['id'] == appid:
 143                 if str(apk['versioncode']) == app['Current Version Code']:
 144                     gotcurrentver = True
 145                 apklist.append(apk)
 146         # Include ones we can't build, as a special case...
 147         for thisbuild in app['builds']:
 148             if thisbuild['disable']:
 149                 if thisbuild['vercode'] == app['Current Version Code']:
 150                     cantupdate = True
 151                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 152                 apklist.append({'versioncode': int(thisbuild['vercode']),
 153                                 'version': thisbuild['version'],
 154                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(thisbuild['disable']),
 155                                 })
 156             else:
 157                 builtit = False
 158                 for apk in apklist:
 159                     if apk['versioncode'] == int(thisbuild['vercode']):
 160                         builtit = True
 161                         break
 162                 if not builtit:
 163                     buildfails = True
 164                     apklist.append({'versioncode': int(thisbuild['vercode']),
 165                                     'version': thisbuild['version'],
 166                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, thisbuild['vercode']),
 167                                     })
 168         if app['Current Version Code'] == '0':
 169             cantupdate = True
 170         # Sort with most recent first...
 171         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 172
 173         wikidata += "=Versions=\n"
 174         if len(apklist) == 0:
 175             wikidata += "We currently have no versions of this app available."
 176         elif not gotcurrentver:
 177             wikidata += "We don't have the current version of this app."
 178         else:
 179             wikidata += "We have the current version of this app."
 180         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 181         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 182         if len(app['No Source Since']) > 0:
 183             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 184         if len(app['Current Version']) > 0:
 185             wikidata += "The current (recommended) version is " + app['Current Version']
 186             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 187         validapks = 0
 188         for apk in apklist:
 189             wikidata += "==" + apk['version'] + "==\n"
 190
 191             if 'buildproblem' in apk:
 192                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 193             else:
 194                 validapks += 1
 195                 wikidata += "This version is built and signed by "
 196                 if 'srcname' in apk:
 197                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 198                 else:
 199                     wikidata += "the original developer.\n\n"
 200             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 201
 202         wikidata += '\n[[Category:' + wikicat + ']]\n'
 203         if len(app['No Source Since']) > 0:
 204             wikidata += '\n[[Category:Apps missing source code]]\n'
 205         if validapks == 0 and not app['Disabled']:
 206             wikidata += '\n[[Category:Apps with no packages]]\n'
 207         if cantupdate and not app['Disabled']:
 208             wikidata += "\n[[Category:Apps we can't update]]\n"
 209         if buildfails and not app['Disabled']:
 210             wikidata += "\n[[Category:Apps with failing builds]]\n"
 211         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 212             wikidata += '\n[[Category:Apps to Update]]\n'
 213         if app['Disabled']:
 214             wikidata += '\n[[Category:Apps that are disabled]]\n'
 215         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 216             wikidata += '\n[[Category:Apps with no update check]]\n'
 217         for appcat in app['Categories']:
 218             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 219
 220         # We can't have underscores in the page name, even if they're in
 221         # the package ID, because MediaWiki messes with them...
 222         pagename = appid.replace('_', ' ')
 223
 224         # Drop a trailing newline, because mediawiki is going to drop it anyway
 225         # and it we don't we'll think the page has changed when it hasn't...
 226         if wikidata.endswith('\n'):
 227             wikidata = wikidata[:-1]
 228
 229         generated_pages[pagename] = wikidata
 230
 231         # Make a redirect from the name to the ID too, unless there's
 232         # already an existing page with the name and it isn't a redirect.
 233         noclobber = False
 234         apppagename = app['Name'].replace('_', ' ')
 235         apppagename = apppagename.replace('{', '')
 236         apppagename = apppagename.replace('}', ' ')
 237         apppagename = apppagename.replace(':', ' ')
 238         # Drop double spaces caused mostly by replacing ':' above
 239         apppagename = apppagename.replace('  ', ' ')
 240         for expagename in site.allpages(prefix=apppagename,
 241                                         filterredir='nonredirects',
 242                                         generator=False):
 243             if expagename == apppagename:
 244                 noclobber = True
 245         # Another reason not to make the redirect page is if the app name
 246         # is the same as it's ID, because that will overwrite the real page
 247         # with an redirect to itself! (Although it seems like an odd
 248         # scenario this happens a lot, e.g. where there is metadata but no
 249         # builds or binaries to extract a name from.
 250         if apppagename == pagename:
 251             noclobber = True
 252         if not noclobber:
 253             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 254
 255     for tcat, genp in [(wikicat, generated_pages),
 256                        (wikiredircat, generated_redirects)]:
 257         catpages = site.Pages['Category:' + tcat]
 258         existingpages = []
 259         for page in catpages:
 260             existingpages.append(page.name)
 261             if page.name in genp:
 262                 pagetxt = page.edit()
 263                 if pagetxt != genp[page.name]:
 264                     logging.debug("Updating modified page " + page.name)
 265                     page.save(genp[page.name], summary='Auto-updated')
 266                 else:
 267                     logging.debug("Page " + page.name + " is unchanged")
 268             else:
 269                 logging.warn("Deleting page " + page.name)
 270                 page.delete('No longer published')
 271         for pagename, text in genp.items():
 272             logging.debug("Checking " + pagename)
 273             if pagename not in existingpages:
 274                 logging.debug("Creating page " + pagename)
 275                 try:
 276                     newpage = site.Pages[pagename]
 277                     newpage.save(text, summary='Auto-created')
 278                 except:
 279                     logging.error("...FAILED to create page '{0}'".format(pagename))
 280
 281     # Purge server cache to ensure counts are up to date
 282     site.pages['Repository Maintenance'].purge()
 283
 284
 285 def delete_disabled_builds(apps, apkcache, repodirs):
 286     """Delete disabled build outputs.
 287
 288     :param apps: list of all applications, as per metadata.read_metadata
 289     :param apkcache: current apk cache information
 290     :param repodirs: the repo directories to process
 291     """
 292     for appid, app in apps.iteritems():
 293         for build in app['builds']:
 294             if not build['disable']:
 295                 continue
 296             apkfilename = appid + '_' + str(build['vercode']) + '.apk'
 297             iconfilename = "%s.%s.png" % (
 298                 appid,
 299                 build['vercode'])
 300             for repodir in repodirs:
 301                 files = [
 302                     os.path.join(repodir, apkfilename),
 303                     os.path.join(repodir, apkfilename + '.asc'),
 304                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 305                 ]
 306                 for density in all_screen_densities:
 307                     repo_dir = get_icon_dir(repodir, density)
 308                     files.append(os.path.join(repo_dir, iconfilename))
 309
 310                 for f in files:
 311                     if os.path.exists(f):
 312                         logging.info("Deleting disabled build output " + f)
 313                         os.remove(f)
 314             if apkfilename in apkcache:
 315                 del apkcache[apkfilename]
 316
 317
 318 def resize_icon(iconpath, density):
 319
 320     if not os.path.isfile(iconpath):
 321         return
 322
 323     try:
 324         im = Image.open(iconpath)
 325         size = dpi_to_px(density)
 326
 327         if any(length > size for length in im.size):
 328             oldsize = im.size
 329             im.thumbnail((size, size), Image.ANTIALIAS)
 330             logging.debug("%s was too large at %s - new size is %s" % (
 331                 iconpath, oldsize, im.size))
 332             im.save(iconpath, "PNG")
 333
 334     except Exception, e:
 335         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 336
 337
 338 def resize_all_icons(repodirs):
 339     """Resize all icons that exceed the max size
 340
 341     :param repodirs: the repo directories to process
 342     """
 343     for repodir in repodirs:
 344         for density in screen_densities:
 345             icon_dir = get_icon_dir(repodir, density)
 346             icon_glob = os.path.join(icon_dir, '*.png')
 347             for iconpath in glob.glob(icon_glob):
 348                 resize_icon(iconpath, density)
 349
 350
 351 # A signature block file with a .DSA, .RSA, or .EC extension
 352 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 353
 354
 355 def getsig(apkpath):
 356     """ Get the signing certificate of an apk. To get the same md5 has that
 357     Android gets, we encode the .RSA certificate in a specific format and pass
 358     it hex-encoded to the md5 digest algorithm.
 359
 360     :param apkpath: path to the apk
 361     :returns: A string containing the md5 of the signature of the apk or None
 362               if an error occurred.
 363     """
 364
 365     cert = None
 366
 367     # verify the jar signature is correct
 368     args = ['jarsigner', '-verify', apkpath]
 369     p = FDroidPopen(args)
 370     if p.returncode != 0:
 371         logging.critical(apkpath + " has a bad signature!")
 372         return None
 373
 374     with zipfile.ZipFile(apkpath, 'r') as apk:
 375
 376         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 377
 378         if len(certs) < 1:
 379             logging.error("Found no signing certificates on %s" % apkpath)
 380             return None
 381         if len(certs) > 1:
 382             logging.error("Found multiple signing certificates on %s" % apkpath)
 383             return None
 384
 385         cert = apk.read(certs[0])
 386
 387     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 388     if content.getComponentByName('contentType') != rfc2315.signedData:
 389         logging.error("Unexpected format.")
 390         return None
 391
 392     content = decoder.decode(content.getComponentByName('content'),
 393                              asn1Spec=rfc2315.SignedData())[0]
 394     try:
 395         certificates = content.getComponentByName('certificates')
 396     except PyAsn1Error:
 397         logging.error("Certificates not found.")
 398         return None
 399
 400     cert_encoded = encoder.encode(certificates)[4:]
 401
 402     return md5(cert_encoded.encode('hex')).hexdigest()
 403
 404
 405 def scan_apks(apps, apkcache, repodir, knownapks):
 406     """Scan the apks in the given repo directory.
 407
 408     This also extracts the icons.
 409
 410     :param apps: list of all applications, as per metadata.read_metadata
 411     :param apkcache: current apk cache information
 412     :param repodir: repo directory to scan
 413     :param knownapks: known apks info
 414     :returns: (apks, cachechanged) where apks is a list of apk information,
 415               and cachechanged is True if the apkcache got changed.
 416     """
 417
 418     cachechanged = False
 419
 420     for icon_dir in get_all_icon_dirs(repodir):
 421         if os.path.exists(icon_dir):
 422             if options.clean:
 423                 shutil.rmtree(icon_dir)
 424                 os.makedirs(icon_dir)
 425         else:
 426             os.makedirs(icon_dir)
 427
 428     apks = []
 429     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 430     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 431     vername_pat = re.compile(".*versionName='([^']*)'.*")
 432     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 433     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 434     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 435     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 436     string_pat = re.compile(".*'([^']*)'.*")
 437     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 438
 439         apkfilename = apkfile[len(repodir) + 1:]
 440         if ' ' in apkfilename:
 441             logging.critical("Spaces in filenames are not allowed.")
 442             sys.exit(1)
 443
 444         # Calculate the sha256...
 445         sha = hashlib.sha256()
 446         with open(apkfile, 'rb') as f:
 447             while True:
 448                 t = f.read(16384)
 449                 if len(t) == 0:
 450                     break
 451                 sha.update(t)
 452             shasum = sha.hexdigest()
 453
 454         usecache = False
 455         if apkfilename in apkcache:
 456             thisinfo = apkcache[apkfilename]
 457             if thisinfo['sha256'] == shasum:
 458                 logging.debug("Reading " + apkfilename + " from cache")
 459                 usecache = True
 460             else:
 461                 logging.debug("Ignoring stale cache data for " + apkfilename)
 462
 463         if not usecache:
 464             logging.debug("Processing " + apkfilename)
 465             thisinfo = {}
 466             thisinfo['apkname'] = apkfilename
 467             thisinfo['sha256'] = shasum
 468             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 469             if os.path.exists(os.path.join(repodir, srcfilename)):
 470                 thisinfo['srcname'] = srcfilename
 471             thisinfo['size'] = os.path.getsize(apkfile)
 472             thisinfo['permissions'] = set()
 473             thisinfo['features'] = set()
 474             thisinfo['icons_src'] = {}
 475             thisinfo['icons'] = {}
 476             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 477             if p.returncode != 0:
 478                 if options.delete_unknown:
 479                     if os.path.exists(apkfile):
 480                         logging.error("Failed to get apk information, deleting " + apkfile)
 481                         os.remove(apkfile)
 482                     else:
 483                         logging.error("Could not find {0} to remove it".format(apkfile))
 484                 else:
 485                     logging.error("Failed to get apk information, skipping " + apkfile)
 486                 continue
 487             for line in p.output.splitlines():
 488                 if line.startswith("package:"):
 489                     try:
 490                         thisinfo['id'] = re.match(name_pat, line).group(1)
 491                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 492                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 493                     except Exception, e:
 494                         logging.error("Package matching failed: " + str(e))
 495                         logging.info("Line was: " + line)
 496                         sys.exit(1)
 497                 elif line.startswith("application:"):
 498                     thisinfo['name'] = re.match(label_pat, line).group(1)
 499                     # Keep path to non-dpi icon in case we need it
 500                     match = re.match(icon_pat_nodpi, line)
 501                     if match:
 502                         thisinfo['icons_src']['-1'] = match.group(1)
 503                 elif line.startswith("launchable-activity:"):
 504                     # Only use launchable-activity as fallback to application
 505                     if not thisinfo['name']:
 506                         thisinfo['name'] = re.match(label_pat, line).group(1)
 507                     if '-1' not in thisinfo['icons_src']:
 508                         match = re.match(icon_pat_nodpi, line)
 509                         if match:
 510                             thisinfo['icons_src']['-1'] = match.group(1)
 511                 elif line.startswith("application-icon-"):
 512                     match = re.match(icon_pat, line)
 513                     if match:
 514                         density = match.group(1)
 515                         path = match.group(2)
 516                         thisinfo['icons_src'][density] = path
 517                 elif line.startswith("sdkVersion:"):
 518                     m = re.match(sdkversion_pat, line)
 519                     if m is None:
 520                         logging.error(line.replace('sdkVersion:', '')
 521                                       + ' is not a valid minSdkVersion!')
 522                     else:
 523                         thisinfo['sdkversion'] = m.group(1)
 524                 elif line.startswith("maxSdkVersion:"):
 525                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 526                 elif line.startswith("native-code:"):
 527                     thisinfo['nativecode'] = []
 528                     for arch in line[13:].split(' '):
 529                         thisinfo['nativecode'].append(arch[1:-1])
 530                 elif line.startswith("uses-permission:"):
 531                     perm = re.match(string_pat, line).group(1)
 532                     if perm.startswith("android.permission."):
 533                         perm = perm[19:]
 534                     thisinfo['permissions'].add(perm)
 535                 elif line.startswith("uses-feature:"):
 536                     perm = re.match(string_pat, line).group(1)
 537                     # Filter out this, it's only added with the latest SDK tools and
 538                     # causes problems for lots of apps.
 539                     if perm != "android.hardware.screen.portrait" \
 540                             and perm != "android.hardware.screen.landscape":
 541                         if perm.startswith("android.feature."):
 542                             perm = perm[16:]
 543                         thisinfo['features'].add(perm)
 544
 545             if 'sdkversion' not in thisinfo:
 546                 logging.warn("No SDK version information found in {0}".format(apkfile))
 547                 thisinfo['sdkversion'] = 0
 548
 549             # Check for debuggable apks...
 550             if common.isApkDebuggable(apkfile, config):
 551                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 552
 553             # Get the signature (or md5 of, to be precise)...
 554             logging.debug('Getting signature of {0}'.format(apkfile))
 555             thisinfo['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 556             if not thisinfo['sig']:
 557                 logging.critical("Failed to get apk signature")
 558                 sys.exit(1)
 559
 560             apk = zipfile.ZipFile(apkfile, 'r')
 561
 562             # if an APK has files newer than the system time, suggest updating
 563             # the system clock.  This is useful for offline systems, used for
 564             # signing, which do not have another source of clock sync info. It
 565             # has to be more than 24 hours newer because ZIP/APK files do not
 566             # store timezone info
 567             info = apk.getinfo('AndroidManifest.xml')
 568             dt_obj = datetime(*info.date_time)
 569             checkdt = dt_obj - timedelta(1)
 570             if datetime.today() < checkdt:
 571                 logging.warn('System clock is older than manifest in: '
 572                              + apkfilename + '\nSet clock to that time using:\n'
 573                              + 'sudo date -s "' + str(dt_obj) + '"')
 574
 575             iconfilename = "%s.%s.png" % (
 576                 thisinfo['id'],
 577                 thisinfo['versioncode'])
 578
 579             # Extract the icon file...
 580             empty_densities = []
 581             for density in screen_densities:
 582                 if density not in thisinfo['icons_src']:
 583                     empty_densities.append(density)
 584                     continue
 585                 iconsrc = thisinfo['icons_src'][density]
 586                 icon_dir = get_icon_dir(repodir, density)
 587                 icondest = os.path.join(icon_dir, iconfilename)
 588
 589                 try:
 590                     with open(icondest, 'wb') as f:
 591                         f.write(apk.read(iconsrc))
 592                     thisinfo['icons'][density] = iconfilename
 593
 594                 except:
 595                     logging.warn("Error retrieving icon file")
 596                     del thisinfo['icons'][density]
 597                     del thisinfo['icons_src'][density]
 598                     empty_densities.append(density)
 599
 600             if '-1' in thisinfo['icons_src']:
 601                 iconsrc = thisinfo['icons_src']['-1']
 602                 iconpath = os.path.join(
 603                     get_icon_dir(repodir, '0'), iconfilename)
 604                 with open(iconpath, 'wb') as f:
 605                     f.write(apk.read(iconsrc))
 606                 try:
 607                     im = Image.open(iconpath)
 608                     dpi = px_to_dpi(im.size[0])
 609                     for density in screen_densities:
 610                         if density in thisinfo['icons']:
 611                             break
 612                         if density == screen_densities[-1] or dpi >= int(density):
 613                             thisinfo['icons'][density] = iconfilename
 614                             shutil.move(iconpath,
 615                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 616                             empty_densities.remove(density)
 617                             break
 618                 except Exception, e:
 619                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 620
 621             if thisinfo['icons']:
 622                 thisinfo['icon'] = iconfilename
 623
 624             apk.close()
 625
 626             # First try resizing down to not lose quality
 627             last_density = None
 628             for density in screen_densities:
 629                 if density not in empty_densities:
 630                     last_density = density
 631                     continue
 632                 if last_density is None:
 633                     continue
 634                 logging.debug("Density %s not available, resizing down from %s"
 635                               % (density, last_density))
 636
 637                 last_iconpath = os.path.join(
 638                     get_icon_dir(repodir, last_density), iconfilename)
 639                 iconpath = os.path.join(
 640                     get_icon_dir(repodir, density), iconfilename)
 641                 try:
 642                     im = Image.open(last_iconpath)
 643                 except:
 644                     logging.warn("Invalid image file at %s" % last_iconpath)
 645                     continue
 646
 647                 size = dpi_to_px(density)
 648
 649                 im.thumbnail((size, size), Image.ANTIALIAS)
 650                 im.save(iconpath, "PNG")
 651                 empty_densities.remove(density)
 652
 653             # Then just copy from the highest resolution available
 654             last_density = None
 655             for density in reversed(screen_densities):
 656                 if density not in empty_densities:
 657                     last_density = density
 658                     continue
 659                 if last_density is None:
 660                     continue
 661                 logging.debug("Density %s not available, copying from lower density %s"
 662                               % (density, last_density))
 663
 664                 shutil.copyfile(
 665                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 666                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 667
 668                 empty_densities.remove(density)
 669
 670             for density in screen_densities:
 671                 icon_dir = get_icon_dir(repodir, density)
 672                 icondest = os.path.join(icon_dir, iconfilename)
 673                 resize_icon(icondest, density)
 674
 675             # Copy from icons-mdpi to icons since mdpi is the baseline density
 676             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 677             if os.path.isfile(baseline):
 678                 thisinfo['icons']['0'] = iconfilename
 679                 shutil.copyfile(baseline,
 680                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 681
 682             # Record in known apks, getting the added date at the same time..
 683             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 684             if added:
 685                 thisinfo['added'] = added
 686
 687             apkcache[apkfilename] = thisinfo
 688             cachechanged = True
 689
 690         apks.append(thisinfo)
 691
 692     return apks, cachechanged
 693
 694
 695 repo_pubkey_fingerprint = None
 696
 697
 698 # Generate a certificate fingerprint the same way keytool does it
 699 # (but with slightly different formatting)
 700 def cert_fingerprint(data):
 701     digest = hashlib.sha256(data).digest()
 702     ret = []
 703     ret.append(' '.join("%02X" % ord(b) for b in digest))
 704     return " ".join(ret)
 705
 706
 707 def extract_pubkey():
 708     global repo_pubkey_fingerprint
 709     if 'repo_pubkey' in config:
 710         pubkey = unhexlify(config['repo_pubkey'])
 711     else:
 712         p = FDroidPopen(['keytool', '-exportcert',
 713                          '-alias', config['repo_keyalias'],
 714                          '-keystore', config['keystore'],
 715                          '-storepass:file', config['keystorepassfile']]
 716                         + config['smartcardoptions'], output=False)
 717         if p.returncode != 0 or len(p.output) < 20:
 718             msg = "Failed to get repo pubkey!"
 719             if config['keystore'] == 'NONE':
 720                 msg += ' Is your crypto smartcard plugged in?'
 721             logging.critical(msg)
 722             sys.exit(1)
 723         pubkey = p.output
 724     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 725     return hexlify(pubkey)
 726
 727
 728 def make_index(apps, sortedids, apks, repodir, archive, categories):
 729     """Make a repo index.
 730
 731     :param apps: fully populated apps list
 732     :param apks: full populated apks list
 733     :param repodir: the repo directory
 734     :param archive: True if this is the archive repo, False if it's the
 735                     main one.
 736     :param categories: list of categories
 737     """
 738
 739     doc = Document()
 740
 741     def addElement(name, value, doc, parent):
 742         el = doc.createElement(name)
 743         el.appendChild(doc.createTextNode(value))
 744         parent.appendChild(el)
 745
 746     def addElementNonEmpty(name, value, doc, parent):
 747         if not value:
 748             return
 749         addElement(name, value, doc, parent)
 750
 751     def addElementCDATA(name, value, doc, parent):
 752         el = doc.createElement(name)
 753         el.appendChild(doc.createCDATASection(value))
 754         parent.appendChild(el)
 755
 756     root = doc.createElement("fdroid")
 757     doc.appendChild(root)
 758
 759     repoel = doc.createElement("repo")
 760
 761     if archive:
 762         repoel.setAttribute("name", config['archive_name'])
 763         if config['repo_maxage'] != 0:
 764             repoel.setAttribute("maxage", str(config['repo_maxage']))
 765         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 766         repoel.setAttribute("url", config['archive_url'])
 767         addElement('description', config['archive_description'], doc, repoel)
 768
 769     else:
 770         repoel.setAttribute("name", config['repo_name'])
 771         if config['repo_maxage'] != 0:
 772             repoel.setAttribute("maxage", str(config['repo_maxage']))
 773         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 774         repoel.setAttribute("url", config['repo_url'])
 775         addElement('description', config['repo_description'], doc, repoel)
 776
 777     repoel.setAttribute("version", "14")
 778     repoel.setAttribute("timestamp", str(int(time.time())))
 779
 780     nosigningkey = False
 781     if not options.nosign:
 782         if 'repo_keyalias' not in config:
 783             nosigningkey = True
 784             logging.critical("'repo_keyalias' not found in config.py!")
 785         if 'keystore' not in config:
 786             nosigningkey = True
 787             logging.critical("'keystore' not found in config.py!")
 788         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 789             nosigningkey = True
 790             logging.critical("'keystorepass' not found in config.py!")
 791         if 'keypass' not in config and 'keypassfile' not in config:
 792             nosigningkey = True
 793             logging.critical("'keypass' not found in config.py!")
 794         if not os.path.exists(config['keystore']):
 795             nosigningkey = True
 796             logging.critical("'" + config['keystore'] + "' does not exist!")
 797         if nosigningkey:
 798             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 799             logging.warning("\tfdroid update --create-key")
 800             sys.exit(1)
 801
 802     repoel.setAttribute("pubkey", extract_pubkey())
 803     root.appendChild(repoel)
 804
 805     for appid in sortedids:
 806         app = apps[appid]
 807
 808         if app['Disabled'] is not None:
 809             continue
 810
 811         # Get a list of the apks for this app...
 812         apklist = []
 813         for apk in apks:
 814             if apk['id'] == appid:
 815                 apklist.append(apk)
 816
 817         if len(apklist) == 0:
 818             continue
 819
 820         apel = doc.createElement("application")
 821         apel.setAttribute("id", app['id'])
 822         root.appendChild(apel)
 823
 824         addElement('id', app['id'], doc, apel)
 825         if 'added' in app:
 826             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 827         if 'lastupdated' in app:
 828             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 829         addElement('name', app['Name'], doc, apel)
 830         addElement('summary', app['Summary'], doc, apel)
 831         if app['icon']:
 832             addElement('icon', app['icon'], doc, apel)
 833
 834         def linkres(appid):
 835             if appid in apps:
 836                 return ("fdroid.app:" + appid, apps[appid]['Name'])
 837             raise MetaDataException("Cannot resolve app id " + appid)
 838
 839         addElement('desc',
 840                    metadata.description_html(app['Description'], linkres),
 841                    doc, apel)
 842         addElement('license', app['License'], doc, apel)
 843         if 'Categories' in app and app['Categories']:
 844             addElement('categories', ','.join(app["Categories"]), doc, apel)
 845             # We put the first (primary) category in LAST, which will have
 846             # the desired effect of making clients that only understand one
 847             # category see that one.
 848             addElement('category', app["Categories"][0], doc, apel)
 849         addElement('web', app['Web Site'], doc, apel)
 850         addElement('source', app['Source Code'], doc, apel)
 851         addElement('tracker', app['Issue Tracker'], doc, apel)
 852         addElementNonEmpty('changelog', app['Changelog'], doc, apel)
 853         addElementNonEmpty('donate', app['Donate'], doc, apel)
 854         addElementNonEmpty('bitcoin', app['Bitcoin'], doc, apel)
 855         addElementNonEmpty('litecoin', app['Litecoin'], doc, apel)
 856         addElementNonEmpty('flattr', app['FlattrID'], doc, apel)
 857
 858         # These elements actually refer to the current version (i.e. which
 859         # one is recommended. They are historically mis-named, and need
 860         # changing, but stay like this for now to support existing clients.
 861         addElement('marketversion', app['Current Version'], doc, apel)
 862         addElement('marketvercode', app['Current Version Code'], doc, apel)
 863
 864         if app['AntiFeatures']:
 865             af = app['AntiFeatures']
 866             if af:
 867                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 868         if app['Provides']:
 869             pv = app['Provides'].split(',')
 870             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 871         if app['Requires Root']:
 872             addElement('requirements', 'root', doc, apel)
 873
 874         # Sort the apk list into version order, just so the web site
 875         # doesn't have to do any work by default...
 876         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 877
 878         # Check for duplicates - they will make the client unhappy...
 879         for i in range(len(apklist) - 1):
 880             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 881                 logging.critical("duplicate versions: '%s' - '%s'" % (
 882                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 883                 sys.exit(1)
 884
 885         current_version_code = 0
 886         current_version_file = None
 887         for apk in apklist:
 888             # find the APK for the "Current Version"
 889             if current_version_code < apk['versioncode']:
 890                 current_version_code = apk['versioncode']
 891             if current_version_code < int(app['Current Version Code']):
 892                 current_version_file = apk['apkname']
 893
 894             apkel = doc.createElement("package")
 895             apel.appendChild(apkel)
 896             addElement('version', apk['version'], doc, apkel)
 897             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 898             addElement('apkname', apk['apkname'], doc, apkel)
 899             if 'srcname' in apk:
 900                 addElement('srcname', apk['srcname'], doc, apkel)
 901             for hash_type in ['sha256']:
 902                 if hash_type not in apk:
 903                     continue
 904                 hashel = doc.createElement("hash")
 905                 hashel.setAttribute("type", hash_type)
 906                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 907                 apkel.appendChild(hashel)
 908             addElement('sig', apk['sig'], doc, apkel)
 909             addElement('size', str(apk['size']), doc, apkel)
 910             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 911             if 'maxsdkversion' in apk:
 912                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 913             if 'added' in apk:
 914                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 915             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
 916             if 'nativecode' in apk:
 917                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 918             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
 919
 920         if current_version_file is not None \
 921                 and config['make_current_version_link'] \
 922                 and repodir == 'repo':  # only create these
 923             sanitized_name = re.sub('''[ '"&%?+=/]''', '',
 924                                     app[config['current_version_name_source']])
 925             apklinkname = sanitized_name + '.apk'
 926             current_version_path = os.path.join(repodir, current_version_file)
 927             if os.path.islink(apklinkname):
 928                 os.remove(apklinkname)
 929             os.symlink(current_version_path, apklinkname)
 930             # also symlink gpg signature, if it exists
 931             for extension in ('.asc', '.sig'):
 932                 sigfile_path = current_version_path + extension
 933                 if os.path.exists(sigfile_path):
 934                     siglinkname = apklinkname + extension
 935                     if os.path.islink(siglinkname):
 936                         os.remove(siglinkname)
 937                     os.symlink(sigfile_path, siglinkname)
 938
 939     if options.pretty:
 940         output = doc.toprettyxml()
 941     else:
 942         output = doc.toxml()
 943
 944     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 945         f.write(output)
 946
 947     if 'repo_keyalias' in config:
 948
 949         if options.nosign:
 950             logging.info("Creating unsigned index in preparation for signing")
 951         else:
 952             logging.info("Creating signed index with this key (SHA256):")
 953             logging.info("%s" % repo_pubkey_fingerprint)
 954
 955         # Create a jar of the index...
 956         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
 957         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 958         if p.returncode != 0:
 959             logging.critical("Failed to create {0}".format(jar_output))
 960             sys.exit(1)
 961
 962         # Sign the index...
 963         signed = os.path.join(repodir, 'index.jar')
 964         if options.nosign:
 965             # Remove old signed index if not signing
 966             if os.path.exists(signed):
 967                 os.remove(signed)
 968         else:
 969             args = ['jarsigner', '-keystore', config['keystore'],
 970                     '-storepass:file', config['keystorepassfile'],
 971                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
 972                     signed, config['repo_keyalias']]
 973             if config['keystore'] == 'NONE':
 974                 args += config['smartcardoptions']
 975             else:  # smardcards never use -keypass
 976                 args += ['-keypass:file', config['keypassfile']]
 977             p = FDroidPopen(args)
 978             if p.returncode != 0:
 979                 logging.critical("Failed to sign index")
 980                 sys.exit(1)
 981
 982     # Copy the repo icon into the repo directory...
 983     icon_dir = os.path.join(repodir, 'icons')
 984     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 985     shutil.copyfile(config['repo_icon'], iconfilename)
 986
 987     # Write a category list in the repo to allow quick access...
 988     catdata = ''
 989     for cat in categories:
 990         catdata += cat + '\n'
 991     with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
 992         f.write(catdata)
 993
 994
 995 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 996
 997     for appid, app in apps.iteritems():
 998
 999         if app['Archive Policy']:
1000             keepversions = int(app['Archive Policy'][:-9])
1001         else:
1002             keepversions = defaultkeepversions
1003
1004         def filter_apk_list_sorted(apk_list):
1005             res = []
1006             for apk in apk_list:
1007                 if apk['id'] == appid:
1008                     res.append(apk)
1009
1010             # Sort the apk list by version code. First is highest/newest.
1011             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1012
1013         def move_file(from_dir, to_dir, filename, ignore_missing):
1014             from_path = os.path.join(from_dir, filename)
1015             if ignore_missing and not os.path.exists(from_path):
1016                 return
1017             to_path = os.path.join(to_dir, filename)
1018             shutil.move(from_path, to_path)
1019
1020         if len(apks) > keepversions:
1021             apklist = filter_apk_list_sorted(apks)
1022             # Move back the ones we don't want.
1023             for apk in apklist[keepversions:]:
1024                 logging.info("Moving " + apk['apkname'] + " to archive")
1025                 move_file(repodir, archivedir, apk['apkname'], False)
1026                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1027                 for density in all_screen_densities:
1028                     repo_icon_dir = get_icon_dir(repodir, density)
1029                     archive_icon_dir = get_icon_dir(archivedir, density)
1030                     if density not in apk['icons']:
1031                         continue
1032                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1033                 if 'srcname' in apk:
1034                     move_file(repodir, archivedir, apk['srcname'], False)
1035                 archapks.append(apk)
1036                 apks.remove(apk)
1037         elif len(apks) < keepversions and len(archapks) > 0:
1038             required = keepversions - len(apks)
1039             archapklist = filter_apk_list_sorted(archapks)
1040             # Move forward the ones we want again.
1041             for apk in archapklist[:required]:
1042                 logging.info("Moving " + apk['apkname'] + " from archive")
1043                 move_file(archivedir, repodir, apk['apkname'], False)
1044                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1045                 for density in all_screen_densities:
1046                     repo_icon_dir = get_icon_dir(repodir, density)
1047                     archive_icon_dir = get_icon_dir(archivedir, density)
1048                     if density not in apk['icons']:
1049                         continue
1050                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1051                 if 'srcname' in apk:
1052                     move_file(archivedir, repodir, apk['srcname'], False)
1053                 archapks.remove(apk)
1054                 apks.append(apk)
1055
1056
1057 def add_apks_to_per_app_repos(repodir, apks):
1058     apks_per_app = dict()
1059     for apk in apks:
1060         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1061         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1062         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1063         apks_per_app[apk['id']] = apk
1064
1065         if not os.path.exists(apk['per_app_icons']):
1066             logging.info('Adding new repo for only ' + apk['id'])
1067             os.makedirs(apk['per_app_icons'])
1068
1069         apkpath = os.path.join(repodir, apk['apkname'])
1070         shutil.copy(apkpath, apk['per_app_repo'])
1071         apksigpath = apkpath + '.sig'
1072         if os.path.exists(apksigpath):
1073             shutil.copy(apksigpath, apk['per_app_repo'])
1074         apkascpath = apkpath + '.asc'
1075         if os.path.exists(apkascpath):
1076             shutil.copy(apkascpath, apk['per_app_repo'])
1077
1078
1079 config = None
1080 options = None
1081
1082
1083 def main():
1084
1085     global config, options
1086
1087     # Parse command line...
1088     parser = ArgumentParser()
1089     common.setup_global_opts(parser)
1090     parser.add_argument("--create-key", action="store_true", default=False,
1091                         help="Create a repo signing key in a keystore")
1092     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1093                         help="Create skeleton metadata files that are missing")
1094     parser.add_argument("--delete-unknown", action="store_true", default=False,
1095                         help="Delete APKs without metadata from the repo")
1096     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1097                         help="Report on build data status")
1098     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1099                         help="Interactively ask about things that need updating.")
1100     parser.add_argument("-I", "--icons", action="store_true", default=False,
1101                         help="Resize all the icons exceeding the max pixel size and exit")
1102     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1103                         help="Specify editor to use in interactive mode. Default " +
1104                         "is /etc/alternatives/editor")
1105     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1106                         help="Update the wiki")
1107     parser.add_argument("--pretty", action="store_true", default=False,
1108                         help="Produce human-readable index.xml")
1109     parser.add_argument("--clean", action="store_true", default=False,
1110                         help="Clean update - don't uses caches, reprocess all apks")
1111     parser.add_argument("--nosign", action="store_true", default=False,
1112                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1113     options = parser.parse_args()
1114
1115     config = common.read_config(options)
1116
1117     repodirs = ['repo']
1118     if config['archive_older'] != 0:
1119         repodirs.append('archive')
1120         if not os.path.exists('archive'):
1121             os.mkdir('archive')
1122
1123     if options.icons:
1124         resize_all_icons(repodirs)
1125         sys.exit(0)
1126
1127     # check that icons exist now, rather than fail at the end of `fdroid update`
1128     for k in ['repo_icon', 'archive_icon']:
1129         if k in config:
1130             if not os.path.exists(config[k]):
1131                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1132                 sys.exit(1)
1133
1134     # if the user asks to create a keystore, do it now, reusing whatever it can
1135     if options.create_key:
1136         if os.path.exists(config['keystore']):
1137             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1138             logging.critical("\t'" + config['keystore'] + "'")
1139             sys.exit(1)
1140
1141         if 'repo_keyalias' not in config:
1142             config['repo_keyalias'] = socket.getfqdn()
1143             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1144         if 'keydname' not in config:
1145             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1146             common.write_to_config(config, 'keydname', config['keydname'])
1147         if 'keystore' not in config:
1148             config['keystore'] = common.default_config.keystore
1149             common.write_to_config(config, 'keystore', config['keystore'])
1150
1151         password = common.genpassword()
1152         if 'keystorepass' not in config:
1153             config['keystorepass'] = password
1154             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1155         if 'keypass' not in config:
1156             config['keypass'] = password
1157             common.write_to_config(config, 'keypass', config['keypass'])
1158         common.genkeystore(config)
1159
1160     # Get all apps...
1161     apps = metadata.read_metadata()
1162
1163     # Generate a list of categories...
1164     categories = set()
1165     for app in apps.itervalues():
1166         categories.update(app['Categories'])
1167
1168     # Read known apks data (will be updated and written back when we've finished)
1169     knownapks = common.KnownApks()
1170
1171     # Gather information about all the apk files in the repo directory, using
1172     # cached data if possible.
1173     apkcachefile = os.path.join('tmp', 'apkcache')
1174     if not options.clean and os.path.exists(apkcachefile):
1175         with open(apkcachefile, 'rb') as cf:
1176             apkcache = pickle.load(cf)
1177     else:
1178         apkcache = {}
1179
1180     delete_disabled_builds(apps, apkcache, repodirs)
1181
1182     # Scan all apks in the main repo
1183     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks)
1184
1185     # Generate warnings for apk's with no metadata (or create skeleton
1186     # metadata files, if requested on the command line)
1187     newmetadata = False
1188     for apk in apks:
1189         if apk['id'] not in apps:
1190             if options.create_metadata:
1191                 if 'name' not in apk:
1192                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1193                     continue
1194                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1195                 f.write("License:Unknown\n")
1196                 f.write("Web Site:\n")
1197                 f.write("Source Code:\n")
1198                 f.write("Issue Tracker:\n")
1199                 f.write("Changelog:\n")
1200                 f.write("Summary:" + apk['name'] + "\n")
1201                 f.write("Description:\n")
1202                 f.write(apk['name'] + "\n")
1203                 f.write(".\n")
1204                 f.close()
1205                 logging.info("Generated skeleton metadata for " + apk['id'])
1206                 newmetadata = True
1207             else:
1208                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1209                 if options.delete_unknown:
1210                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1211                     rmf = os.path.join(repodirs[0], apk['apkname'])
1212                     if not os.path.exists(rmf):
1213                         logging.error("Could not find {0} to remove it".format(rmf))
1214                     else:
1215                         os.remove(rmf)
1216                 else:
1217                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1218
1219     # update the metadata with the newly created ones included
1220     if newmetadata:
1221         apps = metadata.read_metadata()
1222
1223     # Scan the archive repo for apks as well
1224     if len(repodirs) > 1:
1225         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1226         if cc:
1227             cachechanged = True
1228     else:
1229         archapks = []
1230
1231     # Some information from the apks needs to be applied up to the application
1232     # level. When doing this, we use the info from the most recent version's apk.
1233     # We deal with figuring out when the app was added and last updated at the
1234     # same time.
1235     for appid, app in apps.iteritems():
1236         bestver = 0
1237         added = None
1238         lastupdated = None
1239         for apk in apks + archapks:
1240             if apk['id'] == appid:
1241                 if apk['versioncode'] > bestver:
1242                     bestver = apk['versioncode']
1243                     bestapk = apk
1244
1245                 if 'added' in apk:
1246                     if not added or apk['added'] < added:
1247                         added = apk['added']
1248                     if not lastupdated or apk['added'] > lastupdated:
1249                         lastupdated = apk['added']
1250
1251         if added:
1252             app['added'] = added
1253         else:
1254             logging.debug("Don't know when " + appid + " was added")
1255         if lastupdated:
1256             app['lastupdated'] = lastupdated
1257         else:
1258             logging.debug("Don't know when " + appid + " was last updated")
1259
1260         if bestver == 0:
1261             if app['Name'] is None:
1262                 app['Name'] = app['Auto Name'] or appid
1263             app['icon'] = None
1264             logging.debug("Application " + appid + " has no packages")
1265         else:
1266             if app['Name'] is None:
1267                 app['Name'] = bestapk['name']
1268             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1269
1270     # Sort the app list by name, then the web site doesn't have to by default.
1271     # (we had to wait until we'd scanned the apks to do this, because mostly the
1272     # name comes from there!)
1273     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid]['Name'].upper())
1274
1275     # APKs are placed into multiple repos based on the app package, providing
1276     # per-app subscription feeds for nightly builds and things like it
1277     if config['per_app_repos']:
1278         add_apks_to_per_app_repos(repodirs[0], apks)
1279         for appid, app in apps.iteritems():
1280             repodir = os.path.join(appid, 'fdroid', 'repo')
1281             appdict = dict()
1282             appdict[appid] = app
1283             if os.path.isdir(repodir):
1284                 make_index(appdict, [appid], apks, repodir, False, categories)
1285             else:
1286                 logging.info('Skipping index generation for ' + appid)
1287         return
1288
1289     if len(repodirs) > 1:
1290         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1291
1292     # Make the index for the main repo...
1293     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1294
1295     # If there's an archive repo,  make the index for it. We already scanned it
1296     # earlier on.
1297     if len(repodirs) > 1:
1298         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1299
1300     if config['update_stats']:
1301
1302         # Update known apks info...
1303         knownapks.writeifchanged()
1304
1305         # Generate latest apps data for widget
1306         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1307             data = ''
1308             for line in file(os.path.join('stats', 'latestapps.txt')):
1309                 appid = line.rstrip()
1310                 data += appid + "\t"
1311                 app = apps[appid]
1312                 data += app['Name'] + "\t"
1313                 if app['icon'] is not None:
1314                     data += app['icon'] + "\t"
1315                 data += app['License'] + "\n"
1316             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
1317                 f.write(data)
1318
1319     if cachechanged:
1320         with open(apkcachefile, 'wb') as cf:
1321             pickle.dump(apkcache, cf)
1322
1323     # Update the wiki...
1324     if options.wiki:
1325         update_wiki(apps, sortedids, apks + archapks)
1326
1327     logging.info("Finished.")
1328
1329 if __name__ == "__main__":
1330     main()