fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import socket
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from xml.dom.minidom import Document
  31 from optparse import OptionParser
  32 import time
  33 from pyasn1.error import PyAsn1Error
  34 from pyasn1.codec.der import decoder, encoder
  35 from pyasn1_modules import rfc2315
  36 from hashlib import md5
  37 from binascii import hexlify, unhexlify
  38
  39 from PIL import Image
  40 import logging
  41
  42 import common
  43 import metadata
  44 from common import FDroidPopen, SdkToolsPopen
  45 from metadata import MetaDataException
  46
  47
  48 def get_densities():
  49     return ['640', '480', '320', '240', '160', '120']
  50
  51
  52 def dpi_to_px(density):
  53     return (int(density) * 48) / 160
  54
  55
  56 def px_to_dpi(px):
  57     return (int(px) * 160) / 48
  58
  59
  60 def get_icon_dir(repodir, density):
  61     if density is None:
  62         return os.path.join(repodir, "icons")
  63     return os.path.join(repodir, "icons-%s" % density)
  64
  65
  66 def get_icon_dirs(repodir):
  67     for density in get_densities():
  68         yield get_icon_dir(repodir, density)
  69     yield os.path.join(repodir, "icons")
  70
  71
  72 def update_wiki(apps, sortedids, apks):
  73     """Update the wiki
  74
  75     :param apps: fully populated list of all applications
  76     :param apks: all apks, except...
  77     """
  78     logging.info("Updating wiki")
  79     wikicat = 'Apps'
  80     wikiredircat = 'App Redirects'
  81     import mwclient
  82     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  83                          path=config['wiki_path'])
  84     site.login(config['wiki_user'], config['wiki_password'])
  85     generated_pages = {}
  86     generated_redirects = {}
  87
  88     for appid in sortedids:
  89         app = apps[appid]
  90
  91         wikidata = ''
  92         if app['Disabled']:
  93             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  94         if app['AntiFeatures']:
  95             for af in app['AntiFeatures'].split(','):
  96                 wikidata += '{{AntiFeature|' + af + '}}\n'
  97         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
  98             appid,
  99             app['Name'],
 100             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
 101             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
 102             app['Source Code'],
 103             app['Issue Tracker'],
 104             app['Web Site'],
 105             app['Changelog'],
 106             app['Donate'],
 107             app['FlattrID'],
 108             app['Bitcoin'],
 109             app['Litecoin'],
 110             app['Dogecoin'],
 111             app['License'],
 112             app.get('Requires Root', 'No'))
 113
 114         if app['Provides']:
 115             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 116
 117         wikidata += app['Summary']
 118         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 119
 120         wikidata += "=Description=\n"
 121         wikidata += metadata.description_wiki(app['Description']) + "\n"
 122
 123         wikidata += "=Maintainer Notes=\n"
 124         if 'Maintainer Notes' in app:
 125             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 126         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 127
 128         # Get a list of all packages for this application...
 129         apklist = []
 130         gotcurrentver = False
 131         cantupdate = False
 132         buildfails = False
 133         for apk in apks:
 134             if apk['id'] == appid:
 135                 if str(apk['versioncode']) == app['Current Version Code']:
 136                     gotcurrentver = True
 137                 apklist.append(apk)
 138         # Include ones we can't build, as a special case...
 139         for thisbuild in app['builds']:
 140             if thisbuild['disable']:
 141                 if thisbuild['vercode'] == app['Current Version Code']:
 142                     cantupdate = True
 143                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 144                 apklist.append({'versioncode': int(thisbuild['vercode']),
 145                                 'version': thisbuild['version'],
 146                                 'buildproblem': thisbuild['disable']
 147                                 })
 148             else:
 149                 builtit = False
 150                 for apk in apklist:
 151                     if apk['versioncode'] == int(thisbuild['vercode']):
 152                         builtit = True
 153                         break
 154                 if not builtit:
 155                     buildfails = True
 156                     apklist.append({'versioncode': int(thisbuild['vercode']),
 157                                     'version': thisbuild['version'],
 158                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, thisbuild['vercode'])
 159                                     })
 160         if app['Current Version Code'] == '0':
 161             cantupdate = True
 162         # Sort with most recent first...
 163         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 164
 165         wikidata += "=Versions=\n"
 166         if len(apklist) == 0:
 167             wikidata += "We currently have no versions of this app available."
 168         elif not gotcurrentver:
 169             wikidata += "We don't have the current version of this app."
 170         else:
 171             wikidata += "We have the current version of this app."
 172         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 173         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 174         if len(app['No Source Since']) > 0:
 175             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 176         if len(app['Current Version']) > 0:
 177             wikidata += "The current (recommended) version is " + app['Current Version']
 178             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 179         validapks = 0
 180         for apk in apklist:
 181             wikidata += "==" + apk['version'] + "==\n"
 182
 183             if 'buildproblem' in apk:
 184                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 185             else:
 186                 validapks += 1
 187                 wikidata += "This version is built and signed by "
 188                 if 'srcname' in apk:
 189                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 190                 else:
 191                     wikidata += "the original developer.\n\n"
 192             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 193
 194         wikidata += '\n[[Category:' + wikicat + ']]\n'
 195         if len(app['No Source Since']) > 0:
 196             wikidata += '\n[[Category:Apps missing source code]]\n'
 197         if validapks == 0 and not app['Disabled']:
 198             wikidata += '\n[[Category:Apps with no packages]]\n'
 199         if cantupdate and not app['Disabled']:
 200             wikidata += "\n[[Category:Apps we can't update]]\n"
 201         if buildfails and not app['Disabled']:
 202             wikidata += "\n[[Category:Apps with failing builds]]\n"
 203         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 204             wikidata += '\n[[Category:Apps to Update]]\n'
 205         if app['Disabled']:
 206             wikidata += '\n[[Category:Apps that are disabled]]\n'
 207         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 208             wikidata += '\n[[Category:Apps with no update check]]\n'
 209         for appcat in app['Categories']:
 210             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 211
 212         # We can't have underscores in the page name, even if they're in
 213         # the package ID, because MediaWiki messes with them...
 214         pagename = appid.replace('_', ' ')
 215
 216         # Drop a trailing newline, because mediawiki is going to drop it anyway
 217         # and it we don't we'll think the page has changed when it hasn't...
 218         if wikidata.endswith('\n'):
 219             wikidata = wikidata[:-1]
 220
 221         generated_pages[pagename] = wikidata
 222
 223         # Make a redirect from the name to the ID too, unless there's
 224         # already an existing page with the name and it isn't a redirect.
 225         noclobber = False
 226         apppagename = app['Name'].replace('_', ' ')
 227         apppagename = apppagename.replace('{', '')
 228         apppagename = apppagename.replace('}', ' ')
 229         apppagename = apppagename.replace(':', ' ')
 230         # Drop double spaces caused mostly by replacing ':' above
 231         apppagename = apppagename.replace('  ', ' ')
 232         for expagename in site.allpages(prefix=apppagename,
 233                                         filterredir='nonredirects',
 234                                         generator=False):
 235             if expagename == apppagename:
 236                 noclobber = True
 237         # Another reason not to make the redirect page is if the app name
 238         # is the same as it's ID, because that will overwrite the real page
 239         # with an redirect to itself! (Although it seems like an odd
 240         # scenario this happens a lot, e.g. where there is metadata but no
 241         # builds or binaries to extract a name from.
 242         if apppagename == pagename:
 243             noclobber = True
 244         if not noclobber:
 245             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 246
 247     for tcat, genp in [(wikicat, generated_pages),
 248                        (wikiredircat, generated_redirects)]:
 249         catpages = site.Pages['Category:' + tcat]
 250         existingpages = []
 251         for page in catpages:
 252             existingpages.append(page.name)
 253             if page.name in genp:
 254                 pagetxt = page.edit()
 255                 if pagetxt != genp[page.name]:
 256                     logging.debug("Updating modified page " + page.name)
 257                     page.save(genp[page.name], summary='Auto-updated')
 258                 else:
 259                     logging.debug("Page " + page.name + " is unchanged")
 260             else:
 261                 logging.warn("Deleting page " + page.name)
 262                 page.delete('No longer published')
 263         for pagename, text in genp.items():
 264             logging.debug("Checking " + pagename)
 265             if pagename not in existingpages:
 266                 logging.debug("Creating page " + pagename)
 267                 try:
 268                     newpage = site.Pages[pagename]
 269                     newpage.save(text, summary='Auto-created')
 270                 except:
 271                     logging.error("...FAILED to create page '{0}'".format(pagename))
 272
 273     # Purge server cache to ensure counts are up to date
 274     site.pages['Repository Maintenance'].purge()
 275
 276
 277 def delete_disabled_builds(apps, apkcache, repodirs):
 278     """Delete disabled build outputs.
 279
 280     :param apps: list of all applications, as per metadata.read_metadata
 281     :param apkcache: current apk cache information
 282     :param repodirs: the repo directories to process
 283     """
 284     for appid, app in apps.iteritems():
 285         for build in app['builds']:
 286             if build['disable']:
 287                 apkfilename = appid + '_' + str(build['vercode']) + '.apk'
 288                 for repodir in repodirs:
 289                     apkpath = os.path.join(repodir, apkfilename)
 290                     ascpath = apkpath + ".asc"
 291                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 292                     for name in [apkpath, srcpath, ascpath]:
 293                         if os.path.exists(name):
 294                             logging.warn("Deleting disabled build output " + apkfilename)
 295                             os.remove(name)
 296                 if apkfilename in apkcache:
 297                     del apkcache[apkfilename]
 298
 299
 300 def resize_icon(iconpath, density):
 301
 302     if not os.path.isfile(iconpath):
 303         return
 304
 305     try:
 306         im = Image.open(iconpath)
 307         size = dpi_to_px(density)
 308
 309         if any(length > size for length in im.size):
 310             oldsize = im.size
 311             im.thumbnail((size, size), Image.ANTIALIAS)
 312             logging.debug("%s was too large at %s - new size is %s" % (
 313                 iconpath, oldsize, im.size))
 314             im.save(iconpath, "PNG")
 315
 316     except Exception, e:
 317         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 318
 319
 320 def resize_all_icons(repodirs):
 321     """Resize all icons that exceed the max size
 322
 323     :param repodirs: the repo directories to process
 324     """
 325     for repodir in repodirs:
 326         for density in get_densities():
 327             icon_dir = get_icon_dir(repodir, density)
 328             icon_glob = os.path.join(icon_dir, '*.png')
 329             for iconpath in glob.glob(icon_glob):
 330                 resize_icon(iconpath, density)
 331
 332
 333 # A signature block file with a .DSA, .RSA, or .EC extension
 334 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 335
 336
 337 def getsig(apkpath):
 338     """ Get the signing certificate of an apk. To get the same md5 has that
 339     Android gets, we encode the .RSA certificate in a specific format and pass
 340     it hex-encoded to the md5 digest algorithm.
 341
 342     :param apkpath: path to the apk
 343     :returns: A string containing the md5 of the signature of the apk or None
 344               if an error occurred.
 345     """
 346
 347     cert = None
 348
 349     # verify the jar signature is correct
 350     args = ['jarsigner', '-verify', apkpath]
 351     p = FDroidPopen(args)
 352     if p.returncode != 0:
 353         logging.critical(apkpath + " has a bad signature!")
 354         return None
 355
 356     with zipfile.ZipFile(apkpath, 'r') as apk:
 357
 358         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 359
 360         if len(certs) < 1:
 361             logging.error("Found no signing certificates on %s" % apkpath)
 362             return None
 363         if len(certs) > 1:
 364             logging.error("Found multiple signing certificates on %s" % apkpath)
 365             return None
 366
 367         cert = apk.read(certs[0])
 368
 369     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 370     if content.getComponentByName('contentType') != rfc2315.signedData:
 371         logging.error("Unexpected format.")
 372         return None
 373
 374     content = decoder.decode(content.getComponentByName('content'),
 375                              asn1Spec=rfc2315.SignedData())[0]
 376     try:
 377         certificates = content.getComponentByName('certificates')
 378     except PyAsn1Error:
 379         logging.error("Certificates not found.")
 380         return None
 381
 382     cert_encoded = encoder.encode(certificates)[4:]
 383
 384     return md5(cert_encoded.encode('hex')).hexdigest()
 385
 386
 387 def scan_apks(apps, apkcache, repodir, knownapks):
 388     """Scan the apks in the given repo directory.
 389
 390     This also extracts the icons.
 391
 392     :param apps: list of all applications, as per metadata.read_metadata
 393     :param apkcache: current apk cache information
 394     :param repodir: repo directory to scan
 395     :param knownapks: known apks info
 396     :returns: (apks, cachechanged) where apks is a list of apk information,
 397               and cachechanged is True if the apkcache got changed.
 398     """
 399
 400     cachechanged = False
 401
 402     icon_dirs = get_icon_dirs(repodir)
 403     for icon_dir in icon_dirs:
 404         if os.path.exists(icon_dir):
 405             if options.clean:
 406                 shutil.rmtree(icon_dir)
 407                 os.makedirs(icon_dir)
 408         else:
 409             os.makedirs(icon_dir)
 410
 411     apks = []
 412     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 413     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 414     vername_pat = re.compile(".*versionName='([^']*)'.*")
 415     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 416     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 417     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 418     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 419     string_pat = re.compile(".*'([^']*)'.*")
 420     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 421
 422         apkfilename = apkfile[len(repodir) + 1:]
 423         if ' ' in apkfilename:
 424             logging.critical("Spaces in filenames are not allowed.")
 425             sys.exit(1)
 426
 427         # Calculate the sha256...
 428         sha = hashlib.sha256()
 429         with open(apkfile, 'rb') as f:
 430             while True:
 431                 t = f.read(16384)
 432                 if len(t) == 0:
 433                     break
 434                 sha.update(t)
 435             shasum = sha.hexdigest()
 436
 437         usecache = False
 438         if apkfilename in apkcache:
 439             thisinfo = apkcache[apkfilename]
 440             if thisinfo['sha256'] == shasum:
 441                 logging.debug("Reading " + apkfilename + " from cache")
 442                 usecache = True
 443             else:
 444                 logging.debug("Ignoring stale cache data for " + apkfilename)
 445
 446         if not usecache:
 447             logging.debug("Processing " + apkfilename)
 448             thisinfo = {}
 449             thisinfo['apkname'] = apkfilename
 450             thisinfo['sha256'] = shasum
 451             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 452             if os.path.exists(os.path.join(repodir, srcfilename)):
 453                 thisinfo['srcname'] = srcfilename
 454             thisinfo['size'] = os.path.getsize(apkfile)
 455             thisinfo['permissions'] = set()
 456             thisinfo['features'] = set()
 457             thisinfo['icons_src'] = {}
 458             thisinfo['icons'] = {}
 459             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 460             if p.returncode != 0:
 461                 if options.delete_unknown:
 462                     if os.path.exists(apkfile):
 463                         logging.error("Failed to get apk information, deleting " + apkfile)
 464                         os.remove(apkfile)
 465                     else:
 466                         logging.error("Could not find {0} to remove it".format(apkfile))
 467                 else:
 468                     logging.error("Failed to get apk information, skipping " + apkfile)
 469                 continue
 470             for line in p.output.splitlines():
 471                 if line.startswith("package:"):
 472                     try:
 473                         thisinfo['id'] = re.match(name_pat, line).group(1)
 474                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 475                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 476                     except Exception, e:
 477                         logging.error("Package matching failed: " + str(e))
 478                         logging.info("Line was: " + line)
 479                         sys.exit(1)
 480                 elif line.startswith("application:"):
 481                     thisinfo['name'] = re.match(label_pat, line).group(1)
 482                     # Keep path to non-dpi icon in case we need it
 483                     match = re.match(icon_pat_nodpi, line)
 484                     if match:
 485                         thisinfo['icons_src']['-1'] = match.group(1)
 486                 elif line.startswith("launchable-activity:"):
 487                     # Only use launchable-activity as fallback to application
 488                     if not thisinfo['name']:
 489                         thisinfo['name'] = re.match(label_pat, line).group(1)
 490                     if '-1' not in thisinfo['icons_src']:
 491                         match = re.match(icon_pat_nodpi, line)
 492                         if match:
 493                             thisinfo['icons_src']['-1'] = match.group(1)
 494                 elif line.startswith("application-icon-"):
 495                     match = re.match(icon_pat, line)
 496                     if match:
 497                         density = match.group(1)
 498                         path = match.group(2)
 499                         thisinfo['icons_src'][density] = path
 500                 elif line.startswith("sdkVersion:"):
 501                     m = re.match(sdkversion_pat, line)
 502                     if m is None:
 503                         logging.error(line.replace('sdkVersion:', '')
 504                                       + ' is not a valid minSdkVersion!')
 505                     else:
 506                         thisinfo['sdkversion'] = m.group(1)
 507                 elif line.startswith("maxSdkVersion:"):
 508                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 509                 elif line.startswith("native-code:"):
 510                     thisinfo['nativecode'] = []
 511                     for arch in line[13:].split(' '):
 512                         thisinfo['nativecode'].append(arch[1:-1])
 513                 elif line.startswith("uses-permission:"):
 514                     perm = re.match(string_pat, line).group(1)
 515                     if perm.startswith("android.permission."):
 516                         perm = perm[19:]
 517                     thisinfo['permissions'].add(perm)
 518                 elif line.startswith("uses-feature:"):
 519                     perm = re.match(string_pat, line).group(1)
 520                     # Filter out this, it's only added with the latest SDK tools and
 521                     # causes problems for lots of apps.
 522                     if perm != "android.hardware.screen.portrait" \
 523                             and perm != "android.hardware.screen.landscape":
 524                         if perm.startswith("android.feature."):
 525                             perm = perm[16:]
 526                         thisinfo['features'].add(perm)
 527
 528             if 'sdkversion' not in thisinfo:
 529                 logging.warn("No SDK version information found in {0}".format(apkfile))
 530                 thisinfo['sdkversion'] = 0
 531
 532             # Check for debuggable apks...
 533             if common.isApkDebuggable(apkfile, config):
 534                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 535
 536             # Get the signature (or md5 of, to be precise)...
 537             logging.debug('Getting signature of {0}'.format(apkfile))
 538             thisinfo['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 539             if not thisinfo['sig']:
 540                 logging.critical("Failed to get apk signature")
 541                 sys.exit(1)
 542
 543             apk = zipfile.ZipFile(apkfile, 'r')
 544
 545             iconfilename = "%s.%s.png" % (
 546                 thisinfo['id'],
 547                 thisinfo['versioncode'])
 548
 549             # Extract the icon file...
 550             densities = get_densities()
 551             empty_densities = []
 552             for density in densities:
 553                 if density not in thisinfo['icons_src']:
 554                     empty_densities.append(density)
 555                     continue
 556                 iconsrc = thisinfo['icons_src'][density]
 557                 icon_dir = get_icon_dir(repodir, density)
 558                 icondest = os.path.join(icon_dir, iconfilename)
 559
 560                 try:
 561                     iconfile = open(icondest, 'wb')
 562                     iconfile.write(apk.read(iconsrc))
 563                     iconfile.close()
 564                     thisinfo['icons'][density] = iconfilename
 565
 566                 except:
 567                     logging.warn("Error retrieving icon file")
 568                     del thisinfo['icons'][density]
 569                     del thisinfo['icons_src'][density]
 570                     empty_densities.append(density)
 571
 572             if '-1' in thisinfo['icons_src']:
 573                 iconsrc = thisinfo['icons_src']['-1']
 574                 iconpath = os.path.join(
 575                     get_icon_dir(repodir, None), iconfilename)
 576                 iconfile = open(iconpath, 'wb')
 577                 iconfile.write(apk.read(iconsrc))
 578                 iconfile.close()
 579                 try:
 580                     im = Image.open(iconpath)
 581                     dpi = px_to_dpi(im.size[0])
 582                     for density in densities:
 583                         if density in thisinfo['icons']:
 584                             break
 585                         if density == densities[-1] or dpi >= int(density):
 586                             thisinfo['icons'][density] = iconfilename
 587                             shutil.move(iconpath,
 588                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 589                             empty_densities.remove(density)
 590                             break
 591                 except Exception, e:
 592                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 593
 594             if thisinfo['icons']:
 595                 thisinfo['icon'] = iconfilename
 596
 597             apk.close()
 598
 599             # First try resizing down to not lose quality
 600             last_density = None
 601             for density in densities:
 602                 if density not in empty_densities:
 603                     last_density = density
 604                     continue
 605                 if last_density is None:
 606                     continue
 607                 logging.debug("Density %s not available, resizing down from %s"
 608                               % (density, last_density))
 609
 610                 last_iconpath = os.path.join(
 611                     get_icon_dir(repodir, last_density), iconfilename)
 612                 iconpath = os.path.join(
 613                     get_icon_dir(repodir, density), iconfilename)
 614                 try:
 615                     im = Image.open(last_iconpath)
 616                 except:
 617                     logging.warn("Invalid image file at %s" % last_iconpath)
 618                     continue
 619
 620                 size = dpi_to_px(density)
 621
 622                 im.thumbnail((size, size), Image.ANTIALIAS)
 623                 im.save(iconpath, "PNG")
 624                 empty_densities.remove(density)
 625
 626             # Then just copy from the highest resolution available
 627             last_density = None
 628             for density in reversed(densities):
 629                 if density not in empty_densities:
 630                     last_density = density
 631                     continue
 632                 if last_density is None:
 633                     continue
 634                 logging.debug("Density %s not available, copying from lower density %s"
 635                               % (density, last_density))
 636
 637                 shutil.copyfile(
 638                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 639                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 640
 641                 empty_densities.remove(density)
 642
 643             for density in densities:
 644                 icon_dir = get_icon_dir(repodir, density)
 645                 icondest = os.path.join(icon_dir, iconfilename)
 646                 resize_icon(icondest, density)
 647
 648             # Copy from icons-mdpi to icons since mdpi is the baseline density
 649             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 650             if os.path.isfile(baseline):
 651                 shutil.copyfile(baseline,
 652                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 653
 654             # Record in known apks, getting the added date at the same time..
 655             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 656             if added:
 657                 thisinfo['added'] = added
 658
 659             apkcache[apkfilename] = thisinfo
 660             cachechanged = True
 661
 662         apks.append(thisinfo)
 663
 664     return apks, cachechanged
 665
 666
 667 repo_pubkey_fingerprint = None
 668
 669
 670 # Generate a certificate fingerprint the same way keytool does it
 671 # (but with slightly different formatting)
 672 def cert_fingerprint(data):
 673     digest = hashlib.sha256(data).digest()
 674     ret = []
 675     ret.append(' '.join("%02X" % ord(b) for b in digest))
 676     return " ".join(ret)
 677
 678
 679 def extract_pubkey():
 680     global repo_pubkey_fingerprint
 681     if 'repo_pubkey' in config:
 682         pubkey = unhexlify(config['repo_pubkey'])
 683     else:
 684         p = FDroidPopen(['keytool', '-exportcert',
 685                          '-alias', config['repo_keyalias'],
 686                          '-keystore', config['keystore'],
 687                          '-storepass:file', config['keystorepassfile']]
 688                         + config['smartcardoptions'], output=False)
 689         if p.returncode != 0 or len(p.output) < 20:
 690             msg = "Failed to get repo pubkey!"
 691             if config['keystore'] == 'NONE':
 692                 msg += ' Is your crypto smartcard plugged in?'
 693             logging.critical(msg)
 694             sys.exit(1)
 695         pubkey = p.output
 696     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 697     return hexlify(pubkey)
 698
 699
 700 def make_index(apps, sortedids, apks, repodir, archive, categories):
 701     """Make a repo index.
 702
 703     :param apps: fully populated apps list
 704     :param apks: full populated apks list
 705     :param repodir: the repo directory
 706     :param archive: True if this is the archive repo, False if it's the
 707                     main one.
 708     :param categories: list of categories
 709     """
 710
 711     doc = Document()
 712
 713     def addElement(name, value, doc, parent):
 714         el = doc.createElement(name)
 715         el.appendChild(doc.createTextNode(value))
 716         parent.appendChild(el)
 717
 718     def addElementCDATA(name, value, doc, parent):
 719         el = doc.createElement(name)
 720         el.appendChild(doc.createCDATASection(value))
 721         parent.appendChild(el)
 722
 723     root = doc.createElement("fdroid")
 724     doc.appendChild(root)
 725
 726     repoel = doc.createElement("repo")
 727
 728     if archive:
 729         repoel.setAttribute("name", config['archive_name'])
 730         if config['repo_maxage'] != 0:
 731             repoel.setAttribute("maxage", str(config['repo_maxage']))
 732         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 733         repoel.setAttribute("url", config['archive_url'])
 734         addElement('description', config['archive_description'], doc, repoel)
 735
 736     else:
 737         repoel.setAttribute("name", config['repo_name'])
 738         if config['repo_maxage'] != 0:
 739             repoel.setAttribute("maxage", str(config['repo_maxage']))
 740         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 741         repoel.setAttribute("url", config['repo_url'])
 742         addElement('description', config['repo_description'], doc, repoel)
 743
 744     repoel.setAttribute("version", "12")
 745     repoel.setAttribute("timestamp", str(int(time.time())))
 746
 747     nosigningkey = False
 748     if not options.nosign:
 749         if 'repo_keyalias' not in config:
 750             nosigningkey = True
 751             logging.critical("'repo_keyalias' not found in config.py!")
 752         if 'keystore' not in config:
 753             nosigningkey = True
 754             logging.critical("'keystore' not found in config.py!")
 755         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 756             nosigningkey = True
 757             logging.critical("'keystorepass' not found in config.py!")
 758         if 'keypass' not in config and 'keypassfile' not in config:
 759             nosigningkey = True
 760             logging.critical("'keypass' not found in config.py!")
 761         if not os.path.exists(config['keystore']):
 762             nosigningkey = True
 763             logging.critical("'" + config['keystore'] + "' does not exist!")
 764         if nosigningkey:
 765             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 766             logging.warning("\tfdroid update --create-key")
 767             sys.exit(1)
 768
 769     repoel.setAttribute("pubkey", extract_pubkey())
 770     root.appendChild(repoel)
 771
 772     for appid in sortedids:
 773         app = apps[appid]
 774
 775         if app['Disabled'] is not None:
 776             continue
 777
 778         # Get a list of the apks for this app...
 779         apklist = []
 780         for apk in apks:
 781             if apk['id'] == appid:
 782                 apklist.append(apk)
 783
 784         if len(apklist) == 0:
 785             continue
 786
 787         apel = doc.createElement("application")
 788         apel.setAttribute("id", app['id'])
 789         root.appendChild(apel)
 790
 791         addElement('id', app['id'], doc, apel)
 792         if 'added' in app:
 793             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 794         if 'lastupdated' in app:
 795             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 796         addElement('name', app['Name'], doc, apel)
 797         addElement('summary', app['Summary'], doc, apel)
 798         if app['icon']:
 799             addElement('icon', app['icon'], doc, apel)
 800
 801         def linkres(appid):
 802             if appid in apps:
 803                 return ("fdroid.app:" + appid, apps[appid]['Name'])
 804             raise MetaDataException("Cannot resolve app id " + appid)
 805
 806         addElement('desc',
 807                    metadata.description_html(app['Description'], linkres),
 808                    doc, apel)
 809         addElement('license', app['License'], doc, apel)
 810         if 'Categories' in app:
 811             addElement('categories', ','.join(app["Categories"]), doc, apel)
 812             # We put the first (primary) category in LAST, which will have
 813             # the desired effect of making clients that only understand one
 814             # category see that one.
 815             addElement('category', app["Categories"][0], doc, apel)
 816         addElement('web', app['Web Site'], doc, apel)
 817         addElement('source', app['Source Code'], doc, apel)
 818         addElement('tracker', app['Issue Tracker'], doc, apel)
 819         if app['Changelog']:
 820             addElement('changelog', app['Changelog'], doc, apel)
 821         if app['Donate']:
 822             addElement('donate', app['Donate'], doc, apel)
 823         if app['Bitcoin']:
 824             addElement('bitcoin', app['Bitcoin'], doc, apel)
 825         if app['Litecoin']:
 826             addElement('litecoin', app['Litecoin'], doc, apel)
 827         if app['Dogecoin']:
 828             addElement('dogecoin', app['Dogecoin'], doc, apel)
 829         if app['FlattrID']:
 830             addElement('flattr', app['FlattrID'], doc, apel)
 831
 832         # These elements actually refer to the current version (i.e. which
 833         # one is recommended. They are historically mis-named, and need
 834         # changing, but stay like this for now to support existing clients.
 835         addElement('marketversion', app['Current Version'], doc, apel)
 836         addElement('marketvercode', app['Current Version Code'], doc, apel)
 837
 838         if app['AntiFeatures']:
 839             af = app['AntiFeatures'].split(',')
 840             # TODO: Temporarily not including UpstreamNonFree in the index,
 841             # because current F-Droid clients do not understand it, and also
 842             # look ugly when they encounter an unknown antifeature. This
 843             # filtering can be removed in time...
 844             if 'UpstreamNonFree' in af:
 845                 af.remove('UpstreamNonFree')
 846             if af:
 847                 addElement('antifeatures', ','.join(af), doc, apel)
 848         if app['Provides']:
 849             pv = app['Provides'].split(',')
 850             addElement('provides', ','.join(pv), doc, apel)
 851         if app['Requires Root']:
 852             addElement('requirements', 'root', doc, apel)
 853
 854         # Sort the apk list into version order, just so the web site
 855         # doesn't have to do any work by default...
 856         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 857
 858         # Check for duplicates - they will make the client unhappy...
 859         for i in range(len(apklist) - 1):
 860             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 861                 logging.critical("duplicate versions: '%s' - '%s'" % (
 862                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 863                 sys.exit(1)
 864
 865         current_version_code = 0
 866         current_version_file = None
 867         for apk in apklist:
 868             # find the APK for the "Current Version"
 869             if current_version_code < apk['versioncode']:
 870                 current_version_code = apk['versioncode']
 871             if current_version_code < int(app['Current Version Code']):
 872                 current_version_file = apk['apkname']
 873
 874             apkel = doc.createElement("package")
 875             apel.appendChild(apkel)
 876             addElement('version', apk['version'], doc, apkel)
 877             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 878             addElement('apkname', apk['apkname'], doc, apkel)
 879             if 'srcname' in apk:
 880                 addElement('srcname', apk['srcname'], doc, apkel)
 881             for hash_type in ['sha256']:
 882                 if hash_type not in apk:
 883                     continue
 884                 hashel = doc.createElement("hash")
 885                 hashel.setAttribute("type", hash_type)
 886                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 887                 apkel.appendChild(hashel)
 888             addElement('sig', apk['sig'], doc, apkel)
 889             addElement('size', str(apk['size']), doc, apkel)
 890             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 891             if 'maxsdkversion' in apk:
 892                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 893             if 'added' in apk:
 894                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 895             if len(apk['permissions']) > 0:
 896                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 897             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 898                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 899             if len(apk['features']) > 0:
 900                 addElement('features', ','.join(apk['features']), doc, apkel)
 901
 902         if current_version_file is not None \
 903                 and config['make_current_version_link'] \
 904                 and repodir == 'repo':  # only create these
 905             sanitized_name = re.sub('''[ '"&%?+=/]''', '',
 906                                     app[config['current_version_name_source']])
 907             apklinkname = sanitized_name + '.apk'
 908             current_version_path = os.path.join(repodir, current_version_file)
 909             if os.path.exists(apklinkname):
 910                 os.remove(apklinkname)
 911             os.symlink(current_version_path, apklinkname)
 912             # also symlink gpg signature, if it exists
 913             for extension in ('.asc', '.sig'):
 914                 sigfile_path = current_version_path + extension
 915                 if os.path.exists(sigfile_path):
 916                     siglinkname = apklinkname + extension
 917                     if os.path.exists(siglinkname):
 918                         os.remove(siglinkname)
 919                     os.symlink(sigfile_path, siglinkname)
 920
 921     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 922     if options.pretty:
 923         output = doc.toprettyxml()
 924     else:
 925         output = doc.toxml()
 926     of.write(output)
 927     of.close()
 928
 929     if 'repo_keyalias' in config:
 930
 931         if options.nosign:
 932             logging.info("Creating unsigned index in preparation for signing")
 933         else:
 934             logging.info("Creating signed index with this key (SHA256):")
 935             logging.info("%s" % repo_pubkey_fingerprint)
 936
 937         # Create a jar of the index...
 938         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
 939         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 940         if p.returncode != 0:
 941             logging.critical("Failed to create {0}".format(jar_output))
 942             sys.exit(1)
 943
 944         # Sign the index...
 945         signed = os.path.join(repodir, 'index.jar')
 946         if options.nosign:
 947             # Remove old signed index if not signing
 948             if os.path.exists(signed):
 949                 os.remove(signed)
 950         else:
 951             args = ['jarsigner', '-keystore', config['keystore'],
 952                     '-storepass:file', config['keystorepassfile'],
 953                     '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 954                     signed, config['repo_keyalias']]
 955             if config['keystore'] == 'NONE':
 956                 args += config['smartcardoptions']
 957             else:  # smardcards never use -keypass
 958                 args += ['-keypass:file', config['keypassfile']]
 959             p = FDroidPopen(args)
 960             if p.returncode != 0:
 961                 logging.critical("Failed to sign index")
 962                 sys.exit(1)
 963
 964     # Copy the repo icon into the repo directory...
 965     icon_dir = os.path.join(repodir, 'icons')
 966     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 967     shutil.copyfile(config['repo_icon'], iconfilename)
 968
 969     # Write a category list in the repo to allow quick access...
 970     catdata = ''
 971     for cat in categories:
 972         catdata += cat + '\n'
 973     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 974     f.write(catdata)
 975     f.close()
 976
 977
 978 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 979
 980     for appid, app in apps.iteritems():
 981
 982         # Get a list of the apks for this app...
 983         apklist = []
 984         for apk in apks:
 985             if apk['id'] == appid:
 986                 apklist.append(apk)
 987
 988         # Sort the apk list into version order...
 989         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 990
 991         if app['Archive Policy']:
 992             keepversions = int(app['Archive Policy'][:-9])
 993         else:
 994             keepversions = defaultkeepversions
 995
 996         if len(apklist) > keepversions:
 997             for apk in apklist[keepversions:]:
 998                 logging.info("Moving " + apk['apkname'] + " to archive")
 999                 shutil.move(os.path.join(repodir, apk['apkname']),
1000                             os.path.join(archivedir, apk['apkname']))
1001                 if 'srcname' in apk:
1002                     shutil.move(os.path.join(repodir, apk['srcname']),
1003                                 os.path.join(archivedir, apk['srcname']))
1004                     # Move GPG signature too...
1005                     sigfile = apk['srcname'] + '.asc'
1006                     sigsrc = os.path.join(repodir, sigfile)
1007                     if os.path.exists(sigsrc):
1008                         shutil.move(sigsrc, os.path.join(archivedir, sigfile))
1009
1010                 archapks.append(apk)
1011                 apks.remove(apk)
1012
1013
1014 config = None
1015 options = None
1016
1017
1018 def main():
1019
1020     global config, options
1021
1022     # Parse command line...
1023     parser = OptionParser()
1024     parser.add_option("--create-key", action="store_true", default=False,
1025                       help="Create a repo signing key in a keystore")
1026     parser.add_option("-c", "--create-metadata", action="store_true", default=False,
1027                       help="Create skeleton metadata files that are missing")
1028     parser.add_option("--delete-unknown", action="store_true", default=False,
1029                       help="Delete APKs without metadata from the repo")
1030     parser.add_option("-v", "--verbose", action="store_true", default=False,
1031                       help="Spew out even more information than normal")
1032     parser.add_option("-q", "--quiet", action="store_true", default=False,
1033                       help="Restrict output to warnings and errors")
1034     parser.add_option("-b", "--buildreport", action="store_true", default=False,
1035                       help="Report on build data status")
1036     parser.add_option("-i", "--interactive", default=False, action="store_true",
1037                       help="Interactively ask about things that need updating.")
1038     parser.add_option("-I", "--icons", action="store_true", default=False,
1039                       help="Resize all the icons exceeding the max pixel size and exit")
1040     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
1041                       help="Specify editor to use in interactive mode. Default " +
1042                       "is /etc/alternatives/editor")
1043     parser.add_option("-w", "--wiki", default=False, action="store_true",
1044                       help="Update the wiki")
1045     parser.add_option("", "--pretty", action="store_true", default=False,
1046                       help="Produce human-readable index.xml")
1047     parser.add_option("--clean", action="store_true", default=False,
1048                       help="Clean update - don't uses caches, reprocess all apks")
1049     parser.add_option("--nosign", action="store_true", default=False,
1050                       help="When configured for signed indexes, create only unsigned indexes at this stage")
1051     (options, args) = parser.parse_args()
1052
1053     config = common.read_config(options)
1054
1055     repodirs = ['repo']
1056     if config['archive_older'] != 0:
1057         repodirs.append('archive')
1058         if not os.path.exists('archive'):
1059             os.mkdir('archive')
1060
1061     if options.icons:
1062         resize_all_icons(repodirs)
1063         sys.exit(0)
1064
1065     # check that icons exist now, rather than fail at the end of `fdroid update`
1066     for k in ['repo_icon', 'archive_icon']:
1067         if k in config:
1068             if not os.path.exists(config[k]):
1069                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1070                 sys.exit(1)
1071
1072     # if the user asks to create a keystore, do it now, reusing whatever it can
1073     if options.create_key:
1074         if os.path.exists(config['keystore']):
1075             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1076             logging.critical("\t'" + config['keystore'] + "'")
1077             sys.exit(1)
1078
1079         if 'repo_keyalias' not in config:
1080             config['repo_keyalias'] = socket.getfqdn()
1081             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1082         if 'keydname' not in config:
1083             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1084             common.write_to_config(config, 'keydname', config['keydname'])
1085         if 'keystore' not in config:
1086             config['keystore'] = common.default_config.keystore
1087             common.write_to_config(config, 'keystore', config['keystore'])
1088
1089         password = common.genpassword()
1090         if 'keystorepass' not in config:
1091             config['keystorepass'] = password
1092             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1093         if 'keypass' not in config:
1094             config['keypass'] = password
1095             common.write_to_config(config, 'keypass', config['keypass'])
1096         common.genkeystore(config)
1097
1098     # Get all apps...
1099     apps = metadata.read_metadata()
1100
1101     # Generate a list of categories...
1102     categories = set()
1103     for app in apps.itervalues():
1104         categories.update(app['Categories'])
1105
1106     # Read known apks data (will be updated and written back when we've finished)
1107     knownapks = common.KnownApks()
1108
1109     # Gather information about all the apk files in the repo directory, using
1110     # cached data if possible.
1111     apkcachefile = os.path.join('tmp', 'apkcache')
1112     if not options.clean and os.path.exists(apkcachefile):
1113         with open(apkcachefile, 'rb') as cf:
1114             apkcache = pickle.load(cf)
1115     else:
1116         apkcache = {}
1117     cachechanged = False
1118
1119     delete_disabled_builds(apps, apkcache, repodirs)
1120
1121     # Scan all apks in the main repo
1122     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
1123     if cc:
1124         cachechanged = True
1125
1126     # Generate warnings for apk's with no metadata (or create skeleton
1127     # metadata files, if requested on the command line)
1128     newmetadata = False
1129     for apk in apks:
1130         if apk['id'] not in apps:
1131             if options.create_metadata:
1132                 if 'name' not in apk:
1133                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1134                     continue
1135                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1136                 f.write("License:Unknown\n")
1137                 f.write("Web Site:\n")
1138                 f.write("Source Code:\n")
1139                 f.write("Issue Tracker:\n")
1140                 f.write("Changelog:\n")
1141                 f.write("Summary:" + apk['name'] + "\n")
1142                 f.write("Description:\n")
1143                 f.write(apk['name'] + "\n")
1144                 f.write(".\n")
1145                 f.close()
1146                 logging.info("Generated skeleton metadata for " + apk['id'])
1147                 newmetadata = True
1148             else:
1149                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1150                 if options.delete_unknown:
1151                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1152                     rmf = os.path.join(repodirs[0], apk['apkname'])
1153                     if not os.path.exists(rmf):
1154                         logging.error("Could not find {0} to remove it".format(rmf))
1155                     else:
1156                         os.remove(rmf)
1157                 else:
1158                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1159
1160     # update the metadata with the newly created ones included
1161     if newmetadata:
1162         apps = metadata.read_metadata()
1163
1164     # Scan the archive repo for apks as well
1165     if len(repodirs) > 1:
1166         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1167         if cc:
1168             cachechanged = True
1169     else:
1170         archapks = []
1171
1172     # Some information from the apks needs to be applied up to the application
1173     # level. When doing this, we use the info from the most recent version's apk.
1174     # We deal with figuring out when the app was added and last updated at the
1175     # same time.
1176     for appid, app in apps.iteritems():
1177         bestver = 0
1178         added = None
1179         lastupdated = None
1180         for apk in apks + archapks:
1181             if apk['id'] == appid:
1182                 if apk['versioncode'] > bestver:
1183                     bestver = apk['versioncode']
1184                     bestapk = apk
1185
1186                 if 'added' in apk:
1187                     if not added or apk['added'] < added:
1188                         added = apk['added']
1189                     if not lastupdated or apk['added'] > lastupdated:
1190                         lastupdated = apk['added']
1191
1192         if added:
1193             app['added'] = added
1194         else:
1195             logging.warn("Don't know when " + appid + " was added")
1196         if lastupdated:
1197             app['lastupdated'] = lastupdated
1198         else:
1199             logging.warn("Don't know when " + appid + " was last updated")
1200
1201         if bestver == 0:
1202             if app['Name'] is None:
1203                 app['Name'] = app['Auto Name'] or appid
1204             app['icon'] = None
1205             logging.warn("Application " + appid + " has no packages")
1206         else:
1207             if app['Name'] is None:
1208                 app['Name'] = bestapk['name']
1209             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1210
1211     # Sort the app list by name, then the web site doesn't have to by default.
1212     # (we had to wait until we'd scanned the apks to do this, because mostly the
1213     # name comes from there!)
1214     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid]['Name'].upper())
1215
1216     if len(repodirs) > 1:
1217         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1218
1219     # Make the index for the main repo...
1220     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1221
1222     # If there's an archive repo,  make the index for it. We already scanned it
1223     # earlier on.
1224     if len(repodirs) > 1:
1225         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1226
1227     if config['update_stats']:
1228
1229         # Update known apks info...
1230         knownapks.writeifchanged()
1231
1232         # Generate latest apps data for widget
1233         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1234             data = ''
1235             for line in file(os.path.join('stats', 'latestapps.txt')):
1236                 appid = line.rstrip()
1237                 data += appid + "\t"
1238                 app = apps[appid]
1239                 data += app['Name'] + "\t"
1240                 if app['icon'] is not None:
1241                     data += app['icon'] + "\t"
1242                 data += app['License'] + "\n"
1243             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1244             f.write(data)
1245             f.close()
1246
1247     if cachechanged:
1248         with open(apkcachefile, 'wb') as cf:
1249             pickle.dump(apkcache, cf)
1250
1251     # Update the wiki...
1252     if options.wiki:
1253         update_wiki(apps, sortedids, apks + archapks)
1254
1255     logging.info("Finished.")
1256
1257 if __name__ == "__main__":
1258     main()