fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import socket
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from xml.dom.minidom import Document
  31 from optparse import OptionParser
  32 import time
  33 from pyasn1.error import PyAsn1Error
  34 from pyasn1.codec.der import decoder, encoder
  35 from pyasn1_modules import rfc2315
  36 from hashlib import md5
  37 from binascii import hexlify, unhexlify
  38
  39 from PIL import Image
  40 import logging
  41
  42 import common
  43 import metadata
  44 from common import FDroidPopen, SdkToolsPopen
  45 from metadata import MetaDataException
  46
  47
  48 def get_densities():
  49     return ['640', '480', '320', '240', '160', '120']
  50
  51
  52 def dpi_to_px(density):
  53     return (int(density) * 48) / 160
  54
  55
  56 def px_to_dpi(px):
  57     return (int(px) * 160) / 48
  58
  59
  60 def get_icon_dir(repodir, density):
  61     if density is None:
  62         return os.path.join(repodir, "icons")
  63     return os.path.join(repodir, "icons-%s" % density)
  64
  65
  66 def get_icon_dirs(repodir):
  67     for density in get_densities():
  68         yield get_icon_dir(repodir, density)
  69     yield os.path.join(repodir, "icons")
  70
  71
  72 def update_wiki(apps, sortedids, apks):
  73     """Update the wiki
  74
  75     :param apps: fully populated list of all applications
  76     :param apks: all apks, except...
  77     """
  78     logging.info("Updating wiki")
  79     wikicat = 'Apps'
  80     wikiredircat = 'App Redirects'
  81     import mwclient
  82     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  83                          path=config['wiki_path'])
  84     site.login(config['wiki_user'], config['wiki_password'])
  85     generated_pages = {}
  86     generated_redirects = {}
  87
  88     for appid in sortedids:
  89         app = apps[appid]
  90
  91         wikidata = ''
  92         if app['Disabled']:
  93             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  94         if app['AntiFeatures']:
  95             for af in app['AntiFeatures'].split(','):
  96                 wikidata += '{{AntiFeature|' + af + '}}\n'
  97         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
  98             appid,
  99             app['Name'],
 100             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
 101             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
 102             app['Source Code'],
 103             app['Issue Tracker'],
 104             app['Web Site'],
 105             app['Donate'],
 106             app['FlattrID'],
 107             app['Bitcoin'],
 108             app['Litecoin'],
 109             app['Dogecoin'],
 110             app['License'],
 111             app.get('Requires Root', 'No'))
 112
 113         if app['Provides']:
 114             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 115
 116         wikidata += app['Summary']
 117         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 118
 119         wikidata += "=Description=\n"
 120         wikidata += metadata.description_wiki(app['Description']) + "\n"
 121
 122         wikidata += "=Maintainer Notes=\n"
 123         if 'Maintainer Notes' in app:
 124             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 125         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 126
 127         # Get a list of all packages for this application...
 128         apklist = []
 129         gotcurrentver = False
 130         cantupdate = False
 131         buildfails = False
 132         for apk in apks:
 133             if apk['id'] == appid:
 134                 if str(apk['versioncode']) == app['Current Version Code']:
 135                     gotcurrentver = True
 136                 apklist.append(apk)
 137         # Include ones we can't build, as a special case...
 138         for thisbuild in app['builds']:
 139             if thisbuild['disable']:
 140                 if thisbuild['vercode'] == app['Current Version Code']:
 141                     cantupdate = True
 142                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 143                 apklist.append({'versioncode': int(thisbuild['vercode']),
 144                                 'version': thisbuild['version'],
 145                                 'buildproblem': thisbuild['disable']
 146                                 })
 147             else:
 148                 builtit = False
 149                 for apk in apklist:
 150                     if apk['versioncode'] == int(thisbuild['vercode']):
 151                         builtit = True
 152                         break
 153                 if not builtit:
 154                     buildfails = True
 155                     apklist.append({'versioncode': int(thisbuild['vercode']),
 156                                     'version': thisbuild['version'],
 157                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, thisbuild['vercode'])
 158                                     })
 159         if app['Current Version Code'] == '0':
 160             cantupdate = True
 161         # Sort with most recent first...
 162         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 163
 164         wikidata += "=Versions=\n"
 165         if len(apklist) == 0:
 166             wikidata += "We currently have no versions of this app available."
 167         elif not gotcurrentver:
 168             wikidata += "We don't have the current version of this app."
 169         else:
 170             wikidata += "We have the current version of this app."
 171         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 172         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 173         if len(app['No Source Since']) > 0:
 174             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 175         if len(app['Current Version']) > 0:
 176             wikidata += "The current (recommended) version is " + app['Current Version']
 177             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 178         validapks = 0
 179         for apk in apklist:
 180             wikidata += "==" + apk['version'] + "==\n"
 181
 182             if 'buildproblem' in apk:
 183                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 184             else:
 185                 validapks += 1
 186                 wikidata += "This version is built and signed by "
 187                 if 'srcname' in apk:
 188                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 189                 else:
 190                     wikidata += "the original developer.\n\n"
 191             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 192
 193         wikidata += '\n[[Category:' + wikicat + ']]\n'
 194         if len(app['No Source Since']) > 0:
 195             wikidata += '\n[[Category:Apps missing source code]]\n'
 196         if validapks == 0 and not app['Disabled']:
 197             wikidata += '\n[[Category:Apps with no packages]]\n'
 198         if cantupdate and not app['Disabled']:
 199             wikidata += "\n[[Category:Apps we can't update]]\n"
 200         if buildfails and not app['Disabled']:
 201             wikidata += "\n[[Category:Apps with failing builds]]\n"
 202         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 203             wikidata += '\n[[Category:Apps to Update]]\n'
 204         if app['Disabled']:
 205             wikidata += '\n[[Category:Apps that are disabled]]\n'
 206         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 207             wikidata += '\n[[Category:Apps with no update check]]\n'
 208         for appcat in app['Categories']:
 209             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 210
 211         # We can't have underscores in the page name, even if they're in
 212         # the package ID, because MediaWiki messes with them...
 213         pagename = appid.replace('_', ' ')
 214
 215         # Drop a trailing newline, because mediawiki is going to drop it anyway
 216         # and it we don't we'll think the page has changed when it hasn't...
 217         if wikidata.endswith('\n'):
 218             wikidata = wikidata[:-1]
 219
 220         generated_pages[pagename] = wikidata
 221
 222         # Make a redirect from the name to the ID too, unless there's
 223         # already an existing page with the name and it isn't a redirect.
 224         noclobber = False
 225         apppagename = app['Name'].replace('_', ' ')
 226         apppagename = apppagename.replace('{', '')
 227         apppagename = apppagename.replace('}', ' ')
 228         apppagename = apppagename.replace(':', ' ')
 229         # Drop double spaces caused mostly by replacing ':' above
 230         apppagename = apppagename.replace('  ', ' ')
 231         for expagename in site.allpages(prefix=apppagename,
 232                                         filterredir='nonredirects',
 233                                         generator=False):
 234             if expagename == apppagename:
 235                 noclobber = True
 236         # Another reason not to make the redirect page is if the app name
 237         # is the same as it's ID, because that will overwrite the real page
 238         # with an redirect to itself! (Although it seems like an odd
 239         # scenario this happens a lot, e.g. where there is metadata but no
 240         # builds or binaries to extract a name from.
 241         if apppagename == pagename:
 242             noclobber = True
 243         if not noclobber:
 244             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 245
 246     for tcat, genp in [(wikicat, generated_pages),
 247                        (wikiredircat, generated_redirects)]:
 248         catpages = site.Pages['Category:' + tcat]
 249         existingpages = []
 250         for page in catpages:
 251             existingpages.append(page.name)
 252             if page.name in genp:
 253                 pagetxt = page.edit()
 254                 if pagetxt != genp[page.name]:
 255                     logging.debug("Updating modified page " + page.name)
 256                     page.save(genp[page.name], summary='Auto-updated')
 257                 else:
 258                     logging.debug("Page " + page.name + " is unchanged")
 259             else:
 260                 logging.warn("Deleting page " + page.name)
 261                 page.delete('No longer published')
 262         for pagename, text in genp.items():
 263             logging.debug("Checking " + pagename)
 264             if pagename not in existingpages:
 265                 logging.debug("Creating page " + pagename)
 266                 try:
 267                     newpage = site.Pages[pagename]
 268                     newpage.save(text, summary='Auto-created')
 269                 except:
 270                     logging.error("...FAILED to create page '{0}'".format(pagename))
 271
 272     # Purge server cache to ensure counts are up to date
 273     site.pages['Repository Maintenance'].purge()
 274
 275
 276 def delete_disabled_builds(apps, apkcache, repodirs):
 277     """Delete disabled build outputs.
 278
 279     :param apps: list of all applications, as per metadata.read_metadata
 280     :param apkcache: current apk cache information
 281     :param repodirs: the repo directories to process
 282     """
 283     for appid, app in apps.iteritems():
 284         for build in app['builds']:
 285             if build['disable']:
 286                 apkfilename = appid + '_' + str(build['vercode']) + '.apk'
 287                 for repodir in repodirs:
 288                     apkpath = os.path.join(repodir, apkfilename)
 289                     ascpath = apkpath + ".asc"
 290                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 291                     for name in [apkpath, srcpath, ascpath]:
 292                         if os.path.exists(name):
 293                             logging.warn("Deleting disabled build output " + apkfilename)
 294                             os.remove(name)
 295                 if apkfilename in apkcache:
 296                     del apkcache[apkfilename]
 297
 298
 299 def resize_icon(iconpath, density):
 300
 301     if not os.path.isfile(iconpath):
 302         return
 303
 304     try:
 305         im = Image.open(iconpath)
 306         size = dpi_to_px(density)
 307
 308         if any(length > size for length in im.size):
 309             oldsize = im.size
 310             im.thumbnail((size, size), Image.ANTIALIAS)
 311             logging.debug("%s was too large at %s - new size is %s" % (
 312                 iconpath, oldsize, im.size))
 313             im.save(iconpath, "PNG")
 314
 315     except Exception, e:
 316         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 317
 318
 319 def resize_all_icons(repodirs):
 320     """Resize all icons that exceed the max size
 321
 322     :param repodirs: the repo directories to process
 323     """
 324     for repodir in repodirs:
 325         for density in get_densities():
 326             icon_dir = get_icon_dir(repodir, density)
 327             icon_glob = os.path.join(icon_dir, '*.png')
 328             for iconpath in glob.glob(icon_glob):
 329                 resize_icon(iconpath, density)
 330
 331
 332 # A signature block file with a .DSA, .RSA, or .EC extension
 333 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 334
 335
 336 def getsig(apkpath):
 337     """ Get the signing certificate of an apk. To get the same md5 has that
 338     Android gets, we encode the .RSA certificate in a specific format and pass
 339     it hex-encoded to the md5 digest algorithm.
 340
 341     :param apkpath: path to the apk
 342     :returns: A string containing the md5 of the signature of the apk or None
 343               if an error occurred.
 344     """
 345
 346     cert = None
 347
 348     # verify the jar signature is correct
 349     args = ['jarsigner', '-verify', apkpath]
 350     p = FDroidPopen(args)
 351     if p.returncode != 0:
 352         logging.critical(apkpath + " has a bad signature!")
 353         return None
 354
 355     with zipfile.ZipFile(apkpath, 'r') as apk:
 356
 357         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 358
 359         if len(certs) < 1:
 360             logging.error("Found no signing certificates on %s" % apkpath)
 361             return None
 362         if len(certs) > 1:
 363             logging.error("Found multiple signing certificates on %s" % apkpath)
 364             return None
 365
 366         cert = apk.read(certs[0])
 367
 368     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 369     if content.getComponentByName('contentType') != rfc2315.signedData:
 370         logging.error("Unexpected format.")
 371         return None
 372
 373     content = decoder.decode(content.getComponentByName('content'),
 374                              asn1Spec=rfc2315.SignedData())[0]
 375     try:
 376         certificates = content.getComponentByName('certificates')
 377     except PyAsn1Error:
 378         logging.error("Certificates not found.")
 379         return None
 380
 381     cert_encoded = encoder.encode(certificates)[4:]
 382
 383     return md5(cert_encoded.encode('hex')).hexdigest()
 384
 385
 386 def scan_apks(apps, apkcache, repodir, knownapks):
 387     """Scan the apks in the given repo directory.
 388
 389     This also extracts the icons.
 390
 391     :param apps: list of all applications, as per metadata.read_metadata
 392     :param apkcache: current apk cache information
 393     :param repodir: repo directory to scan
 394     :param knownapks: known apks info
 395     :returns: (apks, cachechanged) where apks is a list of apk information,
 396               and cachechanged is True if the apkcache got changed.
 397     """
 398
 399     cachechanged = False
 400
 401     icon_dirs = get_icon_dirs(repodir)
 402     for icon_dir in icon_dirs:
 403         if os.path.exists(icon_dir):
 404             if options.clean:
 405                 shutil.rmtree(icon_dir)
 406                 os.makedirs(icon_dir)
 407         else:
 408             os.makedirs(icon_dir)
 409
 410     apks = []
 411     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 412     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 413     vername_pat = re.compile(".*versionName='([^']*)'.*")
 414     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 415     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 416     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 417     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 418     string_pat = re.compile(".*'([^']*)'.*")
 419     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 420
 421         apkfilename = apkfile[len(repodir) + 1:]
 422         if ' ' in apkfilename:
 423             logging.critical("Spaces in filenames are not allowed.")
 424             sys.exit(1)
 425
 426         # Calculate the sha256...
 427         sha = hashlib.sha256()
 428         with open(apkfile, 'rb') as f:
 429             while True:
 430                 t = f.read(16384)
 431                 if len(t) == 0:
 432                     break
 433                 sha.update(t)
 434             shasum = sha.hexdigest()
 435
 436         usecache = False
 437         if apkfilename in apkcache:
 438             thisinfo = apkcache[apkfilename]
 439             if thisinfo['sha256'] == shasum:
 440                 logging.debug("Reading " + apkfilename + " from cache")
 441                 usecache = True
 442             else:
 443                 logging.debug("Ignoring stale cache data for " + apkfilename)
 444
 445         if not usecache:
 446             logging.debug("Processing " + apkfilename)
 447             thisinfo = {}
 448             thisinfo['apkname'] = apkfilename
 449             thisinfo['sha256'] = shasum
 450             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 451             if os.path.exists(os.path.join(repodir, srcfilename)):
 452                 thisinfo['srcname'] = srcfilename
 453             thisinfo['size'] = os.path.getsize(apkfile)
 454             thisinfo['permissions'] = set()
 455             thisinfo['features'] = set()
 456             thisinfo['icons_src'] = {}
 457             thisinfo['icons'] = {}
 458             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 459             if p.returncode != 0:
 460                 if options.delete_unknown:
 461                     if os.path.exists(apkfile):
 462                         logging.error("Failed to get apk information, deleting " + apkfile)
 463                         os.remove(apkfile)
 464                     else:
 465                         logging.error("Could not find {0} to remove it".format(apkfile))
 466                 else:
 467                     logging.error("Failed to get apk information, skipping " + apkfile)
 468                 continue
 469             for line in p.output.splitlines():
 470                 if line.startswith("package:"):
 471                     try:
 472                         thisinfo['id'] = re.match(name_pat, line).group(1)
 473                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 474                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 475                     except Exception, e:
 476                         logging.error("Package matching failed: " + str(e))
 477                         logging.info("Line was: " + line)
 478                         sys.exit(1)
 479                 elif line.startswith("application:"):
 480                     thisinfo['name'] = re.match(label_pat, line).group(1)
 481                     # Keep path to non-dpi icon in case we need it
 482                     match = re.match(icon_pat_nodpi, line)
 483                     if match:
 484                         thisinfo['icons_src']['-1'] = match.group(1)
 485                 elif line.startswith("launchable-activity:"):
 486                     # Only use launchable-activity as fallback to application
 487                     if not thisinfo['name']:
 488                         thisinfo['name'] = re.match(label_pat, line).group(1)
 489                     if '-1' not in thisinfo['icons_src']:
 490                         match = re.match(icon_pat_nodpi, line)
 491                         if match:
 492                             thisinfo['icons_src']['-1'] = match.group(1)
 493                 elif line.startswith("application-icon-"):
 494                     match = re.match(icon_pat, line)
 495                     if match:
 496                         density = match.group(1)
 497                         path = match.group(2)
 498                         thisinfo['icons_src'][density] = path
 499                 elif line.startswith("sdkVersion:"):
 500                     m = re.match(sdkversion_pat, line)
 501                     if m is None:
 502                         logging.error(line.replace('sdkVersion:', '')
 503                                       + ' is not a valid minSdkVersion!')
 504                     else:
 505                         thisinfo['sdkversion'] = m.group(1)
 506                 elif line.startswith("maxSdkVersion:"):
 507                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 508                 elif line.startswith("native-code:"):
 509                     thisinfo['nativecode'] = []
 510                     for arch in line[13:].split(' '):
 511                         thisinfo['nativecode'].append(arch[1:-1])
 512                 elif line.startswith("uses-permission:"):
 513                     perm = re.match(string_pat, line).group(1)
 514                     if perm.startswith("android.permission."):
 515                         perm = perm[19:]
 516                     thisinfo['permissions'].add(perm)
 517                 elif line.startswith("uses-feature:"):
 518                     perm = re.match(string_pat, line).group(1)
 519                     # Filter out this, it's only added with the latest SDK tools and
 520                     # causes problems for lots of apps.
 521                     if perm != "android.hardware.screen.portrait" \
 522                             and perm != "android.hardware.screen.landscape":
 523                         if perm.startswith("android.feature."):
 524                             perm = perm[16:]
 525                         thisinfo['features'].add(perm)
 526
 527             if 'sdkversion' not in thisinfo:
 528                 logging.warn("No SDK version information found in {0}".format(apkfile))
 529                 thisinfo['sdkversion'] = 0
 530
 531             # Check for debuggable apks...
 532             if common.isApkDebuggable(apkfile, config):
 533                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 534
 535             # Get the signature (or md5 of, to be precise)...
 536             logging.debug('Getting signature of {0}'.format(apkfile))
 537             thisinfo['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 538             if not thisinfo['sig']:
 539                 logging.critical("Failed to get apk signature")
 540                 sys.exit(1)
 541
 542             apk = zipfile.ZipFile(apkfile, 'r')
 543
 544             iconfilename = "%s.%s.png" % (
 545                 thisinfo['id'],
 546                 thisinfo['versioncode'])
 547
 548             # Extract the icon file...
 549             densities = get_densities()
 550             empty_densities = []
 551             for density in densities:
 552                 if density not in thisinfo['icons_src']:
 553                     empty_densities.append(density)
 554                     continue
 555                 iconsrc = thisinfo['icons_src'][density]
 556                 icon_dir = get_icon_dir(repodir, density)
 557                 icondest = os.path.join(icon_dir, iconfilename)
 558
 559                 try:
 560                     iconfile = open(icondest, 'wb')
 561                     iconfile.write(apk.read(iconsrc))
 562                     iconfile.close()
 563                     thisinfo['icons'][density] = iconfilename
 564
 565                 except:
 566                     logging.warn("Error retrieving icon file")
 567                     del thisinfo['icons'][density]
 568                     del thisinfo['icons_src'][density]
 569                     empty_densities.append(density)
 570
 571             if '-1' in thisinfo['icons_src']:
 572                 iconsrc = thisinfo['icons_src']['-1']
 573                 iconpath = os.path.join(
 574                     get_icon_dir(repodir, None), iconfilename)
 575                 iconfile = open(iconpath, 'wb')
 576                 iconfile.write(apk.read(iconsrc))
 577                 iconfile.close()
 578                 try:
 579                     im = Image.open(iconpath)
 580                     dpi = px_to_dpi(im.size[0])
 581                     for density in densities:
 582                         if density in thisinfo['icons']:
 583                             break
 584                         if density == densities[-1] or dpi >= int(density):
 585                             thisinfo['icons'][density] = iconfilename
 586                             shutil.move(iconpath,
 587                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 588                             empty_densities.remove(density)
 589                             break
 590                 except Exception, e:
 591                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 592
 593             if thisinfo['icons']:
 594                 thisinfo['icon'] = iconfilename
 595
 596             apk.close()
 597
 598             # First try resizing down to not lose quality
 599             last_density = None
 600             for density in densities:
 601                 if density not in empty_densities:
 602                     last_density = density
 603                     continue
 604                 if last_density is None:
 605                     continue
 606                 logging.debug("Density %s not available, resizing down from %s"
 607                               % (density, last_density))
 608
 609                 last_iconpath = os.path.join(
 610                     get_icon_dir(repodir, last_density), iconfilename)
 611                 iconpath = os.path.join(
 612                     get_icon_dir(repodir, density), iconfilename)
 613                 try:
 614                     im = Image.open(last_iconpath)
 615                 except:
 616                     logging.warn("Invalid image file at %s" % last_iconpath)
 617                     continue
 618
 619                 size = dpi_to_px(density)
 620
 621                 im.thumbnail((size, size), Image.ANTIALIAS)
 622                 im.save(iconpath, "PNG")
 623                 empty_densities.remove(density)
 624
 625             # Then just copy from the highest resolution available
 626             last_density = None
 627             for density in reversed(densities):
 628                 if density not in empty_densities:
 629                     last_density = density
 630                     continue
 631                 if last_density is None:
 632                     continue
 633                 logging.debug("Density %s not available, copying from lower density %s"
 634                               % (density, last_density))
 635
 636                 shutil.copyfile(
 637                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 638                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 639
 640                 empty_densities.remove(density)
 641
 642             for density in densities:
 643                 icon_dir = get_icon_dir(repodir, density)
 644                 icondest = os.path.join(icon_dir, iconfilename)
 645                 resize_icon(icondest, density)
 646
 647             # Copy from icons-mdpi to icons since mdpi is the baseline density
 648             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 649             if os.path.isfile(baseline):
 650                 shutil.copyfile(baseline,
 651                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 652
 653             # Record in known apks, getting the added date at the same time..
 654             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 655             if added:
 656                 thisinfo['added'] = added
 657
 658             apkcache[apkfilename] = thisinfo
 659             cachechanged = True
 660
 661         apks.append(thisinfo)
 662
 663     return apks, cachechanged
 664
 665
 666 repo_pubkey_fingerprint = None
 667
 668
 669 # Generate a certificate fingerprint the same way keytool does it
 670 # (but with slightly different formatting)
 671 def cert_fingerprint(data):
 672     digest = hashlib.sha256(data).digest()
 673     ret = []
 674     ret.append(' '.join("%02X" % ord(b) for b in digest))
 675     return " ".join(ret)
 676
 677
 678 def extract_pubkey():
 679     global repo_pubkey_fingerprint
 680     if 'repo_pubkey' in config:
 681         pubkey = unhexlify(config['repo_pubkey'])
 682     else:
 683         p = FDroidPopen(['keytool', '-exportcert',
 684                          '-alias', config['repo_keyalias'],
 685                          '-keystore', config['keystore'],
 686                          '-storepass:file', config['keystorepassfile']]
 687                         + config['smartcardoptions'], output=False)
 688         if p.returncode != 0 or len(p.output) < 20:
 689             msg = "Failed to get repo pubkey!"
 690             if config['keystore'] == 'NONE':
 691                 msg += ' Is your crypto smartcard plugged in?'
 692             logging.critical(msg)
 693             sys.exit(1)
 694         pubkey = p.output
 695     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 696     return hexlify(pubkey)
 697
 698
 699 def make_index(apps, sortedids, apks, repodir, archive, categories):
 700     """Make a repo index.
 701
 702     :param apps: fully populated apps list
 703     :param apks: full populated apks list
 704     :param repodir: the repo directory
 705     :param archive: True if this is the archive repo, False if it's the
 706                     main one.
 707     :param categories: list of categories
 708     """
 709
 710     doc = Document()
 711
 712     def addElement(name, value, doc, parent):
 713         el = doc.createElement(name)
 714         el.appendChild(doc.createTextNode(value))
 715         parent.appendChild(el)
 716
 717     def addElementCDATA(name, value, doc, parent):
 718         el = doc.createElement(name)
 719         el.appendChild(doc.createCDATASection(value))
 720         parent.appendChild(el)
 721
 722     root = doc.createElement("fdroid")
 723     doc.appendChild(root)
 724
 725     repoel = doc.createElement("repo")
 726
 727     if archive:
 728         repoel.setAttribute("name", config['archive_name'])
 729         if config['repo_maxage'] != 0:
 730             repoel.setAttribute("maxage", str(config['repo_maxage']))
 731         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 732         repoel.setAttribute("url", config['archive_url'])
 733         addElement('description', config['archive_description'], doc, repoel)
 734
 735     else:
 736         repoel.setAttribute("name", config['repo_name'])
 737         if config['repo_maxage'] != 0:
 738             repoel.setAttribute("maxage", str(config['repo_maxage']))
 739         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 740         repoel.setAttribute("url", config['repo_url'])
 741         addElement('description', config['repo_description'], doc, repoel)
 742
 743     repoel.setAttribute("version", "12")
 744     repoel.setAttribute("timestamp", str(int(time.time())))
 745
 746     nosigningkey = False
 747     if not 'repo_keyalias' in config:
 748         nosigningkey = True
 749         logging.critical("'repo_keyalias' not found in config.py!")
 750     if not 'keystore' in config:
 751         nosigningkey = True
 752         logging.critical("'keystore' not found in config.py!")
 753     if not 'keystorepass' in config and not 'keystorepassfile' in config:
 754         nosigningkey = True
 755         logging.critical("'keystorepass' not found in config.py!")
 756     if not 'keypass' in config and not 'keypassfile' in config:
 757         nosigningkey = True
 758         logging.critical("'keypass' not found in config.py!")
 759     if not os.path.exists(config['keystore']):
 760         nosigningkey = True
 761         logging.critical("'" + config['keystore'] + "' does not exist!")
 762     if nosigningkey:
 763         logging.warning("`fdroid update` requires a signing key, you can create one using:")
 764         logging.warning("\tfdroid update --create-key")
 765         sys.exit(1)
 766
 767     repoel.setAttribute("pubkey", extract_pubkey())
 768     root.appendChild(repoel)
 769
 770     for appid in sortedids:
 771         app = apps[appid]
 772
 773         if app['Disabled'] is not None:
 774             continue
 775
 776         # Get a list of the apks for this app...
 777         apklist = []
 778         for apk in apks:
 779             if apk['id'] == appid:
 780                 apklist.append(apk)
 781
 782         if len(apklist) == 0:
 783             continue
 784
 785         apel = doc.createElement("application")
 786         apel.setAttribute("id", app['id'])
 787         root.appendChild(apel)
 788
 789         addElement('id', app['id'], doc, apel)
 790         if 'added' in app:
 791             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 792         if 'lastupdated' in app:
 793             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 794         addElement('name', app['Name'], doc, apel)
 795         addElement('summary', app['Summary'], doc, apel)
 796         if app['icon']:
 797             addElement('icon', app['icon'], doc, apel)
 798
 799         def linkres(appid):
 800             if appid in apps:
 801                 return ("fdroid.app:" + appid, apps[appid]['Name'])
 802             raise MetaDataException("Cannot resolve app id " + appid)
 803
 804         addElement('desc',
 805                    metadata.description_html(app['Description'], linkres),
 806                    doc, apel)
 807         addElement('license', app['License'], doc, apel)
 808         if 'Categories' in app:
 809             addElement('categories', ','.join(app["Categories"]), doc, apel)
 810             # We put the first (primary) category in LAST, which will have
 811             # the desired effect of making clients that only understand one
 812             # category see that one.
 813             addElement('category', app["Categories"][0], doc, apel)
 814         addElement('web', app['Web Site'], doc, apel)
 815         addElement('source', app['Source Code'], doc, apel)
 816         addElement('tracker', app['Issue Tracker'], doc, apel)
 817         if app['Donate']:
 818             addElement('donate', app['Donate'], doc, apel)
 819         if app['Bitcoin']:
 820             addElement('bitcoin', app['Bitcoin'], doc, apel)
 821         if app['Litecoin']:
 822             addElement('litecoin', app['Litecoin'], doc, apel)
 823         if app['Dogecoin']:
 824             addElement('dogecoin', app['Dogecoin'], doc, apel)
 825         if app['FlattrID']:
 826             addElement('flattr', app['FlattrID'], doc, apel)
 827
 828         # These elements actually refer to the current version (i.e. which
 829         # one is recommended. They are historically mis-named, and need
 830         # changing, but stay like this for now to support existing clients.
 831         addElement('marketversion', app['Current Version'], doc, apel)
 832         addElement('marketvercode', app['Current Version Code'], doc, apel)
 833
 834         if app['AntiFeatures']:
 835             af = app['AntiFeatures'].split(',')
 836             # TODO: Temporarily not including UpstreamNonFree in the index,
 837             # because current F-Droid clients do not understand it, and also
 838             # look ugly when they encounter an unknown antifeature. This
 839             # filtering can be removed in time...
 840             if 'UpstreamNonFree' in af:
 841                 af.remove('UpstreamNonFree')
 842             if af:
 843                 addElement('antifeatures', ','.join(af), doc, apel)
 844         if app['Provides']:
 845             pv = app['Provides'].split(',')
 846             addElement('provides', ','.join(pv), doc, apel)
 847         if app['Requires Root']:
 848             addElement('requirements', 'root', doc, apel)
 849
 850         # Sort the apk list into version order, just so the web site
 851         # doesn't have to do any work by default...
 852         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 853
 854         # Check for duplicates - they will make the client unhappy...
 855         for i in range(len(apklist) - 1):
 856             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 857                 logging.critical("duplicate versions: '%s' - '%s'" % (
 858                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 859                 sys.exit(1)
 860
 861         current_version_code = 0
 862         current_version_file = None
 863         for apk in apklist:
 864             # find the APK for the "Current Version"
 865             if current_version_code < apk['versioncode']:
 866                 current_version_code = apk['versioncode']
 867             if current_version_code < int(app['Current Version Code']):
 868                 current_version_file = apk['apkname']
 869
 870             apkel = doc.createElement("package")
 871             apel.appendChild(apkel)
 872             addElement('version', apk['version'], doc, apkel)
 873             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 874             addElement('apkname', apk['apkname'], doc, apkel)
 875             if 'srcname' in apk:
 876                 addElement('srcname', apk['srcname'], doc, apkel)
 877             for hash_type in ['sha256']:
 878                 if hash_type not in apk:
 879                     continue
 880                 hashel = doc.createElement("hash")
 881                 hashel.setAttribute("type", hash_type)
 882                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 883                 apkel.appendChild(hashel)
 884             addElement('sig', apk['sig'], doc, apkel)
 885             addElement('size', str(apk['size']), doc, apkel)
 886             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 887             if 'maxsdkversion' in apk:
 888                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 889             if 'added' in apk:
 890                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 891             if len(apk['permissions']) > 0:
 892                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 893             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 894                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 895             if len(apk['features']) > 0:
 896                 addElement('features', ','.join(apk['features']), doc, apkel)
 897
 898         if current_version_file is not None \
 899                 and config['make_current_version_link'] \
 900                 and repodir == 'repo':  # only create these
 901             sanitized_name = re.sub('''[ '"&%?+=/]''', '',
 902                                     app[config['current_version_name_source']])
 903             apklinkname = sanitized_name + '.apk'
 904             current_version_path = os.path.join(repodir, current_version_file)
 905             if os.path.exists(apklinkname):
 906                 os.remove(apklinkname)
 907             os.symlink(current_version_path, apklinkname)
 908             # also symlink gpg signature, if it exists
 909             for extension in ('.asc', '.sig'):
 910                 sigfile_path = current_version_path + extension
 911                 if os.path.exists(sigfile_path):
 912                     siglinkname = apklinkname + extension
 913                     if os.path.exists(siglinkname):
 914                         os.remove(siglinkname)
 915                     os.symlink(sigfile_path, siglinkname)
 916
 917     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 918     if options.pretty:
 919         output = doc.toprettyxml()
 920     else:
 921         output = doc.toxml()
 922     of.write(output)
 923     of.close()
 924
 925     if 'repo_keyalias' in config:
 926
 927         if options.nosign:
 928             logging.info("Creating unsigned index in preparation for signing")
 929         else:
 930             logging.info("Creating signed index with this key (SHA256):")
 931             logging.info("%s" % repo_pubkey_fingerprint)
 932
 933         # Create a jar of the index...
 934         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
 935         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 936         if p.returncode != 0:
 937             logging.critical("Failed to create {0}".format(jar_output))
 938             sys.exit(1)
 939
 940         # Sign the index...
 941         signed = os.path.join(repodir, 'index.jar')
 942         if options.nosign:
 943             # Remove old signed index if not signing
 944             if os.path.exists(signed):
 945                 os.remove(signed)
 946         else:
 947             args = ['jarsigner', '-keystore', config['keystore'],
 948                     '-storepass:file', config['keystorepassfile'],
 949                     '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 950                     signed, config['repo_keyalias']]
 951             if config['keystore'] == 'NONE':
 952                 args += config['smartcardoptions']
 953             else:  # smardcards never use -keypass
 954                 args += ['-keypass:file', config['keypassfile']]
 955             p = FDroidPopen(args)
 956             if p.returncode != 0:
 957                 logging.critical("Failed to sign index")
 958                 sys.exit(1)
 959
 960     # Copy the repo icon into the repo directory...
 961     icon_dir = os.path.join(repodir, 'icons')
 962     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 963     shutil.copyfile(config['repo_icon'], iconfilename)
 964
 965     # Write a category list in the repo to allow quick access...
 966     catdata = ''
 967     for cat in categories:
 968         catdata += cat + '\n'
 969     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 970     f.write(catdata)
 971     f.close()
 972
 973
 974 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 975
 976     for appid, app in apps.iteritems():
 977
 978         # Get a list of the apks for this app...
 979         apklist = []
 980         for apk in apks:
 981             if apk['id'] == appid:
 982                 apklist.append(apk)
 983
 984         # Sort the apk list into version order...
 985         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 986
 987         if app['Archive Policy']:
 988             keepversions = int(app['Archive Policy'][:-9])
 989         else:
 990             keepversions = defaultkeepversions
 991
 992         if len(apklist) > keepversions:
 993             for apk in apklist[keepversions:]:
 994                 logging.info("Moving " + apk['apkname'] + " to archive")
 995                 shutil.move(os.path.join(repodir, apk['apkname']),
 996                             os.path.join(archivedir, apk['apkname']))
 997                 if 'srcname' in apk:
 998                     shutil.move(os.path.join(repodir, apk['srcname']),
 999                                 os.path.join(archivedir, apk['srcname']))
1000                     # Move GPG signature too...
1001                     sigfile = apk['srcname'] + '.asc'
1002                     sigsrc = os.path.join(repodir, sigfile)
1003                     if os.path.exists(sigsrc):
1004                         shutil.move(sigsrc, os.path.join(archivedir, sigfile))
1005
1006                 archapks.append(apk)
1007                 apks.remove(apk)
1008
1009
1010 config = None
1011 options = None
1012
1013
1014 def main():
1015
1016     global config, options
1017
1018     # Parse command line...
1019     parser = OptionParser()
1020     parser.add_option("--create-key", action="store_true", default=False,
1021                       help="Create a repo signing key in a keystore")
1022     parser.add_option("-c", "--create-metadata", action="store_true", default=False,
1023                       help="Create skeleton metadata files that are missing")
1024     parser.add_option("--delete-unknown", action="store_true", default=False,
1025                       help="Delete APKs without metadata from the repo")
1026     parser.add_option("-v", "--verbose", action="store_true", default=False,
1027                       help="Spew out even more information than normal")
1028     parser.add_option("-q", "--quiet", action="store_true", default=False,
1029                       help="Restrict output to warnings and errors")
1030     parser.add_option("-b", "--buildreport", action="store_true", default=False,
1031                       help="Report on build data status")
1032     parser.add_option("-i", "--interactive", default=False, action="store_true",
1033                       help="Interactively ask about things that need updating.")
1034     parser.add_option("-I", "--icons", action="store_true", default=False,
1035                       help="Resize all the icons exceeding the max pixel size and exit")
1036     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
1037                       help="Specify editor to use in interactive mode. Default " +
1038                       "is /etc/alternatives/editor")
1039     parser.add_option("-w", "--wiki", default=False, action="store_true",
1040                       help="Update the wiki")
1041     parser.add_option("", "--pretty", action="store_true", default=False,
1042                       help="Produce human-readable index.xml")
1043     parser.add_option("--clean", action="store_true", default=False,
1044                       help="Clean update - don't uses caches, reprocess all apks")
1045     parser.add_option("--nosign", action="store_true", default=False,
1046                       help="When configured for signed indexes, create only unsigned indexes at this stage")
1047     (options, args) = parser.parse_args()
1048
1049     config = common.read_config(options)
1050
1051     repodirs = ['repo']
1052     if config['archive_older'] != 0:
1053         repodirs.append('archive')
1054         if not os.path.exists('archive'):
1055             os.mkdir('archive')
1056
1057     if options.icons:
1058         resize_all_icons(repodirs)
1059         sys.exit(0)
1060
1061     # check that icons exist now, rather than fail at the end of `fdroid update`
1062     for k in ['repo_icon', 'archive_icon']:
1063         if k in config:
1064             if not os.path.exists(config[k]):
1065                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1066                 sys.exit(1)
1067
1068     # if the user asks to create a keystore, do it now, reusing whatever it can
1069     if options.create_key:
1070         if os.path.exists(config['keystore']):
1071             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1072             logging.critical("\t'" + config['keystore'] + "'")
1073             sys.exit(1)
1074
1075         if not 'repo_keyalias' in config:
1076             config['repo_keyalias'] = socket.getfqdn()
1077             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1078         if not 'keydname' in config:
1079             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1080             common.write_to_config(config, 'keydname', config['keydname'])
1081         if not 'keystore' in config:
1082             config['keystore'] = common.default_config.keystore
1083             common.write_to_config(config, 'keystore', config['keystore'])
1084
1085         password = common.genpassword()
1086         if not 'keystorepass' in config:
1087             config['keystorepass'] = password
1088             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1089         if not 'keypass' in config:
1090             config['keypass'] = password
1091             common.write_to_config(config, 'keypass', config['keypass'])
1092         common.genkeystore(config)
1093
1094     # Get all apps...
1095     apps = metadata.read_metadata()
1096
1097     # Generate a list of categories...
1098     categories = set()
1099     for app in apps.itervalues():
1100         categories.update(app['Categories'])
1101
1102     # Read known apks data (will be updated and written back when we've finished)
1103     knownapks = common.KnownApks()
1104
1105     # Gather information about all the apk files in the repo directory, using
1106     # cached data if possible.
1107     apkcachefile = os.path.join('tmp', 'apkcache')
1108     if not options.clean and os.path.exists(apkcachefile):
1109         with open(apkcachefile, 'rb') as cf:
1110             apkcache = pickle.load(cf)
1111     else:
1112         apkcache = {}
1113     cachechanged = False
1114
1115     delete_disabled_builds(apps, apkcache, repodirs)
1116
1117     # Scan all apks in the main repo
1118     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
1119     if cc:
1120         cachechanged = True
1121
1122     # Generate warnings for apk's with no metadata (or create skeleton
1123     # metadata files, if requested on the command line)
1124     newmetadata = False
1125     for apk in apks:
1126         if apk['id'] not in apps:
1127             if options.create_metadata:
1128                 if 'name' not in apk:
1129                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1130                     continue
1131                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1132                 f.write("License:Unknown\n")
1133                 f.write("Web Site:\n")
1134                 f.write("Source Code:\n")
1135                 f.write("Issue Tracker:\n")
1136                 f.write("Summary:" + apk['name'] + "\n")
1137                 f.write("Description:\n")
1138                 f.write(apk['name'] + "\n")
1139                 f.write(".\n")
1140                 f.close()
1141                 logging.info("Generated skeleton metadata for " + apk['id'])
1142                 newmetadata = True
1143             else:
1144                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1145                 if options.delete_unknown:
1146                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1147                     rmf = os.path.join(repodirs[0], apk['apkname'])
1148                     if not os.path.exists(rmf):
1149                         logging.error("Could not find {0} to remove it".format(rmf))
1150                     else:
1151                         os.remove(rmf)
1152                 else:
1153                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1154
1155     # update the metadata with the newly created ones included
1156     if newmetadata:
1157         apps = metadata.read_metadata()
1158
1159     # Scan the archive repo for apks as well
1160     if len(repodirs) > 1:
1161         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1162         if cc:
1163             cachechanged = True
1164     else:
1165         archapks = []
1166
1167     # Some information from the apks needs to be applied up to the application
1168     # level. When doing this, we use the info from the most recent version's apk.
1169     # We deal with figuring out when the app was added and last updated at the
1170     # same time.
1171     for appid, app in apps.iteritems():
1172         bestver = 0
1173         added = None
1174         lastupdated = None
1175         for apk in apks + archapks:
1176             if apk['id'] == appid:
1177                 if apk['versioncode'] > bestver:
1178                     bestver = apk['versioncode']
1179                     bestapk = apk
1180
1181                 if 'added' in apk:
1182                     if not added or apk['added'] < added:
1183                         added = apk['added']
1184                     if not lastupdated or apk['added'] > lastupdated:
1185                         lastupdated = apk['added']
1186
1187         if added:
1188             app['added'] = added
1189         else:
1190             logging.warn("Don't know when " + appid + " was added")
1191         if lastupdated:
1192             app['lastupdated'] = lastupdated
1193         else:
1194             logging.warn("Don't know when " + appid + " was last updated")
1195
1196         if bestver == 0:
1197             if app['Name'] is None:
1198                 app['Name'] = app['Auto Name'] or appid
1199             app['icon'] = None
1200             logging.warn("Application " + appid + " has no packages")
1201         else:
1202             if app['Name'] is None:
1203                 app['Name'] = bestapk['name']
1204             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1205
1206     # Sort the app list by name, then the web site doesn't have to by default.
1207     # (we had to wait until we'd scanned the apks to do this, because mostly the
1208     # name comes from there!)
1209     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid]['Name'].upper())
1210
1211     if len(repodirs) > 1:
1212         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1213
1214     # Make the index for the main repo...
1215     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1216
1217     # If there's an archive repo,  make the index for it. We already scanned it
1218     # earlier on.
1219     if len(repodirs) > 1:
1220         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1221
1222     if config['update_stats']:
1223
1224         # Update known apks info...
1225         knownapks.writeifchanged()
1226
1227         # Generate latest apps data for widget
1228         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1229             data = ''
1230             for line in file(os.path.join('stats', 'latestapps.txt')):
1231                 appid = line.rstrip()
1232                 data += appid + "\t"
1233                 app = apps[appid]
1234                 data += app['Name'] + "\t"
1235                 if app['icon'] is not None:
1236                     data += app['icon'] + "\t"
1237                 data += app['License'] + "\n"
1238             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1239             f.write(data)
1240             f.close()
1241
1242     if cachechanged:
1243         with open(apkcachefile, 'wb') as cf:
1244             pickle.dump(apkcache, cf)
1245
1246     # Update the wiki...
1247     if options.wiki:
1248         update_wiki(apps, sortedids, apks + archapks)
1249
1250     logging.info("Finished.")
1251
1252 if __name__ == "__main__":
1253     main()