fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import socket
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from datetime import datetime, timedelta
  31 from xml.dom.minidom import Document
  32 from argparse import ArgumentParser
  33 import time
  34 from pyasn1.error import PyAsn1Error
  35 from pyasn1.codec.der import decoder, encoder
  36 from pyasn1_modules import rfc2315
  37 from hashlib import md5
  38 from binascii import hexlify, unhexlify
  39
  40 from PIL import Image
  41 import logging
  42
  43 import common
  44 import metadata
  45 from common import FDroidPopen, SdkToolsPopen
  46 from metadata import MetaDataException
  47
  48
  49 def get_densities():
  50     return ['640', '480', '320', '240', '160', '120']
  51
  52
  53 def dpi_to_px(density):
  54     return (int(density) * 48) / 160
  55
  56
  57 def px_to_dpi(px):
  58     return (int(px) * 160) / 48
  59
  60
  61 def get_icon_dir(repodir, density):
  62     if density is None:
  63         return os.path.join(repodir, "icons")
  64     return os.path.join(repodir, "icons-%s" % density)
  65
  66
  67 def get_icon_dirs(repodir):
  68     for density in get_densities():
  69         yield get_icon_dir(repodir, density)
  70     yield os.path.join(repodir, "icons")
  71
  72
  73 def update_wiki(apps, sortedids, apks):
  74     """Update the wiki
  75
  76     :param apps: fully populated list of all applications
  77     :param apks: all apks, except...
  78     """
  79     logging.info("Updating wiki")
  80     wikicat = 'Apps'
  81     wikiredircat = 'App Redirects'
  82     import mwclient
  83     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  84                          path=config['wiki_path'])
  85     site.login(config['wiki_user'], config['wiki_password'])
  86     generated_pages = {}
  87     generated_redirects = {}
  88
  89     for appid in sortedids:
  90         app = apps[appid]
  91
  92         wikidata = ''
  93         if app['Disabled']:
  94             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  95         if 'AntiFeatures' in app:
  96             for af in app['AntiFeatures']:
  97                 wikidata += '{{AntiFeature|' + af + '}}\n'
  98         if app['Requires Root']:
  99             requiresroot = 'Yes'
 100         else:
 101             requiresroot = 'No'
 102         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
 103             appid,
 104             app['Name'],
 105             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
 106             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
 107             app['Source Code'],
 108             app['Issue Tracker'],
 109             app['Web Site'],
 110             app['Changelog'],
 111             app['Donate'],
 112             app['FlattrID'],
 113             app['Bitcoin'],
 114             app['Litecoin'],
 115             app['Dogecoin'],
 116             app['License'],
 117             requiresroot)
 118
 119         if app['Provides']:
 120             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 121
 122         wikidata += app['Summary']
 123         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 124
 125         wikidata += "=Description=\n"
 126         wikidata += metadata.description_wiki(app['Description']) + "\n"
 127
 128         wikidata += "=Maintainer Notes=\n"
 129         if 'Maintainer Notes' in app:
 130             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 131         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 132
 133         # Get a list of all packages for this application...
 134         apklist = []
 135         gotcurrentver = False
 136         cantupdate = False
 137         buildfails = False
 138         for apk in apks:
 139             if apk['id'] == appid:
 140                 if str(apk['versioncode']) == app['Current Version Code']:
 141                     gotcurrentver = True
 142                 apklist.append(apk)
 143         # Include ones we can't build, as a special case...
 144         for thisbuild in app['builds']:
 145             if thisbuild['disable']:
 146                 if thisbuild['vercode'] == app['Current Version Code']:
 147                     cantupdate = True
 148                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 149                 apklist.append({'versioncode': int(thisbuild['vercode']),
 150                                 'version': thisbuild['version'],
 151                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(thisbuild['disable']),
 152                                 })
 153             else:
 154                 builtit = False
 155                 for apk in apklist:
 156                     if apk['versioncode'] == int(thisbuild['vercode']):
 157                         builtit = True
 158                         break
 159                 if not builtit:
 160                     buildfails = True
 161                     apklist.append({'versioncode': int(thisbuild['vercode']),
 162                                     'version': thisbuild['version'],
 163                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, thisbuild['vercode']),
 164                                     })
 165         if app['Current Version Code'] == '0':
 166             cantupdate = True
 167         # Sort with most recent first...
 168         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 169
 170         wikidata += "=Versions=\n"
 171         if len(apklist) == 0:
 172             wikidata += "We currently have no versions of this app available."
 173         elif not gotcurrentver:
 174             wikidata += "We don't have the current version of this app."
 175         else:
 176             wikidata += "We have the current version of this app."
 177         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 178         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 179         if len(app['No Source Since']) > 0:
 180             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 181         if len(app['Current Version']) > 0:
 182             wikidata += "The current (recommended) version is " + app['Current Version']
 183             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 184         validapks = 0
 185         for apk in apklist:
 186             wikidata += "==" + apk['version'] + "==\n"
 187
 188             if 'buildproblem' in apk:
 189                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 190             else:
 191                 validapks += 1
 192                 wikidata += "This version is built and signed by "
 193                 if 'srcname' in apk:
 194                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 195                 else:
 196                     wikidata += "the original developer.\n\n"
 197             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 198
 199         wikidata += '\n[[Category:' + wikicat + ']]\n'
 200         if len(app['No Source Since']) > 0:
 201             wikidata += '\n[[Category:Apps missing source code]]\n'
 202         if validapks == 0 and not app['Disabled']:
 203             wikidata += '\n[[Category:Apps with no packages]]\n'
 204         if cantupdate and not app['Disabled']:
 205             wikidata += "\n[[Category:Apps we can't update]]\n"
 206         if buildfails and not app['Disabled']:
 207             wikidata += "\n[[Category:Apps with failing builds]]\n"
 208         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 209             wikidata += '\n[[Category:Apps to Update]]\n'
 210         if app['Disabled']:
 211             wikidata += '\n[[Category:Apps that are disabled]]\n'
 212         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 213             wikidata += '\n[[Category:Apps with no update check]]\n'
 214         for appcat in app['Categories']:
 215             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 216
 217         # We can't have underscores in the page name, even if they're in
 218         # the package ID, because MediaWiki messes with them...
 219         pagename = appid.replace('_', ' ')
 220
 221         # Drop a trailing newline, because mediawiki is going to drop it anyway
 222         # and it we don't we'll think the page has changed when it hasn't...
 223         if wikidata.endswith('\n'):
 224             wikidata = wikidata[:-1]
 225
 226         generated_pages[pagename] = wikidata
 227
 228         # Make a redirect from the name to the ID too, unless there's
 229         # already an existing page with the name and it isn't a redirect.
 230         noclobber = False
 231         apppagename = app['Name'].replace('_', ' ')
 232         apppagename = apppagename.replace('{', '')
 233         apppagename = apppagename.replace('}', ' ')
 234         apppagename = apppagename.replace(':', ' ')
 235         # Drop double spaces caused mostly by replacing ':' above
 236         apppagename = apppagename.replace('  ', ' ')
 237         for expagename in site.allpages(prefix=apppagename,
 238                                         filterredir='nonredirects',
 239                                         generator=False):
 240             if expagename == apppagename:
 241                 noclobber = True
 242         # Another reason not to make the redirect page is if the app name
 243         # is the same as it's ID, because that will overwrite the real page
 244         # with an redirect to itself! (Although it seems like an odd
 245         # scenario this happens a lot, e.g. where there is metadata but no
 246         # builds or binaries to extract a name from.
 247         if apppagename == pagename:
 248             noclobber = True
 249         if not noclobber:
 250             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 251
 252     for tcat, genp in [(wikicat, generated_pages),
 253                        (wikiredircat, generated_redirects)]:
 254         catpages = site.Pages['Category:' + tcat]
 255         existingpages = []
 256         for page in catpages:
 257             existingpages.append(page.name)
 258             if page.name in genp:
 259                 pagetxt = page.edit()
 260                 if pagetxt != genp[page.name]:
 261                     logging.debug("Updating modified page " + page.name)
 262                     page.save(genp[page.name], summary='Auto-updated')
 263                 else:
 264                     logging.debug("Page " + page.name + " is unchanged")
 265             else:
 266                 logging.warn("Deleting page " + page.name)
 267                 page.delete('No longer published')
 268         for pagename, text in genp.items():
 269             logging.debug("Checking " + pagename)
 270             if pagename not in existingpages:
 271                 logging.debug("Creating page " + pagename)
 272                 try:
 273                     newpage = site.Pages[pagename]
 274                     newpage.save(text, summary='Auto-created')
 275                 except:
 276                     logging.error("...FAILED to create page '{0}'".format(pagename))
 277
 278     # Purge server cache to ensure counts are up to date
 279     site.pages['Repository Maintenance'].purge()
 280
 281
 282 def delete_disabled_builds(apps, apkcache, repodirs):
 283     """Delete disabled build outputs.
 284
 285     :param apps: list of all applications, as per metadata.read_metadata
 286     :param apkcache: current apk cache information
 287     :param repodirs: the repo directories to process
 288     """
 289     for appid, app in apps.iteritems():
 290         for build in app['builds']:
 291             if build['disable']:
 292                 apkfilename = appid + '_' + str(build['vercode']) + '.apk'
 293                 for repodir in repodirs:
 294                     apkpath = os.path.join(repodir, apkfilename)
 295                     ascpath = apkpath + ".asc"
 296                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 297                     for name in [apkpath, srcpath, ascpath]:
 298                         if os.path.exists(name):
 299                             logging.warn("Deleting disabled build output " + apkfilename)
 300                             os.remove(name)
 301                 if apkfilename in apkcache:
 302                     del apkcache[apkfilename]
 303
 304
 305 def resize_icon(iconpath, density):
 306
 307     if not os.path.isfile(iconpath):
 308         return
 309
 310     try:
 311         im = Image.open(iconpath)
 312         size = dpi_to_px(density)
 313
 314         if any(length > size for length in im.size):
 315             oldsize = im.size
 316             im.thumbnail((size, size), Image.ANTIALIAS)
 317             logging.debug("%s was too large at %s - new size is %s" % (
 318                 iconpath, oldsize, im.size))
 319             im.save(iconpath, "PNG")
 320
 321     except Exception, e:
 322         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 323
 324
 325 def resize_all_icons(repodirs):
 326     """Resize all icons that exceed the max size
 327
 328     :param repodirs: the repo directories to process
 329     """
 330     for repodir in repodirs:
 331         for density in get_densities():
 332             icon_dir = get_icon_dir(repodir, density)
 333             icon_glob = os.path.join(icon_dir, '*.png')
 334             for iconpath in glob.glob(icon_glob):
 335                 resize_icon(iconpath, density)
 336
 337
 338 # A signature block file with a .DSA, .RSA, or .EC extension
 339 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 340
 341
 342 def getsig(apkpath):
 343     """ Get the signing certificate of an apk. To get the same md5 has that
 344     Android gets, we encode the .RSA certificate in a specific format and pass
 345     it hex-encoded to the md5 digest algorithm.
 346
 347     :param apkpath: path to the apk
 348     :returns: A string containing the md5 of the signature of the apk or None
 349               if an error occurred.
 350     """
 351
 352     cert = None
 353
 354     # verify the jar signature is correct
 355     args = ['jarsigner', '-verify', apkpath]
 356     p = FDroidPopen(args)
 357     if p.returncode != 0:
 358         logging.critical(apkpath + " has a bad signature!")
 359         return None
 360
 361     with zipfile.ZipFile(apkpath, 'r') as apk:
 362
 363         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 364
 365         if len(certs) < 1:
 366             logging.error("Found no signing certificates on %s" % apkpath)
 367             return None
 368         if len(certs) > 1:
 369             logging.error("Found multiple signing certificates on %s" % apkpath)
 370             return None
 371
 372         cert = apk.read(certs[0])
 373
 374     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 375     if content.getComponentByName('contentType') != rfc2315.signedData:
 376         logging.error("Unexpected format.")
 377         return None
 378
 379     content = decoder.decode(content.getComponentByName('content'),
 380                              asn1Spec=rfc2315.SignedData())[0]
 381     try:
 382         certificates = content.getComponentByName('certificates')
 383     except PyAsn1Error:
 384         logging.error("Certificates not found.")
 385         return None
 386
 387     cert_encoded = encoder.encode(certificates)[4:]
 388
 389     return md5(cert_encoded.encode('hex')).hexdigest()
 390
 391
 392 def scan_apks(apps, apkcache, repodir, knownapks):
 393     """Scan the apks in the given repo directory.
 394
 395     This also extracts the icons.
 396
 397     :param apps: list of all applications, as per metadata.read_metadata
 398     :param apkcache: current apk cache information
 399     :param repodir: repo directory to scan
 400     :param knownapks: known apks info
 401     :returns: (apks, cachechanged) where apks is a list of apk information,
 402               and cachechanged is True if the apkcache got changed.
 403     """
 404
 405     cachechanged = False
 406
 407     icon_dirs = get_icon_dirs(repodir)
 408     for icon_dir in icon_dirs:
 409         if os.path.exists(icon_dir):
 410             if options.clean:
 411                 shutil.rmtree(icon_dir)
 412                 os.makedirs(icon_dir)
 413         else:
 414             os.makedirs(icon_dir)
 415
 416     apks = []
 417     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 418     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 419     vername_pat = re.compile(".*versionName='([^']*)'.*")
 420     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 421     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 422     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 423     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 424     string_pat = re.compile(".*'([^']*)'.*")
 425     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 426
 427         apkfilename = apkfile[len(repodir) + 1:]
 428         if ' ' in apkfilename:
 429             logging.critical("Spaces in filenames are not allowed.")
 430             sys.exit(1)
 431
 432         # Calculate the sha256...
 433         sha = hashlib.sha256()
 434         with open(apkfile, 'rb') as f:
 435             while True:
 436                 t = f.read(16384)
 437                 if len(t) == 0:
 438                     break
 439                 sha.update(t)
 440             shasum = sha.hexdigest()
 441
 442         usecache = False
 443         if apkfilename in apkcache:
 444             thisinfo = apkcache[apkfilename]
 445             if thisinfo['sha256'] == shasum:
 446                 logging.debug("Reading " + apkfilename + " from cache")
 447                 usecache = True
 448             else:
 449                 logging.debug("Ignoring stale cache data for " + apkfilename)
 450
 451         if not usecache:
 452             logging.debug("Processing " + apkfilename)
 453             thisinfo = {}
 454             thisinfo['apkname'] = apkfilename
 455             thisinfo['sha256'] = shasum
 456             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 457             if os.path.exists(os.path.join(repodir, srcfilename)):
 458                 thisinfo['srcname'] = srcfilename
 459             thisinfo['size'] = os.path.getsize(apkfile)
 460             thisinfo['permissions'] = set()
 461             thisinfo['features'] = set()
 462             thisinfo['icons_src'] = {}
 463             thisinfo['icons'] = {}
 464             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 465             if p.returncode != 0:
 466                 if options.delete_unknown:
 467                     if os.path.exists(apkfile):
 468                         logging.error("Failed to get apk information, deleting " + apkfile)
 469                         os.remove(apkfile)
 470                     else:
 471                         logging.error("Could not find {0} to remove it".format(apkfile))
 472                 else:
 473                     logging.error("Failed to get apk information, skipping " + apkfile)
 474                 continue
 475             for line in p.output.splitlines():
 476                 if line.startswith("package:"):
 477                     try:
 478                         thisinfo['id'] = re.match(name_pat, line).group(1)
 479                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 480                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 481                     except Exception, e:
 482                         logging.error("Package matching failed: " + str(e))
 483                         logging.info("Line was: " + line)
 484                         sys.exit(1)
 485                 elif line.startswith("application:"):
 486                     thisinfo['name'] = re.match(label_pat, line).group(1)
 487                     # Keep path to non-dpi icon in case we need it
 488                     match = re.match(icon_pat_nodpi, line)
 489                     if match:
 490                         thisinfo['icons_src']['-1'] = match.group(1)
 491                 elif line.startswith("launchable-activity:"):
 492                     # Only use launchable-activity as fallback to application
 493                     if not thisinfo['name']:
 494                         thisinfo['name'] = re.match(label_pat, line).group(1)
 495                     if '-1' not in thisinfo['icons_src']:
 496                         match = re.match(icon_pat_nodpi, line)
 497                         if match:
 498                             thisinfo['icons_src']['-1'] = match.group(1)
 499                 elif line.startswith("application-icon-"):
 500                     match = re.match(icon_pat, line)
 501                     if match:
 502                         density = match.group(1)
 503                         path = match.group(2)
 504                         thisinfo['icons_src'][density] = path
 505                 elif line.startswith("sdkVersion:"):
 506                     m = re.match(sdkversion_pat, line)
 507                     if m is None:
 508                         logging.error(line.replace('sdkVersion:', '')
 509                                       + ' is not a valid minSdkVersion!')
 510                     else:
 511                         thisinfo['sdkversion'] = m.group(1)
 512                 elif line.startswith("maxSdkVersion:"):
 513                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 514                 elif line.startswith("native-code:"):
 515                     thisinfo['nativecode'] = []
 516                     for arch in line[13:].split(' '):
 517                         thisinfo['nativecode'].append(arch[1:-1])
 518                 elif line.startswith("uses-permission:"):
 519                     perm = re.match(string_pat, line).group(1)
 520                     if perm.startswith("android.permission."):
 521                         perm = perm[19:]
 522                     thisinfo['permissions'].add(perm)
 523                 elif line.startswith("uses-feature:"):
 524                     perm = re.match(string_pat, line).group(1)
 525                     # Filter out this, it's only added with the latest SDK tools and
 526                     # causes problems for lots of apps.
 527                     if perm != "android.hardware.screen.portrait" \
 528                             and perm != "android.hardware.screen.landscape":
 529                         if perm.startswith("android.feature."):
 530                             perm = perm[16:]
 531                         thisinfo['features'].add(perm)
 532
 533             if 'sdkversion' not in thisinfo:
 534                 logging.warn("No SDK version information found in {0}".format(apkfile))
 535                 thisinfo['sdkversion'] = 0
 536
 537             # Check for debuggable apks...
 538             if common.isApkDebuggable(apkfile, config):
 539                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 540
 541             # Get the signature (or md5 of, to be precise)...
 542             logging.debug('Getting signature of {0}'.format(apkfile))
 543             thisinfo['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 544             if not thisinfo['sig']:
 545                 logging.critical("Failed to get apk signature")
 546                 sys.exit(1)
 547
 548             apk = zipfile.ZipFile(apkfile, 'r')
 549
 550             # if an APK has files newer than the system time, suggest updating
 551             # the system clock.  This is useful for offline systems, used for
 552             # signing, which do not have another source of clock sync info. It
 553             # has to be more than 24 hours newer because ZIP/APK files do not
 554             # store timezone info
 555             info = apk.getinfo('AndroidManifest.xml')
 556             dt_obj = datetime(*info.date_time)
 557             checkdt = dt_obj - timedelta(1)
 558             if datetime.today() < checkdt:
 559                 logging.warn('System clock is older than manifest in: '
 560                              + apkfilename + '\nSet clock to that time using:\n'
 561                              + 'sudo date -s "' + str(dt_obj) + '"')
 562
 563             iconfilename = "%s.%s.png" % (
 564                 thisinfo['id'],
 565                 thisinfo['versioncode'])
 566
 567             # Extract the icon file...
 568             densities = get_densities()
 569             empty_densities = []
 570             for density in densities:
 571                 if density not in thisinfo['icons_src']:
 572                     empty_densities.append(density)
 573                     continue
 574                 iconsrc = thisinfo['icons_src'][density]
 575                 icon_dir = get_icon_dir(repodir, density)
 576                 icondest = os.path.join(icon_dir, iconfilename)
 577
 578                 try:
 579                     with open(icondest, 'wb') as f:
 580                         f.write(apk.read(iconsrc))
 581                     thisinfo['icons'][density] = iconfilename
 582
 583                 except:
 584                     logging.warn("Error retrieving icon file")
 585                     del thisinfo['icons'][density]
 586                     del thisinfo['icons_src'][density]
 587                     empty_densities.append(density)
 588
 589             if '-1' in thisinfo['icons_src']:
 590                 iconsrc = thisinfo['icons_src']['-1']
 591                 iconpath = os.path.join(
 592                     get_icon_dir(repodir, None), iconfilename)
 593                 with open(iconpath, 'wb') as f:
 594                     f.write(apk.read(iconsrc))
 595                 try:
 596                     im = Image.open(iconpath)
 597                     dpi = px_to_dpi(im.size[0])
 598                     for density in densities:
 599                         if density in thisinfo['icons']:
 600                             break
 601                         if density == densities[-1] or dpi >= int(density):
 602                             thisinfo['icons'][density] = iconfilename
 603                             shutil.move(iconpath,
 604                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 605                             empty_densities.remove(density)
 606                             break
 607                 except Exception, e:
 608                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 609
 610             if thisinfo['icons']:
 611                 thisinfo['icon'] = iconfilename
 612
 613             apk.close()
 614
 615             # First try resizing down to not lose quality
 616             last_density = None
 617             for density in densities:
 618                 if density not in empty_densities:
 619                     last_density = density
 620                     continue
 621                 if last_density is None:
 622                     continue
 623                 logging.debug("Density %s not available, resizing down from %s"
 624                               % (density, last_density))
 625
 626                 last_iconpath = os.path.join(
 627                     get_icon_dir(repodir, last_density), iconfilename)
 628                 iconpath = os.path.join(
 629                     get_icon_dir(repodir, density), iconfilename)
 630                 try:
 631                     im = Image.open(last_iconpath)
 632                 except:
 633                     logging.warn("Invalid image file at %s" % last_iconpath)
 634                     continue
 635
 636                 size = dpi_to_px(density)
 637
 638                 im.thumbnail((size, size), Image.ANTIALIAS)
 639                 im.save(iconpath, "PNG")
 640                 empty_densities.remove(density)
 641
 642             # Then just copy from the highest resolution available
 643             last_density = None
 644             for density in reversed(densities):
 645                 if density not in empty_densities:
 646                     last_density = density
 647                     continue
 648                 if last_density is None:
 649                     continue
 650                 logging.debug("Density %s not available, copying from lower density %s"
 651                               % (density, last_density))
 652
 653                 shutil.copyfile(
 654                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 655                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 656
 657                 empty_densities.remove(density)
 658
 659             for density in densities:
 660                 icon_dir = get_icon_dir(repodir, density)
 661                 icondest = os.path.join(icon_dir, iconfilename)
 662                 resize_icon(icondest, density)
 663
 664             # Copy from icons-mdpi to icons since mdpi is the baseline density
 665             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 666             if os.path.isfile(baseline):
 667                 shutil.copyfile(baseline,
 668                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 669
 670             # Record in known apks, getting the added date at the same time..
 671             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 672             if added:
 673                 thisinfo['added'] = added
 674
 675             apkcache[apkfilename] = thisinfo
 676             cachechanged = True
 677
 678         apks.append(thisinfo)
 679
 680     return apks, cachechanged
 681
 682
 683 repo_pubkey_fingerprint = None
 684
 685
 686 # Generate a certificate fingerprint the same way keytool does it
 687 # (but with slightly different formatting)
 688 def cert_fingerprint(data):
 689     digest = hashlib.sha256(data).digest()
 690     ret = []
 691     ret.append(' '.join("%02X" % ord(b) for b in digest))
 692     return " ".join(ret)
 693
 694
 695 def extract_pubkey():
 696     global repo_pubkey_fingerprint
 697     if 'repo_pubkey' in config:
 698         pubkey = unhexlify(config['repo_pubkey'])
 699     else:
 700         p = FDroidPopen(['keytool', '-exportcert',
 701                          '-alias', config['repo_keyalias'],
 702                          '-keystore', config['keystore'],
 703                          '-storepass:file', config['keystorepassfile']]
 704                         + config['smartcardoptions'], output=False)
 705         if p.returncode != 0 or len(p.output) < 20:
 706             msg = "Failed to get repo pubkey!"
 707             if config['keystore'] == 'NONE':
 708                 msg += ' Is your crypto smartcard plugged in?'
 709             logging.critical(msg)
 710             sys.exit(1)
 711         pubkey = p.output
 712     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 713     return hexlify(pubkey)
 714
 715
 716 def make_index(apps, sortedids, apks, repodir, archive, categories):
 717     """Make a repo index.
 718
 719     :param apps: fully populated apps list
 720     :param apks: full populated apks list
 721     :param repodir: the repo directory
 722     :param archive: True if this is the archive repo, False if it's the
 723                     main one.
 724     :param categories: list of categories
 725     """
 726
 727     doc = Document()
 728
 729     def addElement(name, value, doc, parent):
 730         el = doc.createElement(name)
 731         el.appendChild(doc.createTextNode(value))
 732         parent.appendChild(el)
 733
 734     def addElementNonEmpty(name, value, doc, parent):
 735         if not value:
 736             return
 737         addElement(name, value, doc, parent)
 738
 739     def addElementCDATA(name, value, doc, parent):
 740         el = doc.createElement(name)
 741         el.appendChild(doc.createCDATASection(value))
 742         parent.appendChild(el)
 743
 744     root = doc.createElement("fdroid")
 745     doc.appendChild(root)
 746
 747     repoel = doc.createElement("repo")
 748
 749     if archive:
 750         repoel.setAttribute("name", config['archive_name'])
 751         if config['repo_maxage'] != 0:
 752             repoel.setAttribute("maxage", str(config['repo_maxage']))
 753         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 754         repoel.setAttribute("url", config['archive_url'])
 755         addElement('description', config['archive_description'], doc, repoel)
 756
 757     else:
 758         repoel.setAttribute("name", config['repo_name'])
 759         if config['repo_maxage'] != 0:
 760             repoel.setAttribute("maxage", str(config['repo_maxage']))
 761         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 762         repoel.setAttribute("url", config['repo_url'])
 763         addElement('description', config['repo_description'], doc, repoel)
 764
 765     repoel.setAttribute("version", "13")
 766     repoel.setAttribute("timestamp", str(int(time.time())))
 767
 768     nosigningkey = False
 769     if not options.nosign:
 770         if 'repo_keyalias' not in config:
 771             nosigningkey = True
 772             logging.critical("'repo_keyalias' not found in config.py!")
 773         if 'keystore' not in config:
 774             nosigningkey = True
 775             logging.critical("'keystore' not found in config.py!")
 776         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 777             nosigningkey = True
 778             logging.critical("'keystorepass' not found in config.py!")
 779         if 'keypass' not in config and 'keypassfile' not in config:
 780             nosigningkey = True
 781             logging.critical("'keypass' not found in config.py!")
 782         if not os.path.exists(config['keystore']):
 783             nosigningkey = True
 784             logging.critical("'" + config['keystore'] + "' does not exist!")
 785         if nosigningkey:
 786             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 787             logging.warning("\tfdroid update --create-key")
 788             sys.exit(1)
 789
 790     repoel.setAttribute("pubkey", extract_pubkey())
 791     root.appendChild(repoel)
 792
 793     for appid in sortedids:
 794         app = apps[appid]
 795
 796         if app['Disabled'] is not None:
 797             continue
 798
 799         # Get a list of the apks for this app...
 800         apklist = []
 801         for apk in apks:
 802             if apk['id'] == appid:
 803                 apklist.append(apk)
 804
 805         if len(apklist) == 0:
 806             continue
 807
 808         apel = doc.createElement("application")
 809         apel.setAttribute("id", app['id'])
 810         root.appendChild(apel)
 811
 812         addElement('id', app['id'], doc, apel)
 813         if 'added' in app:
 814             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 815         if 'lastupdated' in app:
 816             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 817         addElement('name', app['Name'], doc, apel)
 818         addElement('summary', app['Summary'], doc, apel)
 819         if app['icon']:
 820             addElement('icon', app['icon'], doc, apel)
 821
 822         def linkres(appid):
 823             if appid in apps:
 824                 return ("fdroid.app:" + appid, apps[appid]['Name'])
 825             raise MetaDataException("Cannot resolve app id " + appid)
 826
 827         addElement('desc',
 828                    metadata.description_html(app['Description'], linkres),
 829                    doc, apel)
 830         addElement('license', app['License'], doc, apel)
 831         if 'Categories' in app and app['Categories']:
 832             addElement('categories', ','.join(app["Categories"]), doc, apel)
 833             # We put the first (primary) category in LAST, which will have
 834             # the desired effect of making clients that only understand one
 835             # category see that one.
 836             addElement('category', app["Categories"][0], doc, apel)
 837         addElement('web', app['Web Site'], doc, apel)
 838         addElement('source', app['Source Code'], doc, apel)
 839         addElement('tracker', app['Issue Tracker'], doc, apel)
 840         addElementNonEmpty('changelog', app['Changelog'], doc, apel)
 841         addElementNonEmpty('donate', app['Donate'], doc, apel)
 842         addElementNonEmpty('bitcoin', app['Bitcoin'], doc, apel)
 843         addElementNonEmpty('litecoin', app['Litecoin'], doc, apel)
 844         addElementNonEmpty('dogecoin', app['Dogecoin'], doc, apel)
 845         addElementNonEmpty('flattr', app['FlattrID'], doc, apel)
 846
 847         # These elements actually refer to the current version (i.e. which
 848         # one is recommended. They are historically mis-named, and need
 849         # changing, but stay like this for now to support existing clients.
 850         addElement('marketversion', app['Current Version'], doc, apel)
 851         addElement('marketvercode', app['Current Version Code'], doc, apel)
 852
 853         if app['AntiFeatures']:
 854             af = app['AntiFeatures']
 855             if af:
 856                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 857         if app['Provides']:
 858             pv = app['Provides'].split(',')
 859             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 860         if app['Requires Root']:
 861             addElement('requirements', 'root', doc, apel)
 862
 863         # Sort the apk list into version order, just so the web site
 864         # doesn't have to do any work by default...
 865         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 866
 867         # Check for duplicates - they will make the client unhappy...
 868         for i in range(len(apklist) - 1):
 869             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 870                 logging.critical("duplicate versions: '%s' - '%s'" % (
 871                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 872                 sys.exit(1)
 873
 874         current_version_code = 0
 875         current_version_file = None
 876         for apk in apklist:
 877             # find the APK for the "Current Version"
 878             if current_version_code < apk['versioncode']:
 879                 current_version_code = apk['versioncode']
 880             if current_version_code < int(app['Current Version Code']):
 881                 current_version_file = apk['apkname']
 882
 883             apkel = doc.createElement("package")
 884             apel.appendChild(apkel)
 885             addElement('version', apk['version'], doc, apkel)
 886             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 887             addElement('apkname', apk['apkname'], doc, apkel)
 888             if 'srcname' in apk:
 889                 addElement('srcname', apk['srcname'], doc, apkel)
 890             for hash_type in ['sha256']:
 891                 if hash_type not in apk:
 892                     continue
 893                 hashel = doc.createElement("hash")
 894                 hashel.setAttribute("type", hash_type)
 895                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 896                 apkel.appendChild(hashel)
 897             addElement('sig', apk['sig'], doc, apkel)
 898             addElement('size', str(apk['size']), doc, apkel)
 899             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 900             if 'maxsdkversion' in apk:
 901                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 902             if 'added' in apk:
 903                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 904             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
 905             if 'nativecode' in apk:
 906                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 907             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
 908
 909         if current_version_file is not None \
 910                 and config['make_current_version_link'] \
 911                 and repodir == 'repo':  # only create these
 912             sanitized_name = re.sub('''[ '"&%?+=/]''', '',
 913                                     app[config['current_version_name_source']])
 914             apklinkname = sanitized_name + '.apk'
 915             current_version_path = os.path.join(repodir, current_version_file)
 916             if os.path.exists(apklinkname):
 917                 os.remove(apklinkname)
 918             os.symlink(current_version_path, apklinkname)
 919             # also symlink gpg signature, if it exists
 920             for extension in ('.asc', '.sig'):
 921                 sigfile_path = current_version_path + extension
 922                 if os.path.exists(sigfile_path):
 923                     siglinkname = apklinkname + extension
 924                     if os.path.exists(siglinkname):
 925                         os.remove(siglinkname)
 926                     os.symlink(sigfile_path, siglinkname)
 927
 928     if options.pretty:
 929         output = doc.toprettyxml()
 930     else:
 931         output = doc.toxml()
 932
 933     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 934         f.write(output)
 935
 936     if 'repo_keyalias' in config:
 937
 938         if options.nosign:
 939             logging.info("Creating unsigned index in preparation for signing")
 940         else:
 941             logging.info("Creating signed index with this key (SHA256):")
 942             logging.info("%s" % repo_pubkey_fingerprint)
 943
 944         # Create a jar of the index...
 945         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
 946         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 947         if p.returncode != 0:
 948             logging.critical("Failed to create {0}".format(jar_output))
 949             sys.exit(1)
 950
 951         # Sign the index...
 952         signed = os.path.join(repodir, 'index.jar')
 953         if options.nosign:
 954             # Remove old signed index if not signing
 955             if os.path.exists(signed):
 956                 os.remove(signed)
 957         else:
 958             args = ['jarsigner', '-keystore', config['keystore'],
 959                     '-storepass:file', config['keystorepassfile'],
 960                     '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 961                     signed, config['repo_keyalias']]
 962             if config['keystore'] == 'NONE':
 963                 args += config['smartcardoptions']
 964             else:  # smardcards never use -keypass
 965                 args += ['-keypass:file', config['keypassfile']]
 966             p = FDroidPopen(args)
 967             if p.returncode != 0:
 968                 logging.critical("Failed to sign index")
 969                 sys.exit(1)
 970
 971     # Copy the repo icon into the repo directory...
 972     icon_dir = os.path.join(repodir, 'icons')
 973     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 974     shutil.copyfile(config['repo_icon'], iconfilename)
 975
 976     # Write a category list in the repo to allow quick access...
 977     catdata = ''
 978     for cat in categories:
 979         catdata += cat + '\n'
 980     with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
 981         f.write(catdata)
 982
 983
 984 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 985
 986     for appid, app in apps.iteritems():
 987
 988         # Get a list of the apks for this app...
 989         apklist = []
 990         for apk in apks:
 991             if apk['id'] == appid:
 992                 apklist.append(apk)
 993
 994         # Sort the apk list into version order...
 995         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 996
 997         if app['Archive Policy']:
 998             keepversions = int(app['Archive Policy'][:-9])
 999         else:
1000             keepversions = defaultkeepversions
1001
1002         if len(apklist) > keepversions:
1003             for apk in apklist[keepversions:]:
1004                 logging.info("Moving " + apk['apkname'] + " to archive")
1005                 shutil.move(os.path.join(repodir, apk['apkname']),
1006                             os.path.join(archivedir, apk['apkname']))
1007                 if 'srcname' in apk:
1008                     shutil.move(os.path.join(repodir, apk['srcname']),
1009                                 os.path.join(archivedir, apk['srcname']))
1010                     # Move GPG signature too...
1011                     sigfile = apk['srcname'] + '.asc'
1012                     sigsrc = os.path.join(repodir, sigfile)
1013                     if os.path.exists(sigsrc):
1014                         shutil.move(sigsrc, os.path.join(archivedir, sigfile))
1015
1016                 archapks.append(apk)
1017                 apks.remove(apk)
1018
1019
1020 def add_apks_to_per_app_repos(repodir, apks):
1021     apks_per_app = dict()
1022     for apk in apks:
1023         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1024         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1025         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1026         apks_per_app[apk['id']] = apk
1027
1028         if not os.path.exists(apk['per_app_icons']):
1029             logging.info('Adding new repo for only ' + apk['id'])
1030             os.makedirs(apk['per_app_icons'])
1031
1032         apkpath = os.path.join(repodir, apk['apkname'])
1033         shutil.copy(apkpath, apk['per_app_repo'])
1034         apksigpath = apkpath + '.sig'
1035         if os.path.exists(apksigpath):
1036             shutil.copy(apksigpath, apk['per_app_repo'])
1037         apkascpath = apkpath + '.asc'
1038         if os.path.exists(apkascpath):
1039             shutil.copy(apkascpath, apk['per_app_repo'])
1040
1041
1042 config = None
1043 options = None
1044
1045
1046 def main():
1047
1048     global config, options
1049
1050     # Parse command line...
1051     parser = ArgumentParser()
1052     common.setup_global_opts(parser)
1053     parser.add_argument("--create-key", action="store_true", default=False,
1054                         help="Create a repo signing key in a keystore")
1055     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1056                         help="Create skeleton metadata files that are missing")
1057     parser.add_argument("--delete-unknown", action="store_true", default=False,
1058                         help="Delete APKs without metadata from the repo")
1059     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1060                         help="Report on build data status")
1061     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1062                         help="Interactively ask about things that need updating.")
1063     parser.add_argument("-I", "--icons", action="store_true", default=False,
1064                         help="Resize all the icons exceeding the max pixel size and exit")
1065     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1066                         help="Specify editor to use in interactive mode. Default " +
1067                         "is /etc/alternatives/editor")
1068     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1069                         help="Update the wiki")
1070     parser.add_argument("--pretty", action="store_true", default=False,
1071                         help="Produce human-readable index.xml")
1072     parser.add_argument("--clean", action="store_true", default=False,
1073                         help="Clean update - don't uses caches, reprocess all apks")
1074     parser.add_argument("--nosign", action="store_true", default=False,
1075                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1076     options = parser.parse_args()
1077
1078     config = common.read_config(options)
1079
1080     repodirs = ['repo']
1081     if config['archive_older'] != 0:
1082         repodirs.append('archive')
1083         if not os.path.exists('archive'):
1084             os.mkdir('archive')
1085
1086     if options.icons:
1087         resize_all_icons(repodirs)
1088         sys.exit(0)
1089
1090     # check that icons exist now, rather than fail at the end of `fdroid update`
1091     for k in ['repo_icon', 'archive_icon']:
1092         if k in config:
1093             if not os.path.exists(config[k]):
1094                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1095                 sys.exit(1)
1096
1097     # if the user asks to create a keystore, do it now, reusing whatever it can
1098     if options.create_key:
1099         if os.path.exists(config['keystore']):
1100             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1101             logging.critical("\t'" + config['keystore'] + "'")
1102             sys.exit(1)
1103
1104         if 'repo_keyalias' not in config:
1105             config['repo_keyalias'] = socket.getfqdn()
1106             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1107         if 'keydname' not in config:
1108             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1109             common.write_to_config(config, 'keydname', config['keydname'])
1110         if 'keystore' not in config:
1111             config['keystore'] = common.default_config.keystore
1112             common.write_to_config(config, 'keystore', config['keystore'])
1113
1114         password = common.genpassword()
1115         if 'keystorepass' not in config:
1116             config['keystorepass'] = password
1117             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1118         if 'keypass' not in config:
1119             config['keypass'] = password
1120             common.write_to_config(config, 'keypass', config['keypass'])
1121         common.genkeystore(config)
1122
1123     # Get all apps...
1124     apps = metadata.read_metadata()
1125
1126     # Generate a list of categories...
1127     categories = set()
1128     for app in apps.itervalues():
1129         categories.update(app['Categories'])
1130
1131     # Read known apks data (will be updated and written back when we've finished)
1132     knownapks = common.KnownApks()
1133
1134     # Gather information about all the apk files in the repo directory, using
1135     # cached data if possible.
1136     apkcachefile = os.path.join('tmp', 'apkcache')
1137     if not options.clean and os.path.exists(apkcachefile):
1138         with open(apkcachefile, 'rb') as cf:
1139             apkcache = pickle.load(cf)
1140     else:
1141         apkcache = {}
1142
1143     delete_disabled_builds(apps, apkcache, repodirs)
1144
1145     # Scan all apks in the main repo
1146     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks)
1147
1148     # Generate warnings for apk's with no metadata (or create skeleton
1149     # metadata files, if requested on the command line)
1150     newmetadata = False
1151     for apk in apks:
1152         if apk['id'] not in apps:
1153             if options.create_metadata:
1154                 if 'name' not in apk:
1155                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1156                     continue
1157                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1158                 f.write("License:Unknown\n")
1159                 f.write("Web Site:\n")
1160                 f.write("Source Code:\n")
1161                 f.write("Issue Tracker:\n")
1162                 f.write("Changelog:\n")
1163                 f.write("Summary:" + apk['name'] + "\n")
1164                 f.write("Description:\n")
1165                 f.write(apk['name'] + "\n")
1166                 f.write(".\n")
1167                 f.close()
1168                 logging.info("Generated skeleton metadata for " + apk['id'])
1169                 newmetadata = True
1170             else:
1171                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1172                 if options.delete_unknown:
1173                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1174                     rmf = os.path.join(repodirs[0], apk['apkname'])
1175                     if not os.path.exists(rmf):
1176                         logging.error("Could not find {0} to remove it".format(rmf))
1177                     else:
1178                         os.remove(rmf)
1179                 else:
1180                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1181
1182     # update the metadata with the newly created ones included
1183     if newmetadata:
1184         apps = metadata.read_metadata()
1185
1186     # Scan the archive repo for apks as well
1187     if len(repodirs) > 1:
1188         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1189         if cc:
1190             cachechanged = True
1191     else:
1192         archapks = []
1193
1194     # Some information from the apks needs to be applied up to the application
1195     # level. When doing this, we use the info from the most recent version's apk.
1196     # We deal with figuring out when the app was added and last updated at the
1197     # same time.
1198     for appid, app in apps.iteritems():
1199         bestver = 0
1200         added = None
1201         lastupdated = None
1202         for apk in apks + archapks:
1203             if apk['id'] == appid:
1204                 if apk['versioncode'] > bestver:
1205                     bestver = apk['versioncode']
1206                     bestapk = apk
1207
1208                 if 'added' in apk:
1209                     if not added or apk['added'] < added:
1210                         added = apk['added']
1211                     if not lastupdated or apk['added'] > lastupdated:
1212                         lastupdated = apk['added']
1213
1214         if added:
1215             app['added'] = added
1216         else:
1217             logging.warn("Don't know when " + appid + " was added")
1218         if lastupdated:
1219             app['lastupdated'] = lastupdated
1220         else:
1221             logging.warn("Don't know when " + appid + " was last updated")
1222
1223         if bestver == 0:
1224             if app['Name'] is None:
1225                 app['Name'] = app['Auto Name'] or appid
1226             app['icon'] = None
1227             logging.warn("Application " + appid + " has no packages")
1228         else:
1229             if app['Name'] is None:
1230                 app['Name'] = bestapk['name']
1231             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1232
1233     # Sort the app list by name, then the web site doesn't have to by default.
1234     # (we had to wait until we'd scanned the apks to do this, because mostly the
1235     # name comes from there!)
1236     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid]['Name'].upper())
1237
1238     # APKs are placed into multiple repos based on the app package, providing
1239     # per-app subscription feeds for nightly builds and things like it
1240     if config['per_app_repos']:
1241         add_apks_to_per_app_repos(repodirs[0], apks)
1242         for appid, app in apps.iteritems():
1243             repodir = os.path.join(appid, 'fdroid', 'repo')
1244             appdict = dict()
1245             appdict[appid] = app
1246             if os.path.isdir(repodir):
1247                 make_index(appdict, [appid], apks, repodir, False, categories)
1248             else:
1249                 logging.info('Skipping index generation for ' + appid)
1250         return
1251
1252     if len(repodirs) > 1:
1253         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1254
1255     # Make the index for the main repo...
1256     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1257
1258     # If there's an archive repo,  make the index for it. We already scanned it
1259     # earlier on.
1260     if len(repodirs) > 1:
1261         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1262
1263     if config['update_stats']:
1264
1265         # Update known apks info...
1266         knownapks.writeifchanged()
1267
1268         # Generate latest apps data for widget
1269         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1270             data = ''
1271             for line in file(os.path.join('stats', 'latestapps.txt')):
1272                 appid = line.rstrip()
1273                 data += appid + "\t"
1274                 app = apps[appid]
1275                 data += app['Name'] + "\t"
1276                 if app['icon'] is not None:
1277                     data += app['icon'] + "\t"
1278                 data += app['License'] + "\n"
1279             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
1280                 f.write(data)
1281
1282     if cachechanged:
1283         with open(apkcachefile, 'wb') as cf:
1284             pickle.dump(apkcache, cf)
1285
1286     # Update the wiki...
1287     if options.wiki:
1288         update_wiki(apps, sortedids, apks + archapks)
1289
1290     logging.info("Finished.")
1291
1292 if __name__ == "__main__":
1293     main()