fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import socket
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from datetime import datetime, timedelta
  31 from xml.dom.minidom import Document
  32 from argparse import ArgumentParser
  33 import time
  34 from pyasn1.error import PyAsn1Error
  35 from pyasn1.codec.der import decoder, encoder
  36 from pyasn1_modules import rfc2315
  37 from hashlib import md5
  38 from binascii import hexlify, unhexlify
  39
  40 from PIL import Image
  41 import logging
  42
  43 import common
  44 import metadata
  45 from common import FDroidPopen, SdkToolsPopen
  46 from metadata import MetaDataException
  47
  48
  49 def get_densities():
  50     return ['640', '480', '320', '240', '160', '120']
  51
  52
  53 def dpi_to_px(density):
  54     return (int(density) * 48) / 160
  55
  56
  57 def px_to_dpi(px):
  58     return (int(px) * 160) / 48
  59
  60
  61 def get_icon_dir(repodir, density):
  62     if density is None:
  63         return os.path.join(repodir, "icons")
  64     return os.path.join(repodir, "icons-%s" % density)
  65
  66
  67 def get_icon_dirs(repodir):
  68     for density in get_densities():
  69         yield get_icon_dir(repodir, density)
  70     yield os.path.join(repodir, "icons")
  71
  72
  73 def update_wiki(apps, sortedids, apks):
  74     """Update the wiki
  75
  76     :param apps: fully populated list of all applications
  77     :param apks: all apks, except...
  78     """
  79     logging.info("Updating wiki")
  80     wikicat = 'Apps'
  81     wikiredircat = 'App Redirects'
  82     import mwclient
  83     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  84                          path=config['wiki_path'])
  85     site.login(config['wiki_user'], config['wiki_password'])
  86     generated_pages = {}
  87     generated_redirects = {}
  88
  89     for appid in sortedids:
  90         app = apps[appid]
  91
  92         wikidata = ''
  93         if app['Disabled']:
  94             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  95         if 'AntiFeatures' in app:
  96             for af in app['AntiFeatures']:
  97                 wikidata += '{{AntiFeature|' + af + '}}\n'
  98         if app['Requires Root']:
  99             requiresroot = 'Yes'
 100         else:
 101             requiresroot = 'No'
 102         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s}}\n' % (
 103             appid,
 104             app['Name'],
 105             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
 106             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
 107             app['Source Code'],
 108             app['Issue Tracker'],
 109             app['Web Site'],
 110             app['Changelog'],
 111             app['Donate'],
 112             app['FlattrID'],
 113             app['Bitcoin'],
 114             app['Litecoin'],
 115             app['License'],
 116             requiresroot)
 117
 118         if app['Provides']:
 119             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 120
 121         wikidata += app['Summary']
 122         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 123
 124         wikidata += "=Description=\n"
 125         wikidata += metadata.description_wiki(app['Description']) + "\n"
 126
 127         wikidata += "=Maintainer Notes=\n"
 128         if 'Maintainer Notes' in app:
 129             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 130         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 131
 132         # Get a list of all packages for this application...
 133         apklist = []
 134         gotcurrentver = False
 135         cantupdate = False
 136         buildfails = False
 137         for apk in apks:
 138             if apk['id'] == appid:
 139                 if str(apk['versioncode']) == app['Current Version Code']:
 140                     gotcurrentver = True
 141                 apklist.append(apk)
 142         # Include ones we can't build, as a special case...
 143         for thisbuild in app['builds']:
 144             if thisbuild['disable']:
 145                 if thisbuild['vercode'] == app['Current Version Code']:
 146                     cantupdate = True
 147                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 148                 apklist.append({'versioncode': int(thisbuild['vercode']),
 149                                 'version': thisbuild['version'],
 150                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(thisbuild['disable']),
 151                                 })
 152             else:
 153                 builtit = False
 154                 for apk in apklist:
 155                     if apk['versioncode'] == int(thisbuild['vercode']):
 156                         builtit = True
 157                         break
 158                 if not builtit:
 159                     buildfails = True
 160                     apklist.append({'versioncode': int(thisbuild['vercode']),
 161                                     'version': thisbuild['version'],
 162                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, thisbuild['vercode']),
 163                                     })
 164         if app['Current Version Code'] == '0':
 165             cantupdate = True
 166         # Sort with most recent first...
 167         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 168
 169         wikidata += "=Versions=\n"
 170         if len(apklist) == 0:
 171             wikidata += "We currently have no versions of this app available."
 172         elif not gotcurrentver:
 173             wikidata += "We don't have the current version of this app."
 174         else:
 175             wikidata += "We have the current version of this app."
 176         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 177         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 178         if len(app['No Source Since']) > 0:
 179             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 180         if len(app['Current Version']) > 0:
 181             wikidata += "The current (recommended) version is " + app['Current Version']
 182             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 183         validapks = 0
 184         for apk in apklist:
 185             wikidata += "==" + apk['version'] + "==\n"
 186
 187             if 'buildproblem' in apk:
 188                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 189             else:
 190                 validapks += 1
 191                 wikidata += "This version is built and signed by "
 192                 if 'srcname' in apk:
 193                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 194                 else:
 195                     wikidata += "the original developer.\n\n"
 196             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 197
 198         wikidata += '\n[[Category:' + wikicat + ']]\n'
 199         if len(app['No Source Since']) > 0:
 200             wikidata += '\n[[Category:Apps missing source code]]\n'
 201         if validapks == 0 and not app['Disabled']:
 202             wikidata += '\n[[Category:Apps with no packages]]\n'
 203         if cantupdate and not app['Disabled']:
 204             wikidata += "\n[[Category:Apps we can't update]]\n"
 205         if buildfails and not app['Disabled']:
 206             wikidata += "\n[[Category:Apps with failing builds]]\n"
 207         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 208             wikidata += '\n[[Category:Apps to Update]]\n'
 209         if app['Disabled']:
 210             wikidata += '\n[[Category:Apps that are disabled]]\n'
 211         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 212             wikidata += '\n[[Category:Apps with no update check]]\n'
 213         for appcat in app['Categories']:
 214             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 215
 216         # We can't have underscores in the page name, even if they're in
 217         # the package ID, because MediaWiki messes with them...
 218         pagename = appid.replace('_', ' ')
 219
 220         # Drop a trailing newline, because mediawiki is going to drop it anyway
 221         # and it we don't we'll think the page has changed when it hasn't...
 222         if wikidata.endswith('\n'):
 223             wikidata = wikidata[:-1]
 224
 225         generated_pages[pagename] = wikidata
 226
 227         # Make a redirect from the name to the ID too, unless there's
 228         # already an existing page with the name and it isn't a redirect.
 229         noclobber = False
 230         apppagename = app['Name'].replace('_', ' ')
 231         apppagename = apppagename.replace('{', '')
 232         apppagename = apppagename.replace('}', ' ')
 233         apppagename = apppagename.replace(':', ' ')
 234         # Drop double spaces caused mostly by replacing ':' above
 235         apppagename = apppagename.replace('  ', ' ')
 236         for expagename in site.allpages(prefix=apppagename,
 237                                         filterredir='nonredirects',
 238                                         generator=False):
 239             if expagename == apppagename:
 240                 noclobber = True
 241         # Another reason not to make the redirect page is if the app name
 242         # is the same as it's ID, because that will overwrite the real page
 243         # with an redirect to itself! (Although it seems like an odd
 244         # scenario this happens a lot, e.g. where there is metadata but no
 245         # builds or binaries to extract a name from.
 246         if apppagename == pagename:
 247             noclobber = True
 248         if not noclobber:
 249             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 250
 251     for tcat, genp in [(wikicat, generated_pages),
 252                        (wikiredircat, generated_redirects)]:
 253         catpages = site.Pages['Category:' + tcat]
 254         existingpages = []
 255         for page in catpages:
 256             existingpages.append(page.name)
 257             if page.name in genp:
 258                 pagetxt = page.edit()
 259                 if pagetxt != genp[page.name]:
 260                     logging.debug("Updating modified page " + page.name)
 261                     page.save(genp[page.name], summary='Auto-updated')
 262                 else:
 263                     logging.debug("Page " + page.name + " is unchanged")
 264             else:
 265                 logging.warn("Deleting page " + page.name)
 266                 page.delete('No longer published')
 267         for pagename, text in genp.items():
 268             logging.debug("Checking " + pagename)
 269             if pagename not in existingpages:
 270                 logging.debug("Creating page " + pagename)
 271                 try:
 272                     newpage = site.Pages[pagename]
 273                     newpage.save(text, summary='Auto-created')
 274                 except:
 275                     logging.error("...FAILED to create page '{0}'".format(pagename))
 276
 277     # Purge server cache to ensure counts are up to date
 278     site.pages['Repository Maintenance'].purge()
 279
 280
 281 def delete_disabled_builds(apps, apkcache, repodirs):
 282     """Delete disabled build outputs.
 283
 284     :param apps: list of all applications, as per metadata.read_metadata
 285     :param apkcache: current apk cache information
 286     :param repodirs: the repo directories to process
 287     """
 288     for appid, app in apps.iteritems():
 289         for build in app['builds']:
 290             if build['disable']:
 291                 apkfilename = appid + '_' + str(build['vercode']) + '.apk'
 292                 for repodir in repodirs:
 293                     apkpath = os.path.join(repodir, apkfilename)
 294                     ascpath = apkpath + ".asc"
 295                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 296                     for name in [apkpath, srcpath, ascpath]:
 297                         if os.path.exists(name):
 298                             logging.warn("Deleting disabled build output " + apkfilename)
 299                             os.remove(name)
 300                 if apkfilename in apkcache:
 301                     del apkcache[apkfilename]
 302
 303
 304 def resize_icon(iconpath, density):
 305
 306     if not os.path.isfile(iconpath):
 307         return
 308
 309     try:
 310         im = Image.open(iconpath)
 311         size = dpi_to_px(density)
 312
 313         if any(length > size for length in im.size):
 314             oldsize = im.size
 315             im.thumbnail((size, size), Image.ANTIALIAS)
 316             logging.debug("%s was too large at %s - new size is %s" % (
 317                 iconpath, oldsize, im.size))
 318             im.save(iconpath, "PNG")
 319
 320     except Exception, e:
 321         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 322
 323
 324 def resize_all_icons(repodirs):
 325     """Resize all icons that exceed the max size
 326
 327     :param repodirs: the repo directories to process
 328     """
 329     for repodir in repodirs:
 330         for density in get_densities():
 331             icon_dir = get_icon_dir(repodir, density)
 332             icon_glob = os.path.join(icon_dir, '*.png')
 333             for iconpath in glob.glob(icon_glob):
 334                 resize_icon(iconpath, density)
 335
 336
 337 # A signature block file with a .DSA, .RSA, or .EC extension
 338 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 339
 340
 341 def getsig(apkpath):
 342     """ Get the signing certificate of an apk. To get the same md5 has that
 343     Android gets, we encode the .RSA certificate in a specific format and pass
 344     it hex-encoded to the md5 digest algorithm.
 345
 346     :param apkpath: path to the apk
 347     :returns: A string containing the md5 of the signature of the apk or None
 348               if an error occurred.
 349     """
 350
 351     cert = None
 352
 353     # verify the jar signature is correct
 354     args = ['jarsigner', '-verify', apkpath]
 355     p = FDroidPopen(args)
 356     if p.returncode != 0:
 357         logging.critical(apkpath + " has a bad signature!")
 358         return None
 359
 360     with zipfile.ZipFile(apkpath, 'r') as apk:
 361
 362         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 363
 364         if len(certs) < 1:
 365             logging.error("Found no signing certificates on %s" % apkpath)
 366             return None
 367         if len(certs) > 1:
 368             logging.error("Found multiple signing certificates on %s" % apkpath)
 369             return None
 370
 371         cert = apk.read(certs[0])
 372
 373     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 374     if content.getComponentByName('contentType') != rfc2315.signedData:
 375         logging.error("Unexpected format.")
 376         return None
 377
 378     content = decoder.decode(content.getComponentByName('content'),
 379                              asn1Spec=rfc2315.SignedData())[0]
 380     try:
 381         certificates = content.getComponentByName('certificates')
 382     except PyAsn1Error:
 383         logging.error("Certificates not found.")
 384         return None
 385
 386     cert_encoded = encoder.encode(certificates)[4:]
 387
 388     return md5(cert_encoded.encode('hex')).hexdigest()
 389
 390
 391 def scan_apks(apps, apkcache, repodir, knownapks):
 392     """Scan the apks in the given repo directory.
 393
 394     This also extracts the icons.
 395
 396     :param apps: list of all applications, as per metadata.read_metadata
 397     :param apkcache: current apk cache information
 398     :param repodir: repo directory to scan
 399     :param knownapks: known apks info
 400     :returns: (apks, cachechanged) where apks is a list of apk information,
 401               and cachechanged is True if the apkcache got changed.
 402     """
 403
 404     cachechanged = False
 405
 406     icon_dirs = get_icon_dirs(repodir)
 407     for icon_dir in icon_dirs:
 408         if os.path.exists(icon_dir):
 409             if options.clean:
 410                 shutil.rmtree(icon_dir)
 411                 os.makedirs(icon_dir)
 412         else:
 413             os.makedirs(icon_dir)
 414
 415     apks = []
 416     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 417     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 418     vername_pat = re.compile(".*versionName='([^']*)'.*")
 419     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 420     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 421     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 422     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 423     string_pat = re.compile(".*'([^']*)'.*")
 424     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 425
 426         apkfilename = apkfile[len(repodir) + 1:]
 427         if ' ' in apkfilename:
 428             logging.critical("Spaces in filenames are not allowed.")
 429             sys.exit(1)
 430
 431         # Calculate the sha256...
 432         sha = hashlib.sha256()
 433         with open(apkfile, 'rb') as f:
 434             while True:
 435                 t = f.read(16384)
 436                 if len(t) == 0:
 437                     break
 438                 sha.update(t)
 439             shasum = sha.hexdigest()
 440
 441         usecache = False
 442         if apkfilename in apkcache:
 443             thisinfo = apkcache[apkfilename]
 444             if thisinfo['sha256'] == shasum:
 445                 logging.debug("Reading " + apkfilename + " from cache")
 446                 usecache = True
 447             else:
 448                 logging.debug("Ignoring stale cache data for " + apkfilename)
 449
 450         if not usecache:
 451             logging.debug("Processing " + apkfilename)
 452             thisinfo = {}
 453             thisinfo['apkname'] = apkfilename
 454             thisinfo['sha256'] = shasum
 455             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 456             if os.path.exists(os.path.join(repodir, srcfilename)):
 457                 thisinfo['srcname'] = srcfilename
 458             thisinfo['size'] = os.path.getsize(apkfile)
 459             thisinfo['permissions'] = set()
 460             thisinfo['features'] = set()
 461             thisinfo['icons_src'] = {}
 462             thisinfo['icons'] = {}
 463             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 464             if p.returncode != 0:
 465                 if options.delete_unknown:
 466                     if os.path.exists(apkfile):
 467                         logging.error("Failed to get apk information, deleting " + apkfile)
 468                         os.remove(apkfile)
 469                     else:
 470                         logging.error("Could not find {0} to remove it".format(apkfile))
 471                 else:
 472                     logging.error("Failed to get apk information, skipping " + apkfile)
 473                 continue
 474             for line in p.output.splitlines():
 475                 if line.startswith("package:"):
 476                     try:
 477                         thisinfo['id'] = re.match(name_pat, line).group(1)
 478                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 479                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 480                     except Exception, e:
 481                         logging.error("Package matching failed: " + str(e))
 482                         logging.info("Line was: " + line)
 483                         sys.exit(1)
 484                 elif line.startswith("application:"):
 485                     thisinfo['name'] = re.match(label_pat, line).group(1)
 486                     # Keep path to non-dpi icon in case we need it
 487                     match = re.match(icon_pat_nodpi, line)
 488                     if match:
 489                         thisinfo['icons_src']['-1'] = match.group(1)
 490                 elif line.startswith("launchable-activity:"):
 491                     # Only use launchable-activity as fallback to application
 492                     if not thisinfo['name']:
 493                         thisinfo['name'] = re.match(label_pat, line).group(1)
 494                     if '-1' not in thisinfo['icons_src']:
 495                         match = re.match(icon_pat_nodpi, line)
 496                         if match:
 497                             thisinfo['icons_src']['-1'] = match.group(1)
 498                 elif line.startswith("application-icon-"):
 499                     match = re.match(icon_pat, line)
 500                     if match:
 501                         density = match.group(1)
 502                         path = match.group(2)
 503                         thisinfo['icons_src'][density] = path
 504                 elif line.startswith("sdkVersion:"):
 505                     m = re.match(sdkversion_pat, line)
 506                     if m is None:
 507                         logging.error(line.replace('sdkVersion:', '')
 508                                       + ' is not a valid minSdkVersion!')
 509                     else:
 510                         thisinfo['sdkversion'] = m.group(1)
 511                 elif line.startswith("maxSdkVersion:"):
 512                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 513                 elif line.startswith("native-code:"):
 514                     thisinfo['nativecode'] = []
 515                     for arch in line[13:].split(' '):
 516                         thisinfo['nativecode'].append(arch[1:-1])
 517                 elif line.startswith("uses-permission:"):
 518                     perm = re.match(string_pat, line).group(1)
 519                     if perm.startswith("android.permission."):
 520                         perm = perm[19:]
 521                     thisinfo['permissions'].add(perm)
 522                 elif line.startswith("uses-feature:"):
 523                     perm = re.match(string_pat, line).group(1)
 524                     # Filter out this, it's only added with the latest SDK tools and
 525                     # causes problems for lots of apps.
 526                     if perm != "android.hardware.screen.portrait" \
 527                             and perm != "android.hardware.screen.landscape":
 528                         if perm.startswith("android.feature."):
 529                             perm = perm[16:]
 530                         thisinfo['features'].add(perm)
 531
 532             if 'sdkversion' not in thisinfo:
 533                 logging.warn("No SDK version information found in {0}".format(apkfile))
 534                 thisinfo['sdkversion'] = 0
 535
 536             # Check for debuggable apks...
 537             if common.isApkDebuggable(apkfile, config):
 538                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 539
 540             # Get the signature (or md5 of, to be precise)...
 541             logging.debug('Getting signature of {0}'.format(apkfile))
 542             thisinfo['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 543             if not thisinfo['sig']:
 544                 logging.critical("Failed to get apk signature")
 545                 sys.exit(1)
 546
 547             apk = zipfile.ZipFile(apkfile, 'r')
 548
 549             # if an APK has files newer than the system time, suggest updating
 550             # the system clock.  This is useful for offline systems, used for
 551             # signing, which do not have another source of clock sync info. It
 552             # has to be more than 24 hours newer because ZIP/APK files do not
 553             # store timezone info
 554             info = apk.getinfo('AndroidManifest.xml')
 555             dt_obj = datetime(*info.date_time)
 556             checkdt = dt_obj - timedelta(1)
 557             if datetime.today() < checkdt:
 558                 logging.warn('System clock is older than manifest in: '
 559                              + apkfilename + '\nSet clock to that time using:\n'
 560                              + 'sudo date -s "' + str(dt_obj) + '"')
 561
 562             iconfilename = "%s.%s.png" % (
 563                 thisinfo['id'],
 564                 thisinfo['versioncode'])
 565
 566             # Extract the icon file...
 567             densities = get_densities()
 568             empty_densities = []
 569             for density in densities:
 570                 if density not in thisinfo['icons_src']:
 571                     empty_densities.append(density)
 572                     continue
 573                 iconsrc = thisinfo['icons_src'][density]
 574                 icon_dir = get_icon_dir(repodir, density)
 575                 icondest = os.path.join(icon_dir, iconfilename)
 576
 577                 try:
 578                     with open(icondest, 'wb') as f:
 579                         f.write(apk.read(iconsrc))
 580                     thisinfo['icons'][density] = iconfilename
 581
 582                 except:
 583                     logging.warn("Error retrieving icon file")
 584                     del thisinfo['icons'][density]
 585                     del thisinfo['icons_src'][density]
 586                     empty_densities.append(density)
 587
 588             if '-1' in thisinfo['icons_src']:
 589                 iconsrc = thisinfo['icons_src']['-1']
 590                 iconpath = os.path.join(
 591                     get_icon_dir(repodir, None), iconfilename)
 592                 with open(iconpath, 'wb') as f:
 593                     f.write(apk.read(iconsrc))
 594                 try:
 595                     im = Image.open(iconpath)
 596                     dpi = px_to_dpi(im.size[0])
 597                     for density in densities:
 598                         if density in thisinfo['icons']:
 599                             break
 600                         if density == densities[-1] or dpi >= int(density):
 601                             thisinfo['icons'][density] = iconfilename
 602                             shutil.move(iconpath,
 603                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 604                             empty_densities.remove(density)
 605                             break
 606                 except Exception, e:
 607                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 608
 609             if thisinfo['icons']:
 610                 thisinfo['icon'] = iconfilename
 611
 612             apk.close()
 613
 614             # First try resizing down to not lose quality
 615             last_density = None
 616             for density in densities:
 617                 if density not in empty_densities:
 618                     last_density = density
 619                     continue
 620                 if last_density is None:
 621                     continue
 622                 logging.debug("Density %s not available, resizing down from %s"
 623                               % (density, last_density))
 624
 625                 last_iconpath = os.path.join(
 626                     get_icon_dir(repodir, last_density), iconfilename)
 627                 iconpath = os.path.join(
 628                     get_icon_dir(repodir, density), iconfilename)
 629                 try:
 630                     im = Image.open(last_iconpath)
 631                 except:
 632                     logging.warn("Invalid image file at %s" % last_iconpath)
 633                     continue
 634
 635                 size = dpi_to_px(density)
 636
 637                 im.thumbnail((size, size), Image.ANTIALIAS)
 638                 im.save(iconpath, "PNG")
 639                 empty_densities.remove(density)
 640
 641             # Then just copy from the highest resolution available
 642             last_density = None
 643             for density in reversed(densities):
 644                 if density not in empty_densities:
 645                     last_density = density
 646                     continue
 647                 if last_density is None:
 648                     continue
 649                 logging.debug("Density %s not available, copying from lower density %s"
 650                               % (density, last_density))
 651
 652                 shutil.copyfile(
 653                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 654                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 655
 656                 empty_densities.remove(density)
 657
 658             for density in densities:
 659                 icon_dir = get_icon_dir(repodir, density)
 660                 icondest = os.path.join(icon_dir, iconfilename)
 661                 resize_icon(icondest, density)
 662
 663             # Copy from icons-mdpi to icons since mdpi is the baseline density
 664             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 665             if os.path.isfile(baseline):
 666                 shutil.copyfile(baseline,
 667                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 668
 669             # Record in known apks, getting the added date at the same time..
 670             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 671             if added:
 672                 thisinfo['added'] = added
 673
 674             apkcache[apkfilename] = thisinfo
 675             cachechanged = True
 676
 677         apks.append(thisinfo)
 678
 679     return apks, cachechanged
 680
 681
 682 repo_pubkey_fingerprint = None
 683
 684
 685 # Generate a certificate fingerprint the same way keytool does it
 686 # (but with slightly different formatting)
 687 def cert_fingerprint(data):
 688     digest = hashlib.sha256(data).digest()
 689     ret = []
 690     ret.append(' '.join("%02X" % ord(b) for b in digest))
 691     return " ".join(ret)
 692
 693
 694 def extract_pubkey():
 695     global repo_pubkey_fingerprint
 696     if 'repo_pubkey' in config:
 697         pubkey = unhexlify(config['repo_pubkey'])
 698     else:
 699         p = FDroidPopen(['keytool', '-exportcert',
 700                          '-alias', config['repo_keyalias'],
 701                          '-keystore', config['keystore'],
 702                          '-storepass:file', config['keystorepassfile']]
 703                         + config['smartcardoptions'], output=False)
 704         if p.returncode != 0 or len(p.output) < 20:
 705             msg = "Failed to get repo pubkey!"
 706             if config['keystore'] == 'NONE':
 707                 msg += ' Is your crypto smartcard plugged in?'
 708             logging.critical(msg)
 709             sys.exit(1)
 710         pubkey = p.output
 711     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 712     return hexlify(pubkey)
 713
 714
 715 def make_index(apps, sortedids, apks, repodir, archive, categories):
 716     """Make a repo index.
 717
 718     :param apps: fully populated apps list
 719     :param apks: full populated apks list
 720     :param repodir: the repo directory
 721     :param archive: True if this is the archive repo, False if it's the
 722                     main one.
 723     :param categories: list of categories
 724     """
 725
 726     doc = Document()
 727
 728     def addElement(name, value, doc, parent):
 729         el = doc.createElement(name)
 730         el.appendChild(doc.createTextNode(value))
 731         parent.appendChild(el)
 732
 733     def addElementNonEmpty(name, value, doc, parent):
 734         if not value:
 735             return
 736         addElement(name, value, doc, parent)
 737
 738     def addElementCDATA(name, value, doc, parent):
 739         el = doc.createElement(name)
 740         el.appendChild(doc.createCDATASection(value))
 741         parent.appendChild(el)
 742
 743     root = doc.createElement("fdroid")
 744     doc.appendChild(root)
 745
 746     repoel = doc.createElement("repo")
 747
 748     if archive:
 749         repoel.setAttribute("name", config['archive_name'])
 750         if config['repo_maxage'] != 0:
 751             repoel.setAttribute("maxage", str(config['repo_maxage']))
 752         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 753         repoel.setAttribute("url", config['archive_url'])
 754         addElement('description', config['archive_description'], doc, repoel)
 755
 756     else:
 757         repoel.setAttribute("name", config['repo_name'])
 758         if config['repo_maxage'] != 0:
 759             repoel.setAttribute("maxage", str(config['repo_maxage']))
 760         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 761         repoel.setAttribute("url", config['repo_url'])
 762         addElement('description', config['repo_description'], doc, repoel)
 763
 764     repoel.setAttribute("version", "14")
 765     repoel.setAttribute("timestamp", str(int(time.time())))
 766
 767     nosigningkey = False
 768     if not options.nosign:
 769         if 'repo_keyalias' not in config:
 770             nosigningkey = True
 771             logging.critical("'repo_keyalias' not found in config.py!")
 772         if 'keystore' not in config:
 773             nosigningkey = True
 774             logging.critical("'keystore' not found in config.py!")
 775         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 776             nosigningkey = True
 777             logging.critical("'keystorepass' not found in config.py!")
 778         if 'keypass' not in config and 'keypassfile' not in config:
 779             nosigningkey = True
 780             logging.critical("'keypass' not found in config.py!")
 781         if not os.path.exists(config['keystore']):
 782             nosigningkey = True
 783             logging.critical("'" + config['keystore'] + "' does not exist!")
 784         if nosigningkey:
 785             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 786             logging.warning("\tfdroid update --create-key")
 787             sys.exit(1)
 788
 789     repoel.setAttribute("pubkey", extract_pubkey())
 790     root.appendChild(repoel)
 791
 792     for appid in sortedids:
 793         app = apps[appid]
 794
 795         if app['Disabled'] is not None:
 796             continue
 797
 798         # Get a list of the apks for this app...
 799         apklist = []
 800         for apk in apks:
 801             if apk['id'] == appid:
 802                 apklist.append(apk)
 803
 804         if len(apklist) == 0:
 805             continue
 806
 807         apel = doc.createElement("application")
 808         apel.setAttribute("id", app['id'])
 809         root.appendChild(apel)
 810
 811         addElement('id', app['id'], doc, apel)
 812         if 'added' in app:
 813             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 814         if 'lastupdated' in app:
 815             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 816         addElement('name', app['Name'], doc, apel)
 817         addElement('summary', app['Summary'], doc, apel)
 818         if app['icon']:
 819             addElement('icon', app['icon'], doc, apel)
 820
 821         def linkres(appid):
 822             if appid in apps:
 823                 return ("fdroid.app:" + appid, apps[appid]['Name'])
 824             raise MetaDataException("Cannot resolve app id " + appid)
 825
 826         addElement('desc',
 827                    metadata.description_html(app['Description'], linkres),
 828                    doc, apel)
 829         addElement('license', app['License'], doc, apel)
 830         if 'Categories' in app and app['Categories']:
 831             addElement('categories', ','.join(app["Categories"]), doc, apel)
 832             # We put the first (primary) category in LAST, which will have
 833             # the desired effect of making clients that only understand one
 834             # category see that one.
 835             addElement('category', app["Categories"][0], doc, apel)
 836         addElement('web', app['Web Site'], doc, apel)
 837         addElement('source', app['Source Code'], doc, apel)
 838         addElement('tracker', app['Issue Tracker'], doc, apel)
 839         addElementNonEmpty('changelog', app['Changelog'], doc, apel)
 840         addElementNonEmpty('donate', app['Donate'], doc, apel)
 841         addElementNonEmpty('bitcoin', app['Bitcoin'], doc, apel)
 842         addElementNonEmpty('litecoin', app['Litecoin'], doc, apel)
 843         addElementNonEmpty('flattr', app['FlattrID'], doc, apel)
 844
 845         # These elements actually refer to the current version (i.e. which
 846         # one is recommended. They are historically mis-named, and need
 847         # changing, but stay like this for now to support existing clients.
 848         addElement('marketversion', app['Current Version'], doc, apel)
 849         addElement('marketvercode', app['Current Version Code'], doc, apel)
 850
 851         if app['AntiFeatures']:
 852             af = app['AntiFeatures']
 853             if af:
 854                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 855         if app['Provides']:
 856             pv = app['Provides'].split(',')
 857             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 858         if app['Requires Root']:
 859             addElement('requirements', 'root', doc, apel)
 860
 861         # Sort the apk list into version order, just so the web site
 862         # doesn't have to do any work by default...
 863         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 864
 865         # Check for duplicates - they will make the client unhappy...
 866         for i in range(len(apklist) - 1):
 867             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 868                 logging.critical("duplicate versions: '%s' - '%s'" % (
 869                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 870                 sys.exit(1)
 871
 872         current_version_code = 0
 873         current_version_file = None
 874         for apk in apklist:
 875             # find the APK for the "Current Version"
 876             if current_version_code < apk['versioncode']:
 877                 current_version_code = apk['versioncode']
 878             if current_version_code < int(app['Current Version Code']):
 879                 current_version_file = apk['apkname']
 880
 881             apkel = doc.createElement("package")
 882             apel.appendChild(apkel)
 883             addElement('version', apk['version'], doc, apkel)
 884             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 885             addElement('apkname', apk['apkname'], doc, apkel)
 886             if 'srcname' in apk:
 887                 addElement('srcname', apk['srcname'], doc, apkel)
 888             for hash_type in ['sha256']:
 889                 if hash_type not in apk:
 890                     continue
 891                 hashel = doc.createElement("hash")
 892                 hashel.setAttribute("type", hash_type)
 893                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 894                 apkel.appendChild(hashel)
 895             addElement('sig', apk['sig'], doc, apkel)
 896             addElement('size', str(apk['size']), doc, apkel)
 897             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 898             if 'maxsdkversion' in apk:
 899                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 900             if 'added' in apk:
 901                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 902             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
 903             if 'nativecode' in apk:
 904                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 905             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
 906
 907         if current_version_file is not None \
 908                 and config['make_current_version_link'] \
 909                 and repodir == 'repo':  # only create these
 910             sanitized_name = re.sub('''[ '"&%?+=/]''', '',
 911                                     app[config['current_version_name_source']])
 912             apklinkname = sanitized_name + '.apk'
 913             current_version_path = os.path.join(repodir, current_version_file)
 914             if os.path.exists(apklinkname):
 915                 os.remove(apklinkname)
 916             os.symlink(current_version_path, apklinkname)
 917             # also symlink gpg signature, if it exists
 918             for extension in ('.asc', '.sig'):
 919                 sigfile_path = current_version_path + extension
 920                 if os.path.exists(sigfile_path):
 921                     siglinkname = apklinkname + extension
 922                     if os.path.exists(siglinkname):
 923                         os.remove(siglinkname)
 924                     os.symlink(sigfile_path, siglinkname)
 925
 926     if options.pretty:
 927         output = doc.toprettyxml()
 928     else:
 929         output = doc.toxml()
 930
 931     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 932         f.write(output)
 933
 934     if 'repo_keyalias' in config:
 935
 936         if options.nosign:
 937             logging.info("Creating unsigned index in preparation for signing")
 938         else:
 939             logging.info("Creating signed index with this key (SHA256):")
 940             logging.info("%s" % repo_pubkey_fingerprint)
 941
 942         # Create a jar of the index...
 943         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
 944         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 945         if p.returncode != 0:
 946             logging.critical("Failed to create {0}".format(jar_output))
 947             sys.exit(1)
 948
 949         # Sign the index...
 950         signed = os.path.join(repodir, 'index.jar')
 951         if options.nosign:
 952             # Remove old signed index if not signing
 953             if os.path.exists(signed):
 954                 os.remove(signed)
 955         else:
 956             args = ['jarsigner', '-keystore', config['keystore'],
 957                     '-storepass:file', config['keystorepassfile'],
 958                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
 959                     signed, config['repo_keyalias']]
 960             if config['keystore'] == 'NONE':
 961                 args += config['smartcardoptions']
 962             else:  # smardcards never use -keypass
 963                 args += ['-keypass:file', config['keypassfile']]
 964             p = FDroidPopen(args)
 965             if p.returncode != 0:
 966                 logging.critical("Failed to sign index")
 967                 sys.exit(1)
 968
 969     # Copy the repo icon into the repo directory...
 970     icon_dir = os.path.join(repodir, 'icons')
 971     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 972     shutil.copyfile(config['repo_icon'], iconfilename)
 973
 974     # Write a category list in the repo to allow quick access...
 975     catdata = ''
 976     for cat in categories:
 977         catdata += cat + '\n'
 978     with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
 979         f.write(catdata)
 980
 981
 982 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 983
 984     for appid, app in apps.iteritems():
 985
 986         # Get a list of the apks for this app...
 987         apklist = []
 988         for apk in apks:
 989             if apk['id'] == appid:
 990                 apklist.append(apk)
 991
 992         # Sort the apk list into version order...
 993         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 994
 995         if app['Archive Policy']:
 996             keepversions = int(app['Archive Policy'][:-9])
 997         else:
 998             keepversions = defaultkeepversions
 999
1000         if len(apklist) > keepversions:
1001             for apk in apklist[keepversions:]:
1002                 logging.info("Moving " + apk['apkname'] + " to archive")
1003                 shutil.move(os.path.join(repodir, apk['apkname']),
1004                             os.path.join(archivedir, apk['apkname']))
1005                 if 'srcname' in apk:
1006                     shutil.move(os.path.join(repodir, apk['srcname']),
1007                                 os.path.join(archivedir, apk['srcname']))
1008                     # Move GPG signature too...
1009                     sigfile = apk['srcname'] + '.asc'
1010                     sigsrc = os.path.join(repodir, sigfile)
1011                     if os.path.exists(sigsrc):
1012                         shutil.move(sigsrc, os.path.join(archivedir, sigfile))
1013
1014                 archapks.append(apk)
1015                 apks.remove(apk)
1016
1017
1018 def add_apks_to_per_app_repos(repodir, apks):
1019     apks_per_app = dict()
1020     for apk in apks:
1021         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1022         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1023         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1024         apks_per_app[apk['id']] = apk
1025
1026         if not os.path.exists(apk['per_app_icons']):
1027             logging.info('Adding new repo for only ' + apk['id'])
1028             os.makedirs(apk['per_app_icons'])
1029
1030         apkpath = os.path.join(repodir, apk['apkname'])
1031         shutil.copy(apkpath, apk['per_app_repo'])
1032         apksigpath = apkpath + '.sig'
1033         if os.path.exists(apksigpath):
1034             shutil.copy(apksigpath, apk['per_app_repo'])
1035         apkascpath = apkpath + '.asc'
1036         if os.path.exists(apkascpath):
1037             shutil.copy(apkascpath, apk['per_app_repo'])
1038
1039
1040 config = None
1041 options = None
1042
1043
1044 def main():
1045
1046     global config, options
1047
1048     # Parse command line...
1049     parser = ArgumentParser()
1050     common.setup_global_opts(parser)
1051     parser.add_argument("--create-key", action="store_true", default=False,
1052                         help="Create a repo signing key in a keystore")
1053     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1054                         help="Create skeleton metadata files that are missing")
1055     parser.add_argument("--delete-unknown", action="store_true", default=False,
1056                         help="Delete APKs without metadata from the repo")
1057     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1058                         help="Report on build data status")
1059     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1060                         help="Interactively ask about things that need updating.")
1061     parser.add_argument("-I", "--icons", action="store_true", default=False,
1062                         help="Resize all the icons exceeding the max pixel size and exit")
1063     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1064                         help="Specify editor to use in interactive mode. Default " +
1065                         "is /etc/alternatives/editor")
1066     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1067                         help="Update the wiki")
1068     parser.add_argument("--pretty", action="store_true", default=False,
1069                         help="Produce human-readable index.xml")
1070     parser.add_argument("--clean", action="store_true", default=False,
1071                         help="Clean update - don't uses caches, reprocess all apks")
1072     parser.add_argument("--nosign", action="store_true", default=False,
1073                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1074     options = parser.parse_args()
1075
1076     config = common.read_config(options)
1077
1078     repodirs = ['repo']
1079     if config['archive_older'] != 0:
1080         repodirs.append('archive')
1081         if not os.path.exists('archive'):
1082             os.mkdir('archive')
1083
1084     if options.icons:
1085         resize_all_icons(repodirs)
1086         sys.exit(0)
1087
1088     # check that icons exist now, rather than fail at the end of `fdroid update`
1089     for k in ['repo_icon', 'archive_icon']:
1090         if k in config:
1091             if not os.path.exists(config[k]):
1092                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1093                 sys.exit(1)
1094
1095     # if the user asks to create a keystore, do it now, reusing whatever it can
1096     if options.create_key:
1097         if os.path.exists(config['keystore']):
1098             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1099             logging.critical("\t'" + config['keystore'] + "'")
1100             sys.exit(1)
1101
1102         if 'repo_keyalias' not in config:
1103             config['repo_keyalias'] = socket.getfqdn()
1104             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1105         if 'keydname' not in config:
1106             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1107             common.write_to_config(config, 'keydname', config['keydname'])
1108         if 'keystore' not in config:
1109             config['keystore'] = common.default_config.keystore
1110             common.write_to_config(config, 'keystore', config['keystore'])
1111
1112         password = common.genpassword()
1113         if 'keystorepass' not in config:
1114             config['keystorepass'] = password
1115             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1116         if 'keypass' not in config:
1117             config['keypass'] = password
1118             common.write_to_config(config, 'keypass', config['keypass'])
1119         common.genkeystore(config)
1120
1121     # Get all apps...
1122     apps = metadata.read_metadata()
1123
1124     # Generate a list of categories...
1125     categories = set()
1126     for app in apps.itervalues():
1127         categories.update(app['Categories'])
1128
1129     # Read known apks data (will be updated and written back when we've finished)
1130     knownapks = common.KnownApks()
1131
1132     # Gather information about all the apk files in the repo directory, using
1133     # cached data if possible.
1134     apkcachefile = os.path.join('tmp', 'apkcache')
1135     if not options.clean and os.path.exists(apkcachefile):
1136         with open(apkcachefile, 'rb') as cf:
1137             apkcache = pickle.load(cf)
1138     else:
1139         apkcache = {}
1140
1141     delete_disabled_builds(apps, apkcache, repodirs)
1142
1143     # Scan all apks in the main repo
1144     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks)
1145
1146     # Generate warnings for apk's with no metadata (or create skeleton
1147     # metadata files, if requested on the command line)
1148     newmetadata = False
1149     for apk in apks:
1150         if apk['id'] not in apps:
1151             if options.create_metadata:
1152                 if 'name' not in apk:
1153                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1154                     continue
1155                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1156                 f.write("License:Unknown\n")
1157                 f.write("Web Site:\n")
1158                 f.write("Source Code:\n")
1159                 f.write("Issue Tracker:\n")
1160                 f.write("Changelog:\n")
1161                 f.write("Summary:" + apk['name'] + "\n")
1162                 f.write("Description:\n")
1163                 f.write(apk['name'] + "\n")
1164                 f.write(".\n")
1165                 f.close()
1166                 logging.info("Generated skeleton metadata for " + apk['id'])
1167                 newmetadata = True
1168             else:
1169                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1170                 if options.delete_unknown:
1171                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1172                     rmf = os.path.join(repodirs[0], apk['apkname'])
1173                     if not os.path.exists(rmf):
1174                         logging.error("Could not find {0} to remove it".format(rmf))
1175                     else:
1176                         os.remove(rmf)
1177                 else:
1178                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1179
1180     # update the metadata with the newly created ones included
1181     if newmetadata:
1182         apps = metadata.read_metadata()
1183
1184     # Scan the archive repo for apks as well
1185     if len(repodirs) > 1:
1186         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1187         if cc:
1188             cachechanged = True
1189     else:
1190         archapks = []
1191
1192     # Some information from the apks needs to be applied up to the application
1193     # level. When doing this, we use the info from the most recent version's apk.
1194     # We deal with figuring out when the app was added and last updated at the
1195     # same time.
1196     for appid, app in apps.iteritems():
1197         bestver = 0
1198         added = None
1199         lastupdated = None
1200         for apk in apks + archapks:
1201             if apk['id'] == appid:
1202                 if apk['versioncode'] > bestver:
1203                     bestver = apk['versioncode']
1204                     bestapk = apk
1205
1206                 if 'added' in apk:
1207                     if not added or apk['added'] < added:
1208                         added = apk['added']
1209                     if not lastupdated or apk['added'] > lastupdated:
1210                         lastupdated = apk['added']
1211
1212         if added:
1213             app['added'] = added
1214         else:
1215             logging.warn("Don't know when " + appid + " was added")
1216         if lastupdated:
1217             app['lastupdated'] = lastupdated
1218         else:
1219             logging.warn("Don't know when " + appid + " was last updated")
1220
1221         if bestver == 0:
1222             if app['Name'] is None:
1223                 app['Name'] = app['Auto Name'] or appid
1224             app['icon'] = None
1225             logging.warn("Application " + appid + " has no packages")
1226         else:
1227             if app['Name'] is None:
1228                 app['Name'] = bestapk['name']
1229             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1230
1231     # Sort the app list by name, then the web site doesn't have to by default.
1232     # (we had to wait until we'd scanned the apks to do this, because mostly the
1233     # name comes from there!)
1234     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid]['Name'].upper())
1235
1236     # APKs are placed into multiple repos based on the app package, providing
1237     # per-app subscription feeds for nightly builds and things like it
1238     if config['per_app_repos']:
1239         add_apks_to_per_app_repos(repodirs[0], apks)
1240         for appid, app in apps.iteritems():
1241             repodir = os.path.join(appid, 'fdroid', 'repo')
1242             appdict = dict()
1243             appdict[appid] = app
1244             if os.path.isdir(repodir):
1245                 make_index(appdict, [appid], apks, repodir, False, categories)
1246             else:
1247                 logging.info('Skipping index generation for ' + appid)
1248         return
1249
1250     if len(repodirs) > 1:
1251         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1252
1253     # Make the index for the main repo...
1254     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1255
1256     # If there's an archive repo,  make the index for it. We already scanned it
1257     # earlier on.
1258     if len(repodirs) > 1:
1259         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1260
1261     if config['update_stats']:
1262
1263         # Update known apks info...
1264         knownapks.writeifchanged()
1265
1266         # Generate latest apps data for widget
1267         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1268             data = ''
1269             for line in file(os.path.join('stats', 'latestapps.txt')):
1270                 appid = line.rstrip()
1271                 data += appid + "\t"
1272                 app = apps[appid]
1273                 data += app['Name'] + "\t"
1274                 if app['icon'] is not None:
1275                     data += app['icon'] + "\t"
1276                 data += app['License'] + "\n"
1277             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
1278                 f.write(data)
1279
1280     if cachechanged:
1281         with open(apkcachefile, 'wb') as cf:
1282             pickle.dump(apkcache, cf)
1283
1284     # Update the wiki...
1285     if options.wiki:
1286         update_wiki(apps, sortedids, apks + archapks)
1287
1288     logging.info("Finished.")
1289
1290 if __name__ == "__main__":
1291     main()