fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import socket
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from datetime import datetime, timedelta
  31 from xml.dom.minidom import Document
  32 from optparse import OptionParser
  33 import time
  34 from pyasn1.error import PyAsn1Error
  35 from pyasn1.codec.der import decoder, encoder
  36 from pyasn1_modules import rfc2315
  37 from hashlib import md5
  38 from binascii import hexlify, unhexlify
  39
  40 from PIL import Image
  41 import logging
  42
  43 import common
  44 import metadata
  45 from common import FDroidPopen, SdkToolsPopen
  46 from metadata import MetaDataException
  47
  48
  49 def get_densities():
  50     return ['640', '480', '320', '240', '160', '120']
  51
  52
  53 def dpi_to_px(density):
  54     return (int(density) * 48) / 160
  55
  56
  57 def px_to_dpi(px):
  58     return (int(px) * 160) / 48
  59
  60
  61 def get_icon_dir(repodir, density):
  62     if density is None:
  63         return os.path.join(repodir, "icons")
  64     return os.path.join(repodir, "icons-%s" % density)
  65
  66
  67 def get_icon_dirs(repodir):
  68     for density in get_densities():
  69         yield get_icon_dir(repodir, density)
  70     yield os.path.join(repodir, "icons")
  71
  72
  73 def update_wiki(apps, sortedids, apks):
  74     """Update the wiki
  75
  76     :param apps: fully populated list of all applications
  77     :param apks: all apks, except...
  78     """
  79     logging.info("Updating wiki")
  80     wikicat = 'Apps'
  81     wikiredircat = 'App Redirects'
  82     import mwclient
  83     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  84                          path=config['wiki_path'])
  85     site.login(config['wiki_user'], config['wiki_password'])
  86     generated_pages = {}
  87     generated_redirects = {}
  88
  89     for appid in sortedids:
  90         app = apps[appid]
  91
  92         wikidata = ''
  93         if app['Disabled']:
  94             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  95         if app['AntiFeatures']:
  96             for af in app['AntiFeatures'].split(','):
  97                 wikidata += '{{AntiFeature|' + af + '}}\n'
  98         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
  99             appid,
 100             app['Name'],
 101             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
 102             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
 103             app['Source Code'],
 104             app['Issue Tracker'],
 105             app['Web Site'],
 106             app['Changelog'],
 107             app['Donate'],
 108             app['FlattrID'],
 109             app['Bitcoin'],
 110             app['Litecoin'],
 111             app['Dogecoin'],
 112             app['License'],
 113             app.get('Requires Root', 'No'))
 114
 115         if app['Provides']:
 116             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 117
 118         wikidata += app['Summary']
 119         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 120
 121         wikidata += "=Description=\n"
 122         wikidata += metadata.description_wiki(app['Description']) + "\n"
 123
 124         wikidata += "=Maintainer Notes=\n"
 125         if 'Maintainer Notes' in app:
 126             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 127         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 128
 129         # Get a list of all packages for this application...
 130         apklist = []
 131         gotcurrentver = False
 132         cantupdate = False
 133         buildfails = False
 134         for apk in apks:
 135             if apk['id'] == appid:
 136                 if str(apk['versioncode']) == app['Current Version Code']:
 137                     gotcurrentver = True
 138                 apklist.append(apk)
 139         # Include ones we can't build, as a special case...
 140         for thisbuild in app['builds']:
 141             if thisbuild['disable']:
 142                 if thisbuild['vercode'] == app['Current Version Code']:
 143                     cantupdate = True
 144                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 145                 apklist.append({'versioncode': int(thisbuild['vercode']),
 146                                 'version': thisbuild['version'],
 147                                 'buildproblem': thisbuild['disable']
 148                                 })
 149             else:
 150                 builtit = False
 151                 for apk in apklist:
 152                     if apk['versioncode'] == int(thisbuild['vercode']):
 153                         builtit = True
 154                         break
 155                 if not builtit:
 156                     buildfails = True
 157                     apklist.append({'versioncode': int(thisbuild['vercode']),
 158                                     'version': thisbuild['version'],
 159                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, thisbuild['vercode'])
 160                                     })
 161         if app['Current Version Code'] == '0':
 162             cantupdate = True
 163         # Sort with most recent first...
 164         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 165
 166         wikidata += "=Versions=\n"
 167         if len(apklist) == 0:
 168             wikidata += "We currently have no versions of this app available."
 169         elif not gotcurrentver:
 170             wikidata += "We don't have the current version of this app."
 171         else:
 172             wikidata += "We have the current version of this app."
 173         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 174         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 175         if len(app['No Source Since']) > 0:
 176             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 177         if len(app['Current Version']) > 0:
 178             wikidata += "The current (recommended) version is " + app['Current Version']
 179             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 180         validapks = 0
 181         for apk in apklist:
 182             wikidata += "==" + apk['version'] + "==\n"
 183
 184             if 'buildproblem' in apk:
 185                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 186             else:
 187                 validapks += 1
 188                 wikidata += "This version is built and signed by "
 189                 if 'srcname' in apk:
 190                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 191                 else:
 192                     wikidata += "the original developer.\n\n"
 193             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 194
 195         wikidata += '\n[[Category:' + wikicat + ']]\n'
 196         if len(app['No Source Since']) > 0:
 197             wikidata += '\n[[Category:Apps missing source code]]\n'
 198         if validapks == 0 and not app['Disabled']:
 199             wikidata += '\n[[Category:Apps with no packages]]\n'
 200         if cantupdate and not app['Disabled']:
 201             wikidata += "\n[[Category:Apps we can't update]]\n"
 202         if buildfails and not app['Disabled']:
 203             wikidata += "\n[[Category:Apps with failing builds]]\n"
 204         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 205             wikidata += '\n[[Category:Apps to Update]]\n'
 206         if app['Disabled']:
 207             wikidata += '\n[[Category:Apps that are disabled]]\n'
 208         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 209             wikidata += '\n[[Category:Apps with no update check]]\n'
 210         for appcat in app['Categories']:
 211             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 212
 213         # We can't have underscores in the page name, even if they're in
 214         # the package ID, because MediaWiki messes with them...
 215         pagename = appid.replace('_', ' ')
 216
 217         # Drop a trailing newline, because mediawiki is going to drop it anyway
 218         # and it we don't we'll think the page has changed when it hasn't...
 219         if wikidata.endswith('\n'):
 220             wikidata = wikidata[:-1]
 221
 222         generated_pages[pagename] = wikidata
 223
 224         # Make a redirect from the name to the ID too, unless there's
 225         # already an existing page with the name and it isn't a redirect.
 226         noclobber = False
 227         apppagename = app['Name'].replace('_', ' ')
 228         apppagename = apppagename.replace('{', '')
 229         apppagename = apppagename.replace('}', ' ')
 230         apppagename = apppagename.replace(':', ' ')
 231         # Drop double spaces caused mostly by replacing ':' above
 232         apppagename = apppagename.replace('  ', ' ')
 233         for expagename in site.allpages(prefix=apppagename,
 234                                         filterredir='nonredirects',
 235                                         generator=False):
 236             if expagename == apppagename:
 237                 noclobber = True
 238         # Another reason not to make the redirect page is if the app name
 239         # is the same as it's ID, because that will overwrite the real page
 240         # with an redirect to itself! (Although it seems like an odd
 241         # scenario this happens a lot, e.g. where there is metadata but no
 242         # builds or binaries to extract a name from.
 243         if apppagename == pagename:
 244             noclobber = True
 245         if not noclobber:
 246             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 247
 248     for tcat, genp in [(wikicat, generated_pages),
 249                        (wikiredircat, generated_redirects)]:
 250         catpages = site.Pages['Category:' + tcat]
 251         existingpages = []
 252         for page in catpages:
 253             existingpages.append(page.name)
 254             if page.name in genp:
 255                 pagetxt = page.edit()
 256                 if pagetxt != genp[page.name]:
 257                     logging.debug("Updating modified page " + page.name)
 258                     page.save(genp[page.name], summary='Auto-updated')
 259                 else:
 260                     logging.debug("Page " + page.name + " is unchanged")
 261             else:
 262                 logging.warn("Deleting page " + page.name)
 263                 page.delete('No longer published')
 264         for pagename, text in genp.items():
 265             logging.debug("Checking " + pagename)
 266             if pagename not in existingpages:
 267                 logging.debug("Creating page " + pagename)
 268                 try:
 269                     newpage = site.Pages[pagename]
 270                     newpage.save(text, summary='Auto-created')
 271                 except:
 272                     logging.error("...FAILED to create page '{0}'".format(pagename))
 273
 274     # Purge server cache to ensure counts are up to date
 275     site.pages['Repository Maintenance'].purge()
 276
 277
 278 def delete_disabled_builds(apps, apkcache, repodirs):
 279     """Delete disabled build outputs.
 280
 281     :param apps: list of all applications, as per metadata.read_metadata
 282     :param apkcache: current apk cache information
 283     :param repodirs: the repo directories to process
 284     """
 285     for appid, app in apps.iteritems():
 286         for build in app['builds']:
 287             if build['disable']:
 288                 apkfilename = appid + '_' + str(build['vercode']) + '.apk'
 289                 for repodir in repodirs:
 290                     apkpath = os.path.join(repodir, apkfilename)
 291                     ascpath = apkpath + ".asc"
 292                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 293                     for name in [apkpath, srcpath, ascpath]:
 294                         if os.path.exists(name):
 295                             logging.warn("Deleting disabled build output " + apkfilename)
 296                             os.remove(name)
 297                 if apkfilename in apkcache:
 298                     del apkcache[apkfilename]
 299
 300
 301 def resize_icon(iconpath, density):
 302
 303     if not os.path.isfile(iconpath):
 304         return
 305
 306     try:
 307         im = Image.open(iconpath)
 308         size = dpi_to_px(density)
 309
 310         if any(length > size for length in im.size):
 311             oldsize = im.size
 312             im.thumbnail((size, size), Image.ANTIALIAS)
 313             logging.debug("%s was too large at %s - new size is %s" % (
 314                 iconpath, oldsize, im.size))
 315             im.save(iconpath, "PNG")
 316
 317     except Exception, e:
 318         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 319
 320
 321 def resize_all_icons(repodirs):
 322     """Resize all icons that exceed the max size
 323
 324     :param repodirs: the repo directories to process
 325     """
 326     for repodir in repodirs:
 327         for density in get_densities():
 328             icon_dir = get_icon_dir(repodir, density)
 329             icon_glob = os.path.join(icon_dir, '*.png')
 330             for iconpath in glob.glob(icon_glob):
 331                 resize_icon(iconpath, density)
 332
 333
 334 # A signature block file with a .DSA, .RSA, or .EC extension
 335 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 336
 337
 338 def getsig(apkpath):
 339     """ Get the signing certificate of an apk. To get the same md5 has that
 340     Android gets, we encode the .RSA certificate in a specific format and pass
 341     it hex-encoded to the md5 digest algorithm.
 342
 343     :param apkpath: path to the apk
 344     :returns: A string containing the md5 of the signature of the apk or None
 345               if an error occurred.
 346     """
 347
 348     cert = None
 349
 350     # verify the jar signature is correct
 351     args = ['jarsigner', '-verify', apkpath]
 352     p = FDroidPopen(args)
 353     if p.returncode != 0:
 354         logging.critical(apkpath + " has a bad signature!")
 355         return None
 356
 357     with zipfile.ZipFile(apkpath, 'r') as apk:
 358
 359         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 360
 361         if len(certs) < 1:
 362             logging.error("Found no signing certificates on %s" % apkpath)
 363             return None
 364         if len(certs) > 1:
 365             logging.error("Found multiple signing certificates on %s" % apkpath)
 366             return None
 367
 368         cert = apk.read(certs[0])
 369
 370     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 371     if content.getComponentByName('contentType') != rfc2315.signedData:
 372         logging.error("Unexpected format.")
 373         return None
 374
 375     content = decoder.decode(content.getComponentByName('content'),
 376                              asn1Spec=rfc2315.SignedData())[0]
 377     try:
 378         certificates = content.getComponentByName('certificates')
 379     except PyAsn1Error:
 380         logging.error("Certificates not found.")
 381         return None
 382
 383     cert_encoded = encoder.encode(certificates)[4:]
 384
 385     return md5(cert_encoded.encode('hex')).hexdigest()
 386
 387
 388 def scan_apks(apps, apkcache, repodir, knownapks):
 389     """Scan the apks in the given repo directory.
 390
 391     This also extracts the icons.
 392
 393     :param apps: list of all applications, as per metadata.read_metadata
 394     :param apkcache: current apk cache information
 395     :param repodir: repo directory to scan
 396     :param knownapks: known apks info
 397     :returns: (apks, cachechanged) where apks is a list of apk information,
 398               and cachechanged is True if the apkcache got changed.
 399     """
 400
 401     cachechanged = False
 402
 403     icon_dirs = get_icon_dirs(repodir)
 404     for icon_dir in icon_dirs:
 405         if os.path.exists(icon_dir):
 406             if options.clean:
 407                 shutil.rmtree(icon_dir)
 408                 os.makedirs(icon_dir)
 409         else:
 410             os.makedirs(icon_dir)
 411
 412     apks = []
 413     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 414     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 415     vername_pat = re.compile(".*versionName='([^']*)'.*")
 416     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 417     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 418     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 419     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 420     string_pat = re.compile(".*'([^']*)'.*")
 421     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 422
 423         apkfilename = apkfile[len(repodir) + 1:]
 424         if ' ' in apkfilename:
 425             logging.critical("Spaces in filenames are not allowed.")
 426             sys.exit(1)
 427
 428         # Calculate the sha256...
 429         sha = hashlib.sha256()
 430         with open(apkfile, 'rb') as f:
 431             while True:
 432                 t = f.read(16384)
 433                 if len(t) == 0:
 434                     break
 435                 sha.update(t)
 436             shasum = sha.hexdigest()
 437
 438         usecache = False
 439         if apkfilename in apkcache:
 440             thisinfo = apkcache[apkfilename]
 441             if thisinfo['sha256'] == shasum:
 442                 logging.debug("Reading " + apkfilename + " from cache")
 443                 usecache = True
 444             else:
 445                 logging.debug("Ignoring stale cache data for " + apkfilename)
 446
 447         if not usecache:
 448             logging.debug("Processing " + apkfilename)
 449             thisinfo = {}
 450             thisinfo['apkname'] = apkfilename
 451             thisinfo['sha256'] = shasum
 452             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 453             if os.path.exists(os.path.join(repodir, srcfilename)):
 454                 thisinfo['srcname'] = srcfilename
 455             thisinfo['size'] = os.path.getsize(apkfile)
 456             thisinfo['permissions'] = set()
 457             thisinfo['features'] = set()
 458             thisinfo['icons_src'] = {}
 459             thisinfo['icons'] = {}
 460             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 461             if p.returncode != 0:
 462                 if options.delete_unknown:
 463                     if os.path.exists(apkfile):
 464                         logging.error("Failed to get apk information, deleting " + apkfile)
 465                         os.remove(apkfile)
 466                     else:
 467                         logging.error("Could not find {0} to remove it".format(apkfile))
 468                 else:
 469                     logging.error("Failed to get apk information, skipping " + apkfile)
 470                 continue
 471             for line in p.output.splitlines():
 472                 if line.startswith("package:"):
 473                     try:
 474                         thisinfo['id'] = re.match(name_pat, line).group(1)
 475                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 476                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 477                     except Exception, e:
 478                         logging.error("Package matching failed: " + str(e))
 479                         logging.info("Line was: " + line)
 480                         sys.exit(1)
 481                 elif line.startswith("application:"):
 482                     thisinfo['name'] = re.match(label_pat, line).group(1)
 483                     # Keep path to non-dpi icon in case we need it
 484                     match = re.match(icon_pat_nodpi, line)
 485                     if match:
 486                         thisinfo['icons_src']['-1'] = match.group(1)
 487                 elif line.startswith("launchable-activity:"):
 488                     # Only use launchable-activity as fallback to application
 489                     if not thisinfo['name']:
 490                         thisinfo['name'] = re.match(label_pat, line).group(1)
 491                     if '-1' not in thisinfo['icons_src']:
 492                         match = re.match(icon_pat_nodpi, line)
 493                         if match:
 494                             thisinfo['icons_src']['-1'] = match.group(1)
 495                 elif line.startswith("application-icon-"):
 496                     match = re.match(icon_pat, line)
 497                     if match:
 498                         density = match.group(1)
 499                         path = match.group(2)
 500                         thisinfo['icons_src'][density] = path
 501                 elif line.startswith("sdkVersion:"):
 502                     m = re.match(sdkversion_pat, line)
 503                     if m is None:
 504                         logging.error(line.replace('sdkVersion:', '')
 505                                       + ' is not a valid minSdkVersion!')
 506                     else:
 507                         thisinfo['sdkversion'] = m.group(1)
 508                 elif line.startswith("maxSdkVersion:"):
 509                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 510                 elif line.startswith("native-code:"):
 511                     thisinfo['nativecode'] = []
 512                     for arch in line[13:].split(' '):
 513                         thisinfo['nativecode'].append(arch[1:-1])
 514                 elif line.startswith("uses-permission:"):
 515                     perm = re.match(string_pat, line).group(1)
 516                     if perm.startswith("android.permission."):
 517                         perm = perm[19:]
 518                     thisinfo['permissions'].add(perm)
 519                 elif line.startswith("uses-feature:"):
 520                     perm = re.match(string_pat, line).group(1)
 521                     # Filter out this, it's only added with the latest SDK tools and
 522                     # causes problems for lots of apps.
 523                     if perm != "android.hardware.screen.portrait" \
 524                             and perm != "android.hardware.screen.landscape":
 525                         if perm.startswith("android.feature."):
 526                             perm = perm[16:]
 527                         thisinfo['features'].add(perm)
 528
 529             if 'sdkversion' not in thisinfo:
 530                 logging.warn("No SDK version information found in {0}".format(apkfile))
 531                 thisinfo['sdkversion'] = 0
 532
 533             # Check for debuggable apks...
 534             if common.isApkDebuggable(apkfile, config):
 535                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 536
 537             # Get the signature (or md5 of, to be precise)...
 538             logging.debug('Getting signature of {0}'.format(apkfile))
 539             thisinfo['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 540             if not thisinfo['sig']:
 541                 logging.critical("Failed to get apk signature")
 542                 sys.exit(1)
 543
 544             apk = zipfile.ZipFile(apkfile, 'r')
 545
 546             # if an APK has files newer than the system time, suggest updating
 547             # the system clock.  This is useful for offline systems, used for
 548             # signing, which do not have another source of clock sync info. It
 549             # has to be more than 24 hours newer because ZIP/APK files do not
 550             # store timezone info
 551             info = apk.getinfo('AndroidManifest.xml')
 552             dt_obj = datetime(*info.date_time)
 553             checkdt = dt_obj - timedelta(1)
 554             if datetime.today() < checkdt:
 555                 logging.warn('System clock is older than manifest in: '
 556                              + apkfilename + '\nSet clock to that time using:\n'
 557                              + 'sudo date -s "' + str(dt_obj) + '"')
 558
 559             iconfilename = "%s.%s.png" % (
 560                 thisinfo['id'],
 561                 thisinfo['versioncode'])
 562
 563             # Extract the icon file...
 564             densities = get_densities()
 565             empty_densities = []
 566             for density in densities:
 567                 if density not in thisinfo['icons_src']:
 568                     empty_densities.append(density)
 569                     continue
 570                 iconsrc = thisinfo['icons_src'][density]
 571                 icon_dir = get_icon_dir(repodir, density)
 572                 icondest = os.path.join(icon_dir, iconfilename)
 573
 574                 try:
 575                     with open(icondest, 'wb') as f:
 576                         f.write(apk.read(iconsrc))
 577                     thisinfo['icons'][density] = iconfilename
 578
 579                 except:
 580                     logging.warn("Error retrieving icon file")
 581                     del thisinfo['icons'][density]
 582                     del thisinfo['icons_src'][density]
 583                     empty_densities.append(density)
 584
 585             if '-1' in thisinfo['icons_src']:
 586                 iconsrc = thisinfo['icons_src']['-1']
 587                 iconpath = os.path.join(
 588                     get_icon_dir(repodir, None), iconfilename)
 589                 with open(iconpath, 'wb') as f:
 590                     f.write(apk.read(iconsrc))
 591                 try:
 592                     im = Image.open(iconpath)
 593                     dpi = px_to_dpi(im.size[0])
 594                     for density in densities:
 595                         if density in thisinfo['icons']:
 596                             break
 597                         if density == densities[-1] or dpi >= int(density):
 598                             thisinfo['icons'][density] = iconfilename
 599                             shutil.move(iconpath,
 600                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 601                             empty_densities.remove(density)
 602                             break
 603                 except Exception, e:
 604                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 605
 606             if thisinfo['icons']:
 607                 thisinfo['icon'] = iconfilename
 608
 609             apk.close()
 610
 611             # First try resizing down to not lose quality
 612             last_density = None
 613             for density in densities:
 614                 if density not in empty_densities:
 615                     last_density = density
 616                     continue
 617                 if last_density is None:
 618                     continue
 619                 logging.debug("Density %s not available, resizing down from %s"
 620                               % (density, last_density))
 621
 622                 last_iconpath = os.path.join(
 623                     get_icon_dir(repodir, last_density), iconfilename)
 624                 iconpath = os.path.join(
 625                     get_icon_dir(repodir, density), iconfilename)
 626                 try:
 627                     im = Image.open(last_iconpath)
 628                 except:
 629                     logging.warn("Invalid image file at %s" % last_iconpath)
 630                     continue
 631
 632                 size = dpi_to_px(density)
 633
 634                 im.thumbnail((size, size), Image.ANTIALIAS)
 635                 im.save(iconpath, "PNG")
 636                 empty_densities.remove(density)
 637
 638             # Then just copy from the highest resolution available
 639             last_density = None
 640             for density in reversed(densities):
 641                 if density not in empty_densities:
 642                     last_density = density
 643                     continue
 644                 if last_density is None:
 645                     continue
 646                 logging.debug("Density %s not available, copying from lower density %s"
 647                               % (density, last_density))
 648
 649                 shutil.copyfile(
 650                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 651                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 652
 653                 empty_densities.remove(density)
 654
 655             for density in densities:
 656                 icon_dir = get_icon_dir(repodir, density)
 657                 icondest = os.path.join(icon_dir, iconfilename)
 658                 resize_icon(icondest, density)
 659
 660             # Copy from icons-mdpi to icons since mdpi is the baseline density
 661             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 662             if os.path.isfile(baseline):
 663                 shutil.copyfile(baseline,
 664                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 665
 666             # Record in known apks, getting the added date at the same time..
 667             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 668             if added:
 669                 thisinfo['added'] = added
 670
 671             apkcache[apkfilename] = thisinfo
 672             cachechanged = True
 673
 674         apks.append(thisinfo)
 675
 676     return apks, cachechanged
 677
 678
 679 repo_pubkey_fingerprint = None
 680
 681
 682 # Generate a certificate fingerprint the same way keytool does it
 683 # (but with slightly different formatting)
 684 def cert_fingerprint(data):
 685     digest = hashlib.sha256(data).digest()
 686     ret = []
 687     ret.append(' '.join("%02X" % ord(b) for b in digest))
 688     return " ".join(ret)
 689
 690
 691 def extract_pubkey():
 692     global repo_pubkey_fingerprint
 693     if 'repo_pubkey' in config:
 694         pubkey = unhexlify(config['repo_pubkey'])
 695     else:
 696         p = FDroidPopen(['keytool', '-exportcert',
 697                          '-alias', config['repo_keyalias'],
 698                          '-keystore', config['keystore'],
 699                          '-storepass:file', config['keystorepassfile']]
 700                         + config['smartcardoptions'], output=False)
 701         if p.returncode != 0 or len(p.output) < 20:
 702             msg = "Failed to get repo pubkey!"
 703             if config['keystore'] == 'NONE':
 704                 msg += ' Is your crypto smartcard plugged in?'
 705             logging.critical(msg)
 706             sys.exit(1)
 707         pubkey = p.output
 708     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 709     return hexlify(pubkey)
 710
 711
 712 def make_index(apps, sortedids, apks, repodir, archive, categories):
 713     """Make a repo index.
 714
 715     :param apps: fully populated apps list
 716     :param apks: full populated apks list
 717     :param repodir: the repo directory
 718     :param archive: True if this is the archive repo, False if it's the
 719                     main one.
 720     :param categories: list of categories
 721     """
 722
 723     doc = Document()
 724
 725     def addElement(name, value, doc, parent):
 726         el = doc.createElement(name)
 727         el.appendChild(doc.createTextNode(value))
 728         parent.appendChild(el)
 729
 730     def addElementNonEmpty(name, value, doc, parent):
 731         if not value:
 732             return
 733         addElement(name, value, doc, parent)
 734
 735     def addElementCDATA(name, value, doc, parent):
 736         el = doc.createElement(name)
 737         el.appendChild(doc.createCDATASection(value))
 738         parent.appendChild(el)
 739
 740     root = doc.createElement("fdroid")
 741     doc.appendChild(root)
 742
 743     repoel = doc.createElement("repo")
 744
 745     if archive:
 746         repoel.setAttribute("name", config['archive_name'])
 747         if config['repo_maxage'] != 0:
 748             repoel.setAttribute("maxage", str(config['repo_maxage']))
 749         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 750         repoel.setAttribute("url", config['archive_url'])
 751         addElement('description', config['archive_description'], doc, repoel)
 752
 753     else:
 754         repoel.setAttribute("name", config['repo_name'])
 755         if config['repo_maxage'] != 0:
 756             repoel.setAttribute("maxage", str(config['repo_maxage']))
 757         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 758         repoel.setAttribute("url", config['repo_url'])
 759         addElement('description', config['repo_description'], doc, repoel)
 760
 761     repoel.setAttribute("version", "13")
 762     repoel.setAttribute("timestamp", str(int(time.time())))
 763
 764     nosigningkey = False
 765     if not options.nosign:
 766         if 'repo_keyalias' not in config:
 767             nosigningkey = True
 768             logging.critical("'repo_keyalias' not found in config.py!")
 769         if 'keystore' not in config:
 770             nosigningkey = True
 771             logging.critical("'keystore' not found in config.py!")
 772         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 773             nosigningkey = True
 774             logging.critical("'keystorepass' not found in config.py!")
 775         if 'keypass' not in config and 'keypassfile' not in config:
 776             nosigningkey = True
 777             logging.critical("'keypass' not found in config.py!")
 778         if not os.path.exists(config['keystore']):
 779             nosigningkey = True
 780             logging.critical("'" + config['keystore'] + "' does not exist!")
 781         if nosigningkey:
 782             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 783             logging.warning("\tfdroid update --create-key")
 784             sys.exit(1)
 785
 786     repoel.setAttribute("pubkey", extract_pubkey())
 787     root.appendChild(repoel)
 788
 789     for appid in sortedids:
 790         app = apps[appid]
 791
 792         if app['Disabled'] is not None:
 793             continue
 794
 795         # Get a list of the apks for this app...
 796         apklist = []
 797         for apk in apks:
 798             if apk['id'] == appid:
 799                 apklist.append(apk)
 800
 801         if len(apklist) == 0:
 802             continue
 803
 804         apel = doc.createElement("application")
 805         apel.setAttribute("id", app['id'])
 806         root.appendChild(apel)
 807
 808         addElement('id', app['id'], doc, apel)
 809         if 'added' in app:
 810             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 811         if 'lastupdated' in app:
 812             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 813         addElement('name', app['Name'], doc, apel)
 814         addElement('summary', app['Summary'], doc, apel)
 815         if app['icon']:
 816             addElement('icon', app['icon'], doc, apel)
 817
 818         def linkres(appid):
 819             if appid in apps:
 820                 return ("fdroid.app:" + appid, apps[appid]['Name'])
 821             raise MetaDataException("Cannot resolve app id " + appid)
 822
 823         addElement('desc',
 824                    metadata.description_html(app['Description'], linkres),
 825                    doc, apel)
 826         addElement('license', app['License'], doc, apel)
 827         if 'Categories' in app and app['Categories']:
 828             addElement('categories', ','.join(app["Categories"]), doc, apel)
 829             # We put the first (primary) category in LAST, which will have
 830             # the desired effect of making clients that only understand one
 831             # category see that one.
 832             addElement('category', app["Categories"][0], doc, apel)
 833         addElement('web', app['Web Site'], doc, apel)
 834         addElement('source', app['Source Code'], doc, apel)
 835         addElement('tracker', app['Issue Tracker'], doc, apel)
 836         addElementNonEmpty('changelog', app['Changelog'], doc, apel)
 837         addElementNonEmpty('donate', app['Donate'], doc, apel)
 838         addElementNonEmpty('bitcoin', app['Bitcoin'], doc, apel)
 839         addElementNonEmpty('litecoin', app['Litecoin'], doc, apel)
 840         addElementNonEmpty('dogecoin', app['Dogecoin'], doc, apel)
 841         addElementNonEmpty('flattr', app['FlattrID'], doc, apel)
 842
 843         # These elements actually refer to the current version (i.e. which
 844         # one is recommended. They are historically mis-named, and need
 845         # changing, but stay like this for now to support existing clients.
 846         addElement('marketversion', app['Current Version'], doc, apel)
 847         addElement('marketvercode', app['Current Version Code'], doc, apel)
 848
 849         if app['AntiFeatures']:
 850             af = app['AntiFeatures'].split(',')
 851             if af:
 852                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 853         if app['Provides']:
 854             pv = app['Provides'].split(',')
 855             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 856         if app['Requires Root']:
 857             addElement('requirements', 'root', doc, apel)
 858
 859         # Sort the apk list into version order, just so the web site
 860         # doesn't have to do any work by default...
 861         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 862
 863         # Check for duplicates - they will make the client unhappy...
 864         for i in range(len(apklist) - 1):
 865             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 866                 logging.critical("duplicate versions: '%s' - '%s'" % (
 867                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 868                 sys.exit(1)
 869
 870         current_version_code = 0
 871         current_version_file = None
 872         for apk in apklist:
 873             # find the APK for the "Current Version"
 874             if current_version_code < apk['versioncode']:
 875                 current_version_code = apk['versioncode']
 876             if current_version_code < int(app['Current Version Code']):
 877                 current_version_file = apk['apkname']
 878
 879             apkel = doc.createElement("package")
 880             apel.appendChild(apkel)
 881             addElement('version', apk['version'], doc, apkel)
 882             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 883             addElement('apkname', apk['apkname'], doc, apkel)
 884             if 'srcname' in apk:
 885                 addElement('srcname', apk['srcname'], doc, apkel)
 886             for hash_type in ['sha256']:
 887                 if hash_type not in apk:
 888                     continue
 889                 hashel = doc.createElement("hash")
 890                 hashel.setAttribute("type", hash_type)
 891                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 892                 apkel.appendChild(hashel)
 893             addElement('sig', apk['sig'], doc, apkel)
 894             addElement('size', str(apk['size']), doc, apkel)
 895             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 896             if 'maxsdkversion' in apk:
 897                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 898             if 'added' in apk:
 899                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 900             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
 901             if 'nativecode' in apk:
 902                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 903             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
 904
 905         if current_version_file is not None \
 906                 and config['make_current_version_link'] \
 907                 and repodir == 'repo':  # only create these
 908             sanitized_name = re.sub('''[ '"&%?+=/]''', '',
 909                                     app[config['current_version_name_source']])
 910             apklinkname = sanitized_name + '.apk'
 911             current_version_path = os.path.join(repodir, current_version_file)
 912             if os.path.exists(apklinkname):
 913                 os.remove(apklinkname)
 914             os.symlink(current_version_path, apklinkname)
 915             # also symlink gpg signature, if it exists
 916             for extension in ('.asc', '.sig'):
 917                 sigfile_path = current_version_path + extension
 918                 if os.path.exists(sigfile_path):
 919                     siglinkname = apklinkname + extension
 920                     if os.path.exists(siglinkname):
 921                         os.remove(siglinkname)
 922                     os.symlink(sigfile_path, siglinkname)
 923
 924     if options.pretty:
 925         output = doc.toprettyxml()
 926     else:
 927         output = doc.toxml()
 928
 929     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 930         f.write(output)
 931
 932     if 'repo_keyalias' in config:
 933
 934         if options.nosign:
 935             logging.info("Creating unsigned index in preparation for signing")
 936         else:
 937             logging.info("Creating signed index with this key (SHA256):")
 938             logging.info("%s" % repo_pubkey_fingerprint)
 939
 940         # Create a jar of the index...
 941         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
 942         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 943         if p.returncode != 0:
 944             logging.critical("Failed to create {0}".format(jar_output))
 945             sys.exit(1)
 946
 947         # Sign the index...
 948         signed = os.path.join(repodir, 'index.jar')
 949         if options.nosign:
 950             # Remove old signed index if not signing
 951             if os.path.exists(signed):
 952                 os.remove(signed)
 953         else:
 954             args = ['jarsigner', '-keystore', config['keystore'],
 955                     '-storepass:file', config['keystorepassfile'],
 956                     '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 957                     signed, config['repo_keyalias']]
 958             if config['keystore'] == 'NONE':
 959                 args += config['smartcardoptions']
 960             else:  # smardcards never use -keypass
 961                 args += ['-keypass:file', config['keypassfile']]
 962             p = FDroidPopen(args)
 963             if p.returncode != 0:
 964                 logging.critical("Failed to sign index")
 965                 sys.exit(1)
 966
 967     # Copy the repo icon into the repo directory...
 968     icon_dir = os.path.join(repodir, 'icons')
 969     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 970     shutil.copyfile(config['repo_icon'], iconfilename)
 971
 972     # Write a category list in the repo to allow quick access...
 973     catdata = ''
 974     for cat in categories:
 975         catdata += cat + '\n'
 976     with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
 977         f.write(catdata)
 978
 979
 980 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 981
 982     for appid, app in apps.iteritems():
 983
 984         # Get a list of the apks for this app...
 985         apklist = []
 986         for apk in apks:
 987             if apk['id'] == appid:
 988                 apklist.append(apk)
 989
 990         # Sort the apk list into version order...
 991         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 992
 993         if app['Archive Policy']:
 994             keepversions = int(app['Archive Policy'][:-9])
 995         else:
 996             keepversions = defaultkeepversions
 997
 998         if len(apklist) > keepversions:
 999             for apk in apklist[keepversions:]:
1000                 logging.info("Moving " + apk['apkname'] + " to archive")
1001                 shutil.move(os.path.join(repodir, apk['apkname']),
1002                             os.path.join(archivedir, apk['apkname']))
1003                 if 'srcname' in apk:
1004                     shutil.move(os.path.join(repodir, apk['srcname']),
1005                                 os.path.join(archivedir, apk['srcname']))
1006                     # Move GPG signature too...
1007                     sigfile = apk['srcname'] + '.asc'
1008                     sigsrc = os.path.join(repodir, sigfile)
1009                     if os.path.exists(sigsrc):
1010                         shutil.move(sigsrc, os.path.join(archivedir, sigfile))
1011
1012                 archapks.append(apk)
1013                 apks.remove(apk)
1014
1015
1016 def add_apks_to_per_app_repos(repodir, apks):
1017     apks_per_app = dict()
1018     for apk in apks:
1019         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1020         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1021         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1022         apks_per_app[apk['id']] = apk
1023
1024         if not os.path.exists(apk['per_app_icons']):
1025             logging.info('Adding new repo for only ' + apk['id'])
1026             os.makedirs(apk['per_app_icons'])
1027
1028         apkpath = os.path.join(repodir, apk['apkname'])
1029         shutil.copy(apkpath, apk['per_app_repo'])
1030         apksigpath = apkpath + '.sig'
1031         if os.path.exists(apksigpath):
1032             shutil.copy(apksigpath, apk['per_app_repo'])
1033         apkascpath = apkpath + '.asc'
1034         if os.path.exists(apkascpath):
1035             shutil.copy(apkascpath, apk['per_app_repo'])
1036
1037
1038 config = None
1039 options = None
1040
1041
1042 def main():
1043
1044     global config, options
1045
1046     # Parse command line...
1047     parser = OptionParser()
1048     parser.add_option("--create-key", action="store_true", default=False,
1049                       help="Create a repo signing key in a keystore")
1050     parser.add_option("-c", "--create-metadata", action="store_true", default=False,
1051                       help="Create skeleton metadata files that are missing")
1052     parser.add_option("--delete-unknown", action="store_true", default=False,
1053                       help="Delete APKs without metadata from the repo")
1054     parser.add_option("-v", "--verbose", action="store_true", default=False,
1055                       help="Spew out even more information than normal")
1056     parser.add_option("-q", "--quiet", action="store_true", default=False,
1057                       help="Restrict output to warnings and errors")
1058     parser.add_option("-b", "--buildreport", action="store_true", default=False,
1059                       help="Report on build data status")
1060     parser.add_option("-i", "--interactive", default=False, action="store_true",
1061                       help="Interactively ask about things that need updating.")
1062     parser.add_option("-I", "--icons", action="store_true", default=False,
1063                       help="Resize all the icons exceeding the max pixel size and exit")
1064     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
1065                       help="Specify editor to use in interactive mode. Default " +
1066                       "is /etc/alternatives/editor")
1067     parser.add_option("-w", "--wiki", default=False, action="store_true",
1068                       help="Update the wiki")
1069     parser.add_option("", "--pretty", action="store_true", default=False,
1070                       help="Produce human-readable index.xml")
1071     parser.add_option("--clean", action="store_true", default=False,
1072                       help="Clean update - don't uses caches, reprocess all apks")
1073     parser.add_option("--nosign", action="store_true", default=False,
1074                       help="When configured for signed indexes, create only unsigned indexes at this stage")
1075     (options, args) = parser.parse_args()
1076
1077     config = common.read_config(options)
1078
1079     repodirs = ['repo']
1080     if config['archive_older'] != 0:
1081         repodirs.append('archive')
1082         if not os.path.exists('archive'):
1083             os.mkdir('archive')
1084
1085     if options.icons:
1086         resize_all_icons(repodirs)
1087         sys.exit(0)
1088
1089     # check that icons exist now, rather than fail at the end of `fdroid update`
1090     for k in ['repo_icon', 'archive_icon']:
1091         if k in config:
1092             if not os.path.exists(config[k]):
1093                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1094                 sys.exit(1)
1095
1096     # if the user asks to create a keystore, do it now, reusing whatever it can
1097     if options.create_key:
1098         if os.path.exists(config['keystore']):
1099             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1100             logging.critical("\t'" + config['keystore'] + "'")
1101             sys.exit(1)
1102
1103         if 'repo_keyalias' not in config:
1104             config['repo_keyalias'] = socket.getfqdn()
1105             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1106         if 'keydname' not in config:
1107             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1108             common.write_to_config(config, 'keydname', config['keydname'])
1109         if 'keystore' not in config:
1110             config['keystore'] = common.default_config.keystore
1111             common.write_to_config(config, 'keystore', config['keystore'])
1112
1113         password = common.genpassword()
1114         if 'keystorepass' not in config:
1115             config['keystorepass'] = password
1116             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1117         if 'keypass' not in config:
1118             config['keypass'] = password
1119             common.write_to_config(config, 'keypass', config['keypass'])
1120         common.genkeystore(config)
1121
1122     # Get all apps...
1123     apps = metadata.read_metadata()
1124
1125     # Generate a list of categories...
1126     categories = set()
1127     for app in apps.itervalues():
1128         categories.update(app['Categories'])
1129
1130     # Read known apks data (will be updated and written back when we've finished)
1131     knownapks = common.KnownApks()
1132
1133     # Gather information about all the apk files in the repo directory, using
1134     # cached data if possible.
1135     apkcachefile = os.path.join('tmp', 'apkcache')
1136     if not options.clean and os.path.exists(apkcachefile):
1137         with open(apkcachefile, 'rb') as cf:
1138             apkcache = pickle.load(cf)
1139     else:
1140         apkcache = {}
1141
1142     delete_disabled_builds(apps, apkcache, repodirs)
1143
1144     # Scan all apks in the main repo
1145     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks)
1146
1147     # Generate warnings for apk's with no metadata (or create skeleton
1148     # metadata files, if requested on the command line)
1149     newmetadata = False
1150     for apk in apks:
1151         if apk['id'] not in apps:
1152             if options.create_metadata:
1153                 if 'name' not in apk:
1154                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1155                     continue
1156                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1157                 f.write("License:Unknown\n")
1158                 f.write("Web Site:\n")
1159                 f.write("Source Code:\n")
1160                 f.write("Issue Tracker:\n")
1161                 f.write("Changelog:\n")
1162                 f.write("Summary:" + apk['name'] + "\n")
1163                 f.write("Description:\n")
1164                 f.write(apk['name'] + "\n")
1165                 f.write(".\n")
1166                 f.close()
1167                 logging.info("Generated skeleton metadata for " + apk['id'])
1168                 newmetadata = True
1169             else:
1170                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1171                 if options.delete_unknown:
1172                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1173                     rmf = os.path.join(repodirs[0], apk['apkname'])
1174                     if not os.path.exists(rmf):
1175                         logging.error("Could not find {0} to remove it".format(rmf))
1176                     else:
1177                         os.remove(rmf)
1178                 else:
1179                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1180
1181     # update the metadata with the newly created ones included
1182     if newmetadata:
1183         apps = metadata.read_metadata()
1184
1185     # Scan the archive repo for apks as well
1186     if len(repodirs) > 1:
1187         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1188         if cc:
1189             cachechanged = True
1190     else:
1191         archapks = []
1192
1193     # Some information from the apks needs to be applied up to the application
1194     # level. When doing this, we use the info from the most recent version's apk.
1195     # We deal with figuring out when the app was added and last updated at the
1196     # same time.
1197     for appid, app in apps.iteritems():
1198         bestver = 0
1199         added = None
1200         lastupdated = None
1201         for apk in apks + archapks:
1202             if apk['id'] == appid:
1203                 if apk['versioncode'] > bestver:
1204                     bestver = apk['versioncode']
1205                     bestapk = apk
1206
1207                 if 'added' in apk:
1208                     if not added or apk['added'] < added:
1209                         added = apk['added']
1210                     if not lastupdated or apk['added'] > lastupdated:
1211                         lastupdated = apk['added']
1212
1213         if added:
1214             app['added'] = added
1215         else:
1216             logging.warn("Don't know when " + appid + " was added")
1217         if lastupdated:
1218             app['lastupdated'] = lastupdated
1219         else:
1220             logging.warn("Don't know when " + appid + " was last updated")
1221
1222         if bestver == 0:
1223             if app['Name'] is None:
1224                 app['Name'] = app['Auto Name'] or appid
1225             app['icon'] = None
1226             logging.warn("Application " + appid + " has no packages")
1227         else:
1228             if app['Name'] is None:
1229                 app['Name'] = bestapk['name']
1230             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1231
1232     # Sort the app list by name, then the web site doesn't have to by default.
1233     # (we had to wait until we'd scanned the apks to do this, because mostly the
1234     # name comes from there!)
1235     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid]['Name'].upper())
1236
1237     # APKs are placed into multiple repos based on the app package, providing
1238     # per-app subscription feeds for nightly builds and things like it
1239     if config['per_app_repos']:
1240         add_apks_to_per_app_repos(repodirs[0], apks)
1241         for appid, app in apps.iteritems():
1242             repodir = os.path.join(appid, 'fdroid', 'repo')
1243             appdict = dict()
1244             appdict[appid] = app
1245             if os.path.isdir(repodir):
1246                 make_index(appdict, [appid], apks, repodir, False, categories)
1247             else:
1248                 logging.info('Skipping index generation for ' + appid)
1249         return
1250
1251     if len(repodirs) > 1:
1252         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1253
1254     # Make the index for the main repo...
1255     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1256
1257     # If there's an archive repo,  make the index for it. We already scanned it
1258     # earlier on.
1259     if len(repodirs) > 1:
1260         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1261
1262     if config['update_stats']:
1263
1264         # Update known apks info...
1265         knownapks.writeifchanged()
1266
1267         # Generate latest apps data for widget
1268         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1269             data = ''
1270             for line in file(os.path.join('stats', 'latestapps.txt')):
1271                 appid = line.rstrip()
1272                 data += appid + "\t"
1273                 app = apps[appid]
1274                 data += app['Name'] + "\t"
1275                 if app['icon'] is not None:
1276                     data += app['icon'] + "\t"
1277                 data += app['License'] + "\n"
1278             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
1279                 f.write(data)
1280
1281     if cachechanged:
1282         with open(apkcachefile, 'wb') as cf:
1283             pickle.dump(apkcache, cf)
1284
1285     # Update the wiki...
1286     if options.wiki:
1287         update_wiki(apps, sortedids, apks + archapks)
1288
1289     logging.info("Finished.")
1290
1291 if __name__ == "__main__":
1292     main()