fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import socket
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from datetime import datetime, timedelta
  31 from xml.dom.minidom import Document
  32 from argparse import ArgumentParser
  33 import time
  34 from pyasn1.error import PyAsn1Error
  35 from pyasn1.codec.der import decoder, encoder
  36 from pyasn1_modules import rfc2315
  37 from hashlib import md5
  38 from binascii import hexlify, unhexlify
  39
  40 from PIL import Image
  41 import logging
  42
  43 import common
  44 import metadata
  45 from common import FDroidPopen, SdkToolsPopen
  46 from metadata import MetaDataException
  47
  48 screen_densities = ['640', '480', '320', '240', '160', '120']
  49
  50
  51 def dpi_to_px(density):
  52     return (int(density) * 48) / 160
  53
  54
  55 def px_to_dpi(px):
  56     return (int(px) * 160) / 48
  57
  58
  59 def get_icon_dir(repodir, density):
  60     if density is None:
  61         return os.path.join(repodir, "icons")
  62     return os.path.join(repodir, "icons-%s" % density)
  63
  64
  65 def get_icon_dirs(repodir):
  66     for density in screen_densities:
  67         yield get_icon_dir(repodir, density)
  68     yield os.path.join(repodir, "icons")
  69
  70
  71 def update_wiki(apps, sortedids, apks):
  72     """Update the wiki
  73
  74     :param apps: fully populated list of all applications
  75     :param apks: all apks, except...
  76     """
  77     logging.info("Updating wiki")
  78     wikicat = 'Apps'
  79     wikiredircat = 'App Redirects'
  80     import mwclient
  81     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  82                          path=config['wiki_path'])
  83     site.login(config['wiki_user'], config['wiki_password'])
  84     generated_pages = {}
  85     generated_redirects = {}
  86
  87     for appid in sortedids:
  88         app = apps[appid]
  89
  90         wikidata = ''
  91         if app['Disabled']:
  92             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  93         if 'AntiFeatures' in app:
  94             for af in app['AntiFeatures']:
  95                 wikidata += '{{AntiFeature|' + af + '}}\n'
  96         if app['Requires Root']:
  97             requiresroot = 'Yes'
  98         else:
  99             requiresroot = 'No'
 100         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s}}\n' % (
 101             appid,
 102             app['Name'],
 103             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
 104             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
 105             app['Source Code'],
 106             app['Issue Tracker'],
 107             app['Web Site'],
 108             app['Changelog'],
 109             app['Donate'],
 110             app['FlattrID'],
 111             app['Bitcoin'],
 112             app['Litecoin'],
 113             app['License'],
 114             requiresroot)
 115
 116         if app['Provides']:
 117             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 118
 119         wikidata += app['Summary']
 120         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 121
 122         wikidata += "=Description=\n"
 123         wikidata += metadata.description_wiki(app['Description']) + "\n"
 124
 125         wikidata += "=Maintainer Notes=\n"
 126         if 'Maintainer Notes' in app:
 127             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 128         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 129
 130         # Get a list of all packages for this application...
 131         apklist = []
 132         gotcurrentver = False
 133         cantupdate = False
 134         buildfails = False
 135         for apk in apks:
 136             if apk['id'] == appid:
 137                 if str(apk['versioncode']) == app['Current Version Code']:
 138                     gotcurrentver = True
 139                 apklist.append(apk)
 140         # Include ones we can't build, as a special case...
 141         for thisbuild in app['builds']:
 142             if thisbuild['disable']:
 143                 if thisbuild['vercode'] == app['Current Version Code']:
 144                     cantupdate = True
 145                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 146                 apklist.append({'versioncode': int(thisbuild['vercode']),
 147                                 'version': thisbuild['version'],
 148                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(thisbuild['disable']),
 149                                 })
 150             else:
 151                 builtit = False
 152                 for apk in apklist:
 153                     if apk['versioncode'] == int(thisbuild['vercode']):
 154                         builtit = True
 155                         break
 156                 if not builtit:
 157                     buildfails = True
 158                     apklist.append({'versioncode': int(thisbuild['vercode']),
 159                                     'version': thisbuild['version'],
 160                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, thisbuild['vercode']),
 161                                     })
 162         if app['Current Version Code'] == '0':
 163             cantupdate = True
 164         # Sort with most recent first...
 165         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 166
 167         wikidata += "=Versions=\n"
 168         if len(apklist) == 0:
 169             wikidata += "We currently have no versions of this app available."
 170         elif not gotcurrentver:
 171             wikidata += "We don't have the current version of this app."
 172         else:
 173             wikidata += "We have the current version of this app."
 174         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 175         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 176         if len(app['No Source Since']) > 0:
 177             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 178         if len(app['Current Version']) > 0:
 179             wikidata += "The current (recommended) version is " + app['Current Version']
 180             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 181         validapks = 0
 182         for apk in apklist:
 183             wikidata += "==" + apk['version'] + "==\n"
 184
 185             if 'buildproblem' in apk:
 186                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 187             else:
 188                 validapks += 1
 189                 wikidata += "This version is built and signed by "
 190                 if 'srcname' in apk:
 191                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 192                 else:
 193                     wikidata += "the original developer.\n\n"
 194             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 195
 196         wikidata += '\n[[Category:' + wikicat + ']]\n'
 197         if len(app['No Source Since']) > 0:
 198             wikidata += '\n[[Category:Apps missing source code]]\n'
 199         if validapks == 0 and not app['Disabled']:
 200             wikidata += '\n[[Category:Apps with no packages]]\n'
 201         if cantupdate and not app['Disabled']:
 202             wikidata += "\n[[Category:Apps we can't update]]\n"
 203         if buildfails and not app['Disabled']:
 204             wikidata += "\n[[Category:Apps with failing builds]]\n"
 205         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 206             wikidata += '\n[[Category:Apps to Update]]\n'
 207         if app['Disabled']:
 208             wikidata += '\n[[Category:Apps that are disabled]]\n'
 209         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 210             wikidata += '\n[[Category:Apps with no update check]]\n'
 211         for appcat in app['Categories']:
 212             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 213
 214         # We can't have underscores in the page name, even if they're in
 215         # the package ID, because MediaWiki messes with them...
 216         pagename = appid.replace('_', ' ')
 217
 218         # Drop a trailing newline, because mediawiki is going to drop it anyway
 219         # and it we don't we'll think the page has changed when it hasn't...
 220         if wikidata.endswith('\n'):
 221             wikidata = wikidata[:-1]
 222
 223         generated_pages[pagename] = wikidata
 224
 225         # Make a redirect from the name to the ID too, unless there's
 226         # already an existing page with the name and it isn't a redirect.
 227         noclobber = False
 228         apppagename = app['Name'].replace('_', ' ')
 229         apppagename = apppagename.replace('{', '')
 230         apppagename = apppagename.replace('}', ' ')
 231         apppagename = apppagename.replace(':', ' ')
 232         # Drop double spaces caused mostly by replacing ':' above
 233         apppagename = apppagename.replace('  ', ' ')
 234         for expagename in site.allpages(prefix=apppagename,
 235                                         filterredir='nonredirects',
 236                                         generator=False):
 237             if expagename == apppagename:
 238                 noclobber = True
 239         # Another reason not to make the redirect page is if the app name
 240         # is the same as it's ID, because that will overwrite the real page
 241         # with an redirect to itself! (Although it seems like an odd
 242         # scenario this happens a lot, e.g. where there is metadata but no
 243         # builds or binaries to extract a name from.
 244         if apppagename == pagename:
 245             noclobber = True
 246         if not noclobber:
 247             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 248
 249     for tcat, genp in [(wikicat, generated_pages),
 250                        (wikiredircat, generated_redirects)]:
 251         catpages = site.Pages['Category:' + tcat]
 252         existingpages = []
 253         for page in catpages:
 254             existingpages.append(page.name)
 255             if page.name in genp:
 256                 pagetxt = page.edit()
 257                 if pagetxt != genp[page.name]:
 258                     logging.debug("Updating modified page " + page.name)
 259                     page.save(genp[page.name], summary='Auto-updated')
 260                 else:
 261                     logging.debug("Page " + page.name + " is unchanged")
 262             else:
 263                 logging.warn("Deleting page " + page.name)
 264                 page.delete('No longer published')
 265         for pagename, text in genp.items():
 266             logging.debug("Checking " + pagename)
 267             if pagename not in existingpages:
 268                 logging.debug("Creating page " + pagename)
 269                 try:
 270                     newpage = site.Pages[pagename]
 271                     newpage.save(text, summary='Auto-created')
 272                 except:
 273                     logging.error("...FAILED to create page '{0}'".format(pagename))
 274
 275     # Purge server cache to ensure counts are up to date
 276     site.pages['Repository Maintenance'].purge()
 277
 278
 279 def delete_disabled_builds(apps, apkcache, repodirs):
 280     """Delete disabled build outputs.
 281
 282     :param apps: list of all applications, as per metadata.read_metadata
 283     :param apkcache: current apk cache information
 284     :param repodirs: the repo directories to process
 285     """
 286     for appid, app in apps.iteritems():
 287         for build in app['builds']:
 288             if build['disable']:
 289                 apkfilename = appid + '_' + str(build['vercode']) + '.apk'
 290                 for repodir in repodirs:
 291                     apkpath = os.path.join(repodir, apkfilename)
 292                     ascpath = apkpath + ".asc"
 293                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 294                     for name in [apkpath, srcpath, ascpath]:
 295                         if os.path.exists(name):
 296                             logging.warn("Deleting disabled build output " + apkfilename)
 297                             os.remove(name)
 298                 if apkfilename in apkcache:
 299                     del apkcache[apkfilename]
 300
 301
 302 def resize_icon(iconpath, density):
 303
 304     if not os.path.isfile(iconpath):
 305         return
 306
 307     try:
 308         im = Image.open(iconpath)
 309         size = dpi_to_px(density)
 310
 311         if any(length > size for length in im.size):
 312             oldsize = im.size
 313             im.thumbnail((size, size), Image.ANTIALIAS)
 314             logging.debug("%s was too large at %s - new size is %s" % (
 315                 iconpath, oldsize, im.size))
 316             im.save(iconpath, "PNG")
 317
 318     except Exception, e:
 319         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 320
 321
 322 def resize_all_icons(repodirs):
 323     """Resize all icons that exceed the max size
 324
 325     :param repodirs: the repo directories to process
 326     """
 327     for repodir in repodirs:
 328         for density in screen_densities:
 329             icon_dir = get_icon_dir(repodir, density)
 330             icon_glob = os.path.join(icon_dir, '*.png')
 331             for iconpath in glob.glob(icon_glob):
 332                 resize_icon(iconpath, density)
 333
 334
 335 # A signature block file with a .DSA, .RSA, or .EC extension
 336 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 337
 338
 339 def getsig(apkpath):
 340     """ Get the signing certificate of an apk. To get the same md5 has that
 341     Android gets, we encode the .RSA certificate in a specific format and pass
 342     it hex-encoded to the md5 digest algorithm.
 343
 344     :param apkpath: path to the apk
 345     :returns: A string containing the md5 of the signature of the apk or None
 346               if an error occurred.
 347     """
 348
 349     cert = None
 350
 351     # verify the jar signature is correct
 352     args = ['jarsigner', '-verify', apkpath]
 353     p = FDroidPopen(args)
 354     if p.returncode != 0:
 355         logging.critical(apkpath + " has a bad signature!")
 356         return None
 357
 358     with zipfile.ZipFile(apkpath, 'r') as apk:
 359
 360         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 361
 362         if len(certs) < 1:
 363             logging.error("Found no signing certificates on %s" % apkpath)
 364             return None
 365         if len(certs) > 1:
 366             logging.error("Found multiple signing certificates on %s" % apkpath)
 367             return None
 368
 369         cert = apk.read(certs[0])
 370
 371     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 372     if content.getComponentByName('contentType') != rfc2315.signedData:
 373         logging.error("Unexpected format.")
 374         return None
 375
 376     content = decoder.decode(content.getComponentByName('content'),
 377                              asn1Spec=rfc2315.SignedData())[0]
 378     try:
 379         certificates = content.getComponentByName('certificates')
 380     except PyAsn1Error:
 381         logging.error("Certificates not found.")
 382         return None
 383
 384     cert_encoded = encoder.encode(certificates)[4:]
 385
 386     return md5(cert_encoded.encode('hex')).hexdigest()
 387
 388
 389 def scan_apks(apps, apkcache, repodir, knownapks):
 390     """Scan the apks in the given repo directory.
 391
 392     This also extracts the icons.
 393
 394     :param apps: list of all applications, as per metadata.read_metadata
 395     :param apkcache: current apk cache information
 396     :param repodir: repo directory to scan
 397     :param knownapks: known apks info
 398     :returns: (apks, cachechanged) where apks is a list of apk information,
 399               and cachechanged is True if the apkcache got changed.
 400     """
 401
 402     cachechanged = False
 403
 404     icon_dirs = get_icon_dirs(repodir)
 405     for icon_dir in icon_dirs:
 406         if os.path.exists(icon_dir):
 407             if options.clean:
 408                 shutil.rmtree(icon_dir)
 409                 os.makedirs(icon_dir)
 410         else:
 411             os.makedirs(icon_dir)
 412
 413     apks = []
 414     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 415     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 416     vername_pat = re.compile(".*versionName='([^']*)'.*")
 417     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 418     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 419     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 420     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 421     string_pat = re.compile(".*'([^']*)'.*")
 422     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 423
 424         apkfilename = apkfile[len(repodir) + 1:]
 425         if ' ' in apkfilename:
 426             logging.critical("Spaces in filenames are not allowed.")
 427             sys.exit(1)
 428
 429         # Calculate the sha256...
 430         sha = hashlib.sha256()
 431         with open(apkfile, 'rb') as f:
 432             while True:
 433                 t = f.read(16384)
 434                 if len(t) == 0:
 435                     break
 436                 sha.update(t)
 437             shasum = sha.hexdigest()
 438
 439         usecache = False
 440         if apkfilename in apkcache:
 441             thisinfo = apkcache[apkfilename]
 442             if thisinfo['sha256'] == shasum:
 443                 logging.debug("Reading " + apkfilename + " from cache")
 444                 usecache = True
 445             else:
 446                 logging.debug("Ignoring stale cache data for " + apkfilename)
 447
 448         if not usecache:
 449             logging.debug("Processing " + apkfilename)
 450             thisinfo = {}
 451             thisinfo['apkname'] = apkfilename
 452             thisinfo['sha256'] = shasum
 453             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 454             if os.path.exists(os.path.join(repodir, srcfilename)):
 455                 thisinfo['srcname'] = srcfilename
 456             thisinfo['size'] = os.path.getsize(apkfile)
 457             thisinfo['permissions'] = set()
 458             thisinfo['features'] = set()
 459             thisinfo['icons_src'] = {}
 460             thisinfo['icons'] = {}
 461             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 462             if p.returncode != 0:
 463                 if options.delete_unknown:
 464                     if os.path.exists(apkfile):
 465                         logging.error("Failed to get apk information, deleting " + apkfile)
 466                         os.remove(apkfile)
 467                     else:
 468                         logging.error("Could not find {0} to remove it".format(apkfile))
 469                 else:
 470                     logging.error("Failed to get apk information, skipping " + apkfile)
 471                 continue
 472             for line in p.output.splitlines():
 473                 if line.startswith("package:"):
 474                     try:
 475                         thisinfo['id'] = re.match(name_pat, line).group(1)
 476                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 477                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 478                     except Exception, e:
 479                         logging.error("Package matching failed: " + str(e))
 480                         logging.info("Line was: " + line)
 481                         sys.exit(1)
 482                 elif line.startswith("application:"):
 483                     thisinfo['name'] = re.match(label_pat, line).group(1)
 484                     # Keep path to non-dpi icon in case we need it
 485                     match = re.match(icon_pat_nodpi, line)
 486                     if match:
 487                         thisinfo['icons_src']['-1'] = match.group(1)
 488                 elif line.startswith("launchable-activity:"):
 489                     # Only use launchable-activity as fallback to application
 490                     if not thisinfo['name']:
 491                         thisinfo['name'] = re.match(label_pat, line).group(1)
 492                     if '-1' not in thisinfo['icons_src']:
 493                         match = re.match(icon_pat_nodpi, line)
 494                         if match:
 495                             thisinfo['icons_src']['-1'] = match.group(1)
 496                 elif line.startswith("application-icon-"):
 497                     match = re.match(icon_pat, line)
 498                     if match:
 499                         density = match.group(1)
 500                         path = match.group(2)
 501                         thisinfo['icons_src'][density] = path
 502                 elif line.startswith("sdkVersion:"):
 503                     m = re.match(sdkversion_pat, line)
 504                     if m is None:
 505                         logging.error(line.replace('sdkVersion:', '')
 506                                       + ' is not a valid minSdkVersion!')
 507                     else:
 508                         thisinfo['sdkversion'] = m.group(1)
 509                 elif line.startswith("maxSdkVersion:"):
 510                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 511                 elif line.startswith("native-code:"):
 512                     thisinfo['nativecode'] = []
 513                     for arch in line[13:].split(' '):
 514                         thisinfo['nativecode'].append(arch[1:-1])
 515                 elif line.startswith("uses-permission:"):
 516                     perm = re.match(string_pat, line).group(1)
 517                     if perm.startswith("android.permission."):
 518                         perm = perm[19:]
 519                     thisinfo['permissions'].add(perm)
 520                 elif line.startswith("uses-feature:"):
 521                     perm = re.match(string_pat, line).group(1)
 522                     # Filter out this, it's only added with the latest SDK tools and
 523                     # causes problems for lots of apps.
 524                     if perm != "android.hardware.screen.portrait" \
 525                             and perm != "android.hardware.screen.landscape":
 526                         if perm.startswith("android.feature."):
 527                             perm = perm[16:]
 528                         thisinfo['features'].add(perm)
 529
 530             if 'sdkversion' not in thisinfo:
 531                 logging.warn("No SDK version information found in {0}".format(apkfile))
 532                 thisinfo['sdkversion'] = 0
 533
 534             # Check for debuggable apks...
 535             if common.isApkDebuggable(apkfile, config):
 536                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 537
 538             # Get the signature (or md5 of, to be precise)...
 539             logging.debug('Getting signature of {0}'.format(apkfile))
 540             thisinfo['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 541             if not thisinfo['sig']:
 542                 logging.critical("Failed to get apk signature")
 543                 sys.exit(1)
 544
 545             apk = zipfile.ZipFile(apkfile, 'r')
 546
 547             # if an APK has files newer than the system time, suggest updating
 548             # the system clock.  This is useful for offline systems, used for
 549             # signing, which do not have another source of clock sync info. It
 550             # has to be more than 24 hours newer because ZIP/APK files do not
 551             # store timezone info
 552             info = apk.getinfo('AndroidManifest.xml')
 553             dt_obj = datetime(*info.date_time)
 554             checkdt = dt_obj - timedelta(1)
 555             if datetime.today() < checkdt:
 556                 logging.warn('System clock is older than manifest in: '
 557                              + apkfilename + '\nSet clock to that time using:\n'
 558                              + 'sudo date -s "' + str(dt_obj) + '"')
 559
 560             iconfilename = "%s.%s.png" % (
 561                 thisinfo['id'],
 562                 thisinfo['versioncode'])
 563
 564             # Extract the icon file...
 565             empty_densities = []
 566             for density in screen_densities:
 567                 if density not in thisinfo['icons_src']:
 568                     empty_densities.append(density)
 569                     continue
 570                 iconsrc = thisinfo['icons_src'][density]
 571                 icon_dir = get_icon_dir(repodir, density)
 572                 icondest = os.path.join(icon_dir, iconfilename)
 573
 574                 try:
 575                     with open(icondest, 'wb') as f:
 576                         f.write(apk.read(iconsrc))
 577                     thisinfo['icons'][density] = iconfilename
 578
 579                 except:
 580                     logging.warn("Error retrieving icon file")
 581                     del thisinfo['icons'][density]
 582                     del thisinfo['icons_src'][density]
 583                     empty_densities.append(density)
 584
 585             if '-1' in thisinfo['icons_src']:
 586                 iconsrc = thisinfo['icons_src']['-1']
 587                 iconpath = os.path.join(
 588                     get_icon_dir(repodir, None), iconfilename)
 589                 with open(iconpath, 'wb') as f:
 590                     f.write(apk.read(iconsrc))
 591                 try:
 592                     im = Image.open(iconpath)
 593                     dpi = px_to_dpi(im.size[0])
 594                     for density in densities:
 595                         if density in thisinfo['icons']:
 596                             break
 597                         if density == densities[-1] or dpi >= int(density):
 598                             thisinfo['icons'][density] = iconfilename
 599                             shutil.move(iconpath,
 600                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 601                             empty_densities.remove(density)
 602                             break
 603                 except Exception, e:
 604                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 605
 606             if thisinfo['icons']:
 607                 thisinfo['icon'] = iconfilename
 608
 609             apk.close()
 610
 611             # First try resizing down to not lose quality
 612             last_density = None
 613             for density in densities:
 614                 if density not in empty_densities:
 615                     last_density = density
 616                     continue
 617                 if last_density is None:
 618                     continue
 619                 logging.debug("Density %s not available, resizing down from %s"
 620                               % (density, last_density))
 621
 622                 last_iconpath = os.path.join(
 623                     get_icon_dir(repodir, last_density), iconfilename)
 624                 iconpath = os.path.join(
 625                     get_icon_dir(repodir, density), iconfilename)
 626                 try:
 627                     im = Image.open(last_iconpath)
 628                 except:
 629                     logging.warn("Invalid image file at %s" % last_iconpath)
 630                     continue
 631
 632                 size = dpi_to_px(density)
 633
 634                 im.thumbnail((size, size), Image.ANTIALIAS)
 635                 im.save(iconpath, "PNG")
 636                 empty_densities.remove(density)
 637
 638             # Then just copy from the highest resolution available
 639             last_density = None
 640             for density in reversed(densities):
 641                 if density not in empty_densities:
 642                     last_density = density
 643                     continue
 644                 if last_density is None:
 645                     continue
 646                 logging.debug("Density %s not available, copying from lower density %s"
 647                               % (density, last_density))
 648
 649                 shutil.copyfile(
 650                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 651                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 652
 653                 empty_densities.remove(density)
 654
 655             for density in densities:
 656                 icon_dir = get_icon_dir(repodir, density)
 657                 icondest = os.path.join(icon_dir, iconfilename)
 658                 resize_icon(icondest, density)
 659
 660             # Copy from icons-mdpi to icons since mdpi is the baseline density
 661             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 662             if os.path.isfile(baseline):
 663                 shutil.copyfile(baseline,
 664                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 665
 666             # Record in known apks, getting the added date at the same time..
 667             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 668             if added:
 669                 thisinfo['added'] = added
 670
 671             apkcache[apkfilename] = thisinfo
 672             cachechanged = True
 673
 674         apks.append(thisinfo)
 675
 676     return apks, cachechanged
 677
 678
 679 repo_pubkey_fingerprint = None
 680
 681
 682 # Generate a certificate fingerprint the same way keytool does it
 683 # (but with slightly different formatting)
 684 def cert_fingerprint(data):
 685     digest = hashlib.sha256(data).digest()
 686     ret = []
 687     ret.append(' '.join("%02X" % ord(b) for b in digest))
 688     return " ".join(ret)
 689
 690
 691 def extract_pubkey():
 692     global repo_pubkey_fingerprint
 693     if 'repo_pubkey' in config:
 694         pubkey = unhexlify(config['repo_pubkey'])
 695     else:
 696         p = FDroidPopen(['keytool', '-exportcert',
 697                          '-alias', config['repo_keyalias'],
 698                          '-keystore', config['keystore'],
 699                          '-storepass:file', config['keystorepassfile']]
 700                         + config['smartcardoptions'], output=False)
 701         if p.returncode != 0 or len(p.output) < 20:
 702             msg = "Failed to get repo pubkey!"
 703             if config['keystore'] == 'NONE':
 704                 msg += ' Is your crypto smartcard plugged in?'
 705             logging.critical(msg)
 706             sys.exit(1)
 707         pubkey = p.output
 708     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 709     return hexlify(pubkey)
 710
 711
 712 def make_index(apps, sortedids, apks, repodir, archive, categories):
 713     """Make a repo index.
 714
 715     :param apps: fully populated apps list
 716     :param apks: full populated apks list
 717     :param repodir: the repo directory
 718     :param archive: True if this is the archive repo, False if it's the
 719                     main one.
 720     :param categories: list of categories
 721     """
 722
 723     doc = Document()
 724
 725     def addElement(name, value, doc, parent):
 726         el = doc.createElement(name)
 727         el.appendChild(doc.createTextNode(value))
 728         parent.appendChild(el)
 729
 730     def addElementNonEmpty(name, value, doc, parent):
 731         if not value:
 732             return
 733         addElement(name, value, doc, parent)
 734
 735     def addElementCDATA(name, value, doc, parent):
 736         el = doc.createElement(name)
 737         el.appendChild(doc.createCDATASection(value))
 738         parent.appendChild(el)
 739
 740     root = doc.createElement("fdroid")
 741     doc.appendChild(root)
 742
 743     repoel = doc.createElement("repo")
 744
 745     if archive:
 746         repoel.setAttribute("name", config['archive_name'])
 747         if config['repo_maxage'] != 0:
 748             repoel.setAttribute("maxage", str(config['repo_maxage']))
 749         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 750         repoel.setAttribute("url", config['archive_url'])
 751         addElement('description', config['archive_description'], doc, repoel)
 752
 753     else:
 754         repoel.setAttribute("name", config['repo_name'])
 755         if config['repo_maxage'] != 0:
 756             repoel.setAttribute("maxage", str(config['repo_maxage']))
 757         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 758         repoel.setAttribute("url", config['repo_url'])
 759         addElement('description', config['repo_description'], doc, repoel)
 760
 761     repoel.setAttribute("version", "14")
 762     repoel.setAttribute("timestamp", str(int(time.time())))
 763
 764     nosigningkey = False
 765     if not options.nosign:
 766         if 'repo_keyalias' not in config:
 767             nosigningkey = True
 768             logging.critical("'repo_keyalias' not found in config.py!")
 769         if 'keystore' not in config:
 770             nosigningkey = True
 771             logging.critical("'keystore' not found in config.py!")
 772         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 773             nosigningkey = True
 774             logging.critical("'keystorepass' not found in config.py!")
 775         if 'keypass' not in config and 'keypassfile' not in config:
 776             nosigningkey = True
 777             logging.critical("'keypass' not found in config.py!")
 778         if not os.path.exists(config['keystore']):
 779             nosigningkey = True
 780             logging.critical("'" + config['keystore'] + "' does not exist!")
 781         if nosigningkey:
 782             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 783             logging.warning("\tfdroid update --create-key")
 784             sys.exit(1)
 785
 786     repoel.setAttribute("pubkey", extract_pubkey())
 787     root.appendChild(repoel)
 788
 789     for appid in sortedids:
 790         app = apps[appid]
 791
 792         if app['Disabled'] is not None:
 793             continue
 794
 795         # Get a list of the apks for this app...
 796         apklist = []
 797         for apk in apks:
 798             if apk['id'] == appid:
 799                 apklist.append(apk)
 800
 801         if len(apklist) == 0:
 802             continue
 803
 804         apel = doc.createElement("application")
 805         apel.setAttribute("id", app['id'])
 806         root.appendChild(apel)
 807
 808         addElement('id', app['id'], doc, apel)
 809         if 'added' in app:
 810             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 811         if 'lastupdated' in app:
 812             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 813         addElement('name', app['Name'], doc, apel)
 814         addElement('summary', app['Summary'], doc, apel)
 815         if app['icon']:
 816             addElement('icon', app['icon'], doc, apel)
 817
 818         def linkres(appid):
 819             if appid in apps:
 820                 return ("fdroid.app:" + appid, apps[appid]['Name'])
 821             raise MetaDataException("Cannot resolve app id " + appid)
 822
 823         addElement('desc',
 824                    metadata.description_html(app['Description'], linkres),
 825                    doc, apel)
 826         addElement('license', app['License'], doc, apel)
 827         if 'Categories' in app and app['Categories']:
 828             addElement('categories', ','.join(app["Categories"]), doc, apel)
 829             # We put the first (primary) category in LAST, which will have
 830             # the desired effect of making clients that only understand one
 831             # category see that one.
 832             addElement('category', app["Categories"][0], doc, apel)
 833         addElement('web', app['Web Site'], doc, apel)
 834         addElement('source', app['Source Code'], doc, apel)
 835         addElement('tracker', app['Issue Tracker'], doc, apel)
 836         addElementNonEmpty('changelog', app['Changelog'], doc, apel)
 837         addElementNonEmpty('donate', app['Donate'], doc, apel)
 838         addElementNonEmpty('bitcoin', app['Bitcoin'], doc, apel)
 839         addElementNonEmpty('litecoin', app['Litecoin'], doc, apel)
 840         addElementNonEmpty('flattr', app['FlattrID'], doc, apel)
 841
 842         # These elements actually refer to the current version (i.e. which
 843         # one is recommended. They are historically mis-named, and need
 844         # changing, but stay like this for now to support existing clients.
 845         addElement('marketversion', app['Current Version'], doc, apel)
 846         addElement('marketvercode', app['Current Version Code'], doc, apel)
 847
 848         if app['AntiFeatures']:
 849             af = app['AntiFeatures']
 850             if af:
 851                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 852         if app['Provides']:
 853             pv = app['Provides'].split(',')
 854             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 855         if app['Requires Root']:
 856             addElement('requirements', 'root', doc, apel)
 857
 858         # Sort the apk list into version order, just so the web site
 859         # doesn't have to do any work by default...
 860         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 861
 862         # Check for duplicates - they will make the client unhappy...
 863         for i in range(len(apklist) - 1):
 864             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 865                 logging.critical("duplicate versions: '%s' - '%s'" % (
 866                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 867                 sys.exit(1)
 868
 869         current_version_code = 0
 870         current_version_file = None
 871         for apk in apklist:
 872             # find the APK for the "Current Version"
 873             if current_version_code < apk['versioncode']:
 874                 current_version_code = apk['versioncode']
 875             if current_version_code < int(app['Current Version Code']):
 876                 current_version_file = apk['apkname']
 877
 878             apkel = doc.createElement("package")
 879             apel.appendChild(apkel)
 880             addElement('version', apk['version'], doc, apkel)
 881             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 882             addElement('apkname', apk['apkname'], doc, apkel)
 883             if 'srcname' in apk:
 884                 addElement('srcname', apk['srcname'], doc, apkel)
 885             for hash_type in ['sha256']:
 886                 if hash_type not in apk:
 887                     continue
 888                 hashel = doc.createElement("hash")
 889                 hashel.setAttribute("type", hash_type)
 890                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 891                 apkel.appendChild(hashel)
 892             addElement('sig', apk['sig'], doc, apkel)
 893             addElement('size', str(apk['size']), doc, apkel)
 894             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 895             if 'maxsdkversion' in apk:
 896                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 897             if 'added' in apk:
 898                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 899             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
 900             if 'nativecode' in apk:
 901                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 902             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
 903
 904         if current_version_file is not None \
 905                 and config['make_current_version_link'] \
 906                 and repodir == 'repo':  # only create these
 907             sanitized_name = re.sub('''[ '"&%?+=/]''', '',
 908                                     app[config['current_version_name_source']])
 909             apklinkname = sanitized_name + '.apk'
 910             current_version_path = os.path.join(repodir, current_version_file)
 911             if os.path.islink(apklinkname):
 912                 os.remove(apklinkname)
 913             os.symlink(current_version_path, apklinkname)
 914             # also symlink gpg signature, if it exists
 915             for extension in ('.asc', '.sig'):
 916                 sigfile_path = current_version_path + extension
 917                 if os.path.exists(sigfile_path):
 918                     siglinkname = apklinkname + extension
 919                     if os.path.islink(siglinkname):
 920                         os.remove(siglinkname)
 921                     os.symlink(sigfile_path, siglinkname)
 922
 923     if options.pretty:
 924         output = doc.toprettyxml()
 925     else:
 926         output = doc.toxml()
 927
 928     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 929         f.write(output)
 930
 931     if 'repo_keyalias' in config:
 932
 933         if options.nosign:
 934             logging.info("Creating unsigned index in preparation for signing")
 935         else:
 936             logging.info("Creating signed index with this key (SHA256):")
 937             logging.info("%s" % repo_pubkey_fingerprint)
 938
 939         # Create a jar of the index...
 940         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
 941         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 942         if p.returncode != 0:
 943             logging.critical("Failed to create {0}".format(jar_output))
 944             sys.exit(1)
 945
 946         # Sign the index...
 947         signed = os.path.join(repodir, 'index.jar')
 948         if options.nosign:
 949             # Remove old signed index if not signing
 950             if os.path.exists(signed):
 951                 os.remove(signed)
 952         else:
 953             args = ['jarsigner', '-keystore', config['keystore'],
 954                     '-storepass:file', config['keystorepassfile'],
 955                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
 956                     signed, config['repo_keyalias']]
 957             if config['keystore'] == 'NONE':
 958                 args += config['smartcardoptions']
 959             else:  # smardcards never use -keypass
 960                 args += ['-keypass:file', config['keypassfile']]
 961             p = FDroidPopen(args)
 962             if p.returncode != 0:
 963                 logging.critical("Failed to sign index")
 964                 sys.exit(1)
 965
 966     # Copy the repo icon into the repo directory...
 967     icon_dir = os.path.join(repodir, 'icons')
 968     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 969     shutil.copyfile(config['repo_icon'], iconfilename)
 970
 971     # Write a category list in the repo to allow quick access...
 972     catdata = ''
 973     for cat in categories:
 974         catdata += cat + '\n'
 975     with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
 976         f.write(catdata)
 977
 978
 979 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 980
 981     for appid, app in apps.iteritems():
 982
 983         if app['Archive Policy']:
 984             keepversions = int(app['Archive Policy'][:-9])
 985         else:
 986             keepversions = defaultkeepversions
 987
 988         def filter_apk_list_sorted(apk_list):
 989             res = []
 990             for apk in apk_list:
 991                 if apk['id'] == appid:
 992                     res.append(apk)
 993
 994             # Sort the apk list by version code. First is highest/newest.
 995             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
 996
 997         def move_file(from_dir, to_dir, filename, ignore_missing):
 998             from_path = os.path.join(from_dir, filename)
 999             if ignore_missing and not os.path.exists(from_path):
1000                 return
1001             to_path = os.path.join(to_dir, filename)
1002             shutil.move(from_path, to_path)
1003
1004         if len(apks) > keepversions:
1005             apklist = filter_apk_list_sorted(apks)
1006             # Move back the ones we don't want.
1007             for apk in apklist[keepversions:]:
1008                 logging.info("Moving " + apk['apkname'] + " to archive")
1009                 move_file(repodir, archivedir, apk['apkname'], False)
1010                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1011                 if 'srcname' in apk:
1012                     move_file(repodir, archivedir, apk['srcname'], False)
1013                 archapks.append(apk)
1014                 apks.remove(apk)
1015         elif len(apks) < keepversions and len(archapks) > 0:
1016             required = keepversions - len(apks)
1017             archapklist = filter_apk_list_sorted(archapks)
1018             # Move forward the ones we want again.
1019             for apk in archapklist[:required]:
1020                 logging.info("Moving " + apk['apkname'] + " from archive")
1021                 move_file(archivedir, repodir, apk['apkname'], False)
1022                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1023                 if 'srcname' in apk:
1024                     move_file(archivedir, repodir, apk['srcname'], False)
1025                 archapks.remove(apk)
1026                 apks.append(apk)
1027
1028
1029 def add_apks_to_per_app_repos(repodir, apks):
1030     apks_per_app = dict()
1031     for apk in apks:
1032         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1033         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1034         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1035         apks_per_app[apk['id']] = apk
1036
1037         if not os.path.exists(apk['per_app_icons']):
1038             logging.info('Adding new repo for only ' + apk['id'])
1039             os.makedirs(apk['per_app_icons'])
1040
1041         apkpath = os.path.join(repodir, apk['apkname'])
1042         shutil.copy(apkpath, apk['per_app_repo'])
1043         apksigpath = apkpath + '.sig'
1044         if os.path.exists(apksigpath):
1045             shutil.copy(apksigpath, apk['per_app_repo'])
1046         apkascpath = apkpath + '.asc'
1047         if os.path.exists(apkascpath):
1048             shutil.copy(apkascpath, apk['per_app_repo'])
1049
1050
1051 config = None
1052 options = None
1053
1054
1055 def main():
1056
1057     global config, options
1058
1059     # Parse command line...
1060     parser = ArgumentParser()
1061     common.setup_global_opts(parser)
1062     parser.add_argument("--create-key", action="store_true", default=False,
1063                         help="Create a repo signing key in a keystore")
1064     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1065                         help="Create skeleton metadata files that are missing")
1066     parser.add_argument("--delete-unknown", action="store_true", default=False,
1067                         help="Delete APKs without metadata from the repo")
1068     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1069                         help="Report on build data status")
1070     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1071                         help="Interactively ask about things that need updating.")
1072     parser.add_argument("-I", "--icons", action="store_true", default=False,
1073                         help="Resize all the icons exceeding the max pixel size and exit")
1074     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1075                         help="Specify editor to use in interactive mode. Default " +
1076                         "is /etc/alternatives/editor")
1077     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1078                         help="Update the wiki")
1079     parser.add_argument("--pretty", action="store_true", default=False,
1080                         help="Produce human-readable index.xml")
1081     parser.add_argument("--clean", action="store_true", default=False,
1082                         help="Clean update - don't uses caches, reprocess all apks")
1083     parser.add_argument("--nosign", action="store_true", default=False,
1084                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1085     options = parser.parse_args()
1086
1087     config = common.read_config(options)
1088
1089     repodirs = ['repo']
1090     if config['archive_older'] != 0:
1091         repodirs.append('archive')
1092         if not os.path.exists('archive'):
1093             os.mkdir('archive')
1094
1095     if options.icons:
1096         resize_all_icons(repodirs)
1097         sys.exit(0)
1098
1099     # check that icons exist now, rather than fail at the end of `fdroid update`
1100     for k in ['repo_icon', 'archive_icon']:
1101         if k in config:
1102             if not os.path.exists(config[k]):
1103                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1104                 sys.exit(1)
1105
1106     # if the user asks to create a keystore, do it now, reusing whatever it can
1107     if options.create_key:
1108         if os.path.exists(config['keystore']):
1109             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1110             logging.critical("\t'" + config['keystore'] + "'")
1111             sys.exit(1)
1112
1113         if 'repo_keyalias' not in config:
1114             config['repo_keyalias'] = socket.getfqdn()
1115             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1116         if 'keydname' not in config:
1117             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1118             common.write_to_config(config, 'keydname', config['keydname'])
1119         if 'keystore' not in config:
1120             config['keystore'] = common.default_config.keystore
1121             common.write_to_config(config, 'keystore', config['keystore'])
1122
1123         password = common.genpassword()
1124         if 'keystorepass' not in config:
1125             config['keystorepass'] = password
1126             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1127         if 'keypass' not in config:
1128             config['keypass'] = password
1129             common.write_to_config(config, 'keypass', config['keypass'])
1130         common.genkeystore(config)
1131
1132     # Get all apps...
1133     apps = metadata.read_metadata()
1134
1135     # Generate a list of categories...
1136     categories = set()
1137     for app in apps.itervalues():
1138         categories.update(app['Categories'])
1139
1140     # Read known apks data (will be updated and written back when we've finished)
1141     knownapks = common.KnownApks()
1142
1143     # Gather information about all the apk files in the repo directory, using
1144     # cached data if possible.
1145     apkcachefile = os.path.join('tmp', 'apkcache')
1146     if not options.clean and os.path.exists(apkcachefile):
1147         with open(apkcachefile, 'rb') as cf:
1148             apkcache = pickle.load(cf)
1149     else:
1150         apkcache = {}
1151
1152     delete_disabled_builds(apps, apkcache, repodirs)
1153
1154     # Scan all apks in the main repo
1155     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks)
1156
1157     # Generate warnings for apk's with no metadata (or create skeleton
1158     # metadata files, if requested on the command line)
1159     newmetadata = False
1160     for apk in apks:
1161         if apk['id'] not in apps:
1162             if options.create_metadata:
1163                 if 'name' not in apk:
1164                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1165                     continue
1166                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1167                 f.write("License:Unknown\n")
1168                 f.write("Web Site:\n")
1169                 f.write("Source Code:\n")
1170                 f.write("Issue Tracker:\n")
1171                 f.write("Changelog:\n")
1172                 f.write("Summary:" + apk['name'] + "\n")
1173                 f.write("Description:\n")
1174                 f.write(apk['name'] + "\n")
1175                 f.write(".\n")
1176                 f.close()
1177                 logging.info("Generated skeleton metadata for " + apk['id'])
1178                 newmetadata = True
1179             else:
1180                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1181                 if options.delete_unknown:
1182                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1183                     rmf = os.path.join(repodirs[0], apk['apkname'])
1184                     if not os.path.exists(rmf):
1185                         logging.error("Could not find {0} to remove it".format(rmf))
1186                     else:
1187                         os.remove(rmf)
1188                 else:
1189                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1190
1191     # update the metadata with the newly created ones included
1192     if newmetadata:
1193         apps = metadata.read_metadata()
1194
1195     # Scan the archive repo for apks as well
1196     if len(repodirs) > 1:
1197         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1198         if cc:
1199             cachechanged = True
1200     else:
1201         archapks = []
1202
1203     # Some information from the apks needs to be applied up to the application
1204     # level. When doing this, we use the info from the most recent version's apk.
1205     # We deal with figuring out when the app was added and last updated at the
1206     # same time.
1207     for appid, app in apps.iteritems():
1208         bestver = 0
1209         added = None
1210         lastupdated = None
1211         for apk in apks + archapks:
1212             if apk['id'] == appid:
1213                 if apk['versioncode'] > bestver:
1214                     bestver = apk['versioncode']
1215                     bestapk = apk
1216
1217                 if 'added' in apk:
1218                     if not added or apk['added'] < added:
1219                         added = apk['added']
1220                     if not lastupdated or apk['added'] > lastupdated:
1221                         lastupdated = apk['added']
1222
1223         if added:
1224             app['added'] = added
1225         else:
1226             logging.warn("Don't know when " + appid + " was added")
1227         if lastupdated:
1228             app['lastupdated'] = lastupdated
1229         else:
1230             logging.warn("Don't know when " + appid + " was last updated")
1231
1232         if bestver == 0:
1233             if app['Name'] is None:
1234                 app['Name'] = app['Auto Name'] or appid
1235             app['icon'] = None
1236             logging.warn("Application " + appid + " has no packages")
1237         else:
1238             if app['Name'] is None:
1239                 app['Name'] = bestapk['name']
1240             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1241
1242     # Sort the app list by name, then the web site doesn't have to by default.
1243     # (we had to wait until we'd scanned the apks to do this, because mostly the
1244     # name comes from there!)
1245     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid]['Name'].upper())
1246
1247     # APKs are placed into multiple repos based on the app package, providing
1248     # per-app subscription feeds for nightly builds and things like it
1249     if config['per_app_repos']:
1250         add_apks_to_per_app_repos(repodirs[0], apks)
1251         for appid, app in apps.iteritems():
1252             repodir = os.path.join(appid, 'fdroid', 'repo')
1253             appdict = dict()
1254             appdict[appid] = app
1255             if os.path.isdir(repodir):
1256                 make_index(appdict, [appid], apks, repodir, False, categories)
1257             else:
1258                 logging.info('Skipping index generation for ' + appid)
1259         return
1260
1261     if len(repodirs) > 1:
1262         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1263
1264     # Make the index for the main repo...
1265     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1266
1267     # If there's an archive repo,  make the index for it. We already scanned it
1268     # earlier on.
1269     if len(repodirs) > 1:
1270         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1271
1272     if config['update_stats']:
1273
1274         # Update known apks info...
1275         knownapks.writeifchanged()
1276
1277         # Generate latest apps data for widget
1278         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1279             data = ''
1280             for line in file(os.path.join('stats', 'latestapps.txt')):
1281                 appid = line.rstrip()
1282                 data += appid + "\t"
1283                 app = apps[appid]
1284                 data += app['Name'] + "\t"
1285                 if app['icon'] is not None:
1286                     data += app['icon'] + "\t"
1287                 data += app['License'] + "\n"
1288             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
1289                 f.write(data)
1290
1291     if cachechanged:
1292         with open(apkcachefile, 'wb') as cf:
1293             pickle.dump(apkcache, cf)
1294
1295     # Update the wiki...
1296     if options.wiki:
1297         update_wiki(apps, sortedids, apks + archapks)
1298
1299     logging.info("Finished.")
1300
1301 if __name__ == "__main__":
1302     main()