fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import socket
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from datetime import datetime, timedelta
  31 from xml.dom.minidom import Document
  32 from argparse import ArgumentParser
  33 import time
  34 from pyasn1.error import PyAsn1Error
  35 from pyasn1.codec.der import decoder, encoder
  36 from pyasn1_modules import rfc2315
  37 from hashlib import md5
  38 from binascii import hexlify, unhexlify
  39
  40 from PIL import Image
  41 import logging
  42
  43 import common
  44 import metadata
  45 from common import FDroidPopen, SdkToolsPopen
  46 from metadata import MetaDataException
  47
  48
  49 def get_densities():
  50     return ['640', '480', '320', '240', '160', '120']
  51
  52
  53 def dpi_to_px(density):
  54     return (int(density) * 48) / 160
  55
  56
  57 def px_to_dpi(px):
  58     return (int(px) * 160) / 48
  59
  60
  61 def get_icon_dir(repodir, density):
  62     if density is None:
  63         return os.path.join(repodir, "icons")
  64     return os.path.join(repodir, "icons-%s" % density)
  65
  66
  67 def get_icon_dirs(repodir):
  68     for density in get_densities():
  69         yield get_icon_dir(repodir, density)
  70     yield os.path.join(repodir, "icons")
  71
  72
  73 def update_wiki(apps, sortedids, apks):
  74     """Update the wiki
  75
  76     :param apps: fully populated list of all applications
  77     :param apks: all apks, except...
  78     """
  79     logging.info("Updating wiki")
  80     wikicat = 'Apps'
  81     wikiredircat = 'App Redirects'
  82     import mwclient
  83     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  84                          path=config['wiki_path'])
  85     site.login(config['wiki_user'], config['wiki_password'])
  86     generated_pages = {}
  87     generated_redirects = {}
  88
  89     for appid in sortedids:
  90         app = apps[appid]
  91
  92         wikidata = ''
  93         if app['Disabled']:
  94             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  95         if 'AntiFeatures' in app:
  96             for af in app['AntiFeatures']:
  97                 wikidata += '{{AntiFeature|' + af + '}}\n'
  98         if app['Requires Root']:
  99             requiresroot = 'Yes'
 100         else:
 101             requiresroot = 'No'
 102         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s}}\n' % (
 103             appid,
 104             app['Name'],
 105             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
 106             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
 107             app['Source Code'],
 108             app['Issue Tracker'],
 109             app['Web Site'],
 110             app['Changelog'],
 111             app['Donate'],
 112             app['FlattrID'],
 113             app['Bitcoin'],
 114             app['Litecoin'],
 115             app['License'],
 116             requiresroot)
 117
 118         if app['Provides']:
 119             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 120
 121         wikidata += app['Summary']
 122         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 123
 124         wikidata += "=Description=\n"
 125         wikidata += metadata.description_wiki(app['Description']) + "\n"
 126
 127         wikidata += "=Maintainer Notes=\n"
 128         if 'Maintainer Notes' in app:
 129             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 130         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 131
 132         # Get a list of all packages for this application...
 133         apklist = []
 134         gotcurrentver = False
 135         cantupdate = False
 136         buildfails = False
 137         for apk in apks:
 138             if apk['id'] == appid:
 139                 if str(apk['versioncode']) == app['Current Version Code']:
 140                     gotcurrentver = True
 141                 apklist.append(apk)
 142         # Include ones we can't build, as a special case...
 143         for thisbuild in app['builds']:
 144             if thisbuild['disable']:
 145                 if thisbuild['vercode'] == app['Current Version Code']:
 146                     cantupdate = True
 147                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 148                 apklist.append({'versioncode': int(thisbuild['vercode']),
 149                                 'version': thisbuild['version'],
 150                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(thisbuild['disable']),
 151                                 })
 152             else:
 153                 builtit = False
 154                 for apk in apklist:
 155                     if apk['versioncode'] == int(thisbuild['vercode']):
 156                         builtit = True
 157                         break
 158                 if not builtit:
 159                     buildfails = True
 160                     apklist.append({'versioncode': int(thisbuild['vercode']),
 161                                     'version': thisbuild['version'],
 162                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, thisbuild['vercode']),
 163                                     })
 164         if app['Current Version Code'] == '0':
 165             cantupdate = True
 166         # Sort with most recent first...
 167         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 168
 169         wikidata += "=Versions=\n"
 170         if len(apklist) == 0:
 171             wikidata += "We currently have no versions of this app available."
 172         elif not gotcurrentver:
 173             wikidata += "We don't have the current version of this app."
 174         else:
 175             wikidata += "We have the current version of this app."
 176         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 177         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 178         if len(app['No Source Since']) > 0:
 179             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 180         if len(app['Current Version']) > 0:
 181             wikidata += "The current (recommended) version is " + app['Current Version']
 182             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 183         validapks = 0
 184         for apk in apklist:
 185             wikidata += "==" + apk['version'] + "==\n"
 186
 187             if 'buildproblem' in apk:
 188                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 189             else:
 190                 validapks += 1
 191                 wikidata += "This version is built and signed by "
 192                 if 'srcname' in apk:
 193                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 194                 else:
 195                     wikidata += "the original developer.\n\n"
 196             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 197
 198         wikidata += '\n[[Category:' + wikicat + ']]\n'
 199         if len(app['No Source Since']) > 0:
 200             wikidata += '\n[[Category:Apps missing source code]]\n'
 201         if validapks == 0 and not app['Disabled']:
 202             wikidata += '\n[[Category:Apps with no packages]]\n'
 203         if cantupdate and not app['Disabled']:
 204             wikidata += "\n[[Category:Apps we can't update]]\n"
 205         if buildfails and not app['Disabled']:
 206             wikidata += "\n[[Category:Apps with failing builds]]\n"
 207         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 208             wikidata += '\n[[Category:Apps to Update]]\n'
 209         if app['Disabled']:
 210             wikidata += '\n[[Category:Apps that are disabled]]\n'
 211         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 212             wikidata += '\n[[Category:Apps with no update check]]\n'
 213         for appcat in app['Categories']:
 214             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 215
 216         # We can't have underscores in the page name, even if they're in
 217         # the package ID, because MediaWiki messes with them...
 218         pagename = appid.replace('_', ' ')
 219
 220         # Drop a trailing newline, because mediawiki is going to drop it anyway
 221         # and it we don't we'll think the page has changed when it hasn't...
 222         if wikidata.endswith('\n'):
 223             wikidata = wikidata[:-1]
 224
 225         generated_pages[pagename] = wikidata
 226
 227         # Make a redirect from the name to the ID too, unless there's
 228         # already an existing page with the name and it isn't a redirect.
 229         noclobber = False
 230         apppagename = app['Name'].replace('_', ' ')
 231         apppagename = apppagename.replace('{', '')
 232         apppagename = apppagename.replace('}', ' ')
 233         apppagename = apppagename.replace(':', ' ')
 234         # Drop double spaces caused mostly by replacing ':' above
 235         apppagename = apppagename.replace('  ', ' ')
 236         for expagename in site.allpages(prefix=apppagename,
 237                                         filterredir='nonredirects',
 238                                         generator=False):
 239             if expagename == apppagename:
 240                 noclobber = True
 241         # Another reason not to make the redirect page is if the app name
 242         # is the same as it's ID, because that will overwrite the real page
 243         # with an redirect to itself! (Although it seems like an odd
 244         # scenario this happens a lot, e.g. where there is metadata but no
 245         # builds or binaries to extract a name from.
 246         if apppagename == pagename:
 247             noclobber = True
 248         if not noclobber:
 249             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 250
 251     for tcat, genp in [(wikicat, generated_pages),
 252                        (wikiredircat, generated_redirects)]:
 253         catpages = site.Pages['Category:' + tcat]
 254         existingpages = []
 255         for page in catpages:
 256             existingpages.append(page.name)
 257             if page.name in genp:
 258                 pagetxt = page.edit()
 259                 if pagetxt != genp[page.name]:
 260                     logging.debug("Updating modified page " + page.name)
 261                     page.save(genp[page.name], summary='Auto-updated')
 262                 else:
 263                     logging.debug("Page " + page.name + " is unchanged")
 264             else:
 265                 logging.warn("Deleting page " + page.name)
 266                 page.delete('No longer published')
 267         for pagename, text in genp.items():
 268             logging.debug("Checking " + pagename)
 269             if pagename not in existingpages:
 270                 logging.debug("Creating page " + pagename)
 271                 try:
 272                     newpage = site.Pages[pagename]
 273                     newpage.save(text, summary='Auto-created')
 274                 except:
 275                     logging.error("...FAILED to create page '{0}'".format(pagename))
 276
 277     # Purge server cache to ensure counts are up to date
 278     site.pages['Repository Maintenance'].purge()
 279
 280
 281 def delete_disabled_builds(apps, apkcache, repodirs):
 282     """Delete disabled build outputs.
 283
 284     :param apps: list of all applications, as per metadata.read_metadata
 285     :param apkcache: current apk cache information
 286     :param repodirs: the repo directories to process
 287     """
 288     for appid, app in apps.iteritems():
 289         for build in app['builds']:
 290             if build['disable']:
 291                 apkfilename = appid + '_' + str(build['vercode']) + '.apk'
 292                 for repodir in repodirs:
 293                     apkpath = os.path.join(repodir, apkfilename)
 294                     ascpath = apkpath + ".asc"
 295                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 296                     for name in [apkpath, srcpath, ascpath]:
 297                         if os.path.exists(name):
 298                             logging.warn("Deleting disabled build output " + apkfilename)
 299                             os.remove(name)
 300                 if apkfilename in apkcache:
 301                     del apkcache[apkfilename]
 302
 303
 304 def resize_icon(iconpath, density):
 305
 306     if not os.path.isfile(iconpath):
 307         return
 308
 309     try:
 310         im = Image.open(iconpath)
 311         size = dpi_to_px(density)
 312
 313         if any(length > size for length in im.size):
 314             oldsize = im.size
 315             im.thumbnail((size, size), Image.ANTIALIAS)
 316             logging.debug("%s was too large at %s - new size is %s" % (
 317                 iconpath, oldsize, im.size))
 318             im.save(iconpath, "PNG")
 319
 320     except Exception, e:
 321         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 322
 323
 324 def resize_all_icons(repodirs):
 325     """Resize all icons that exceed the max size
 326
 327     :param repodirs: the repo directories to process
 328     """
 329     for repodir in repodirs:
 330         for density in get_densities():
 331             icon_dir = get_icon_dir(repodir, density)
 332             icon_glob = os.path.join(icon_dir, '*.png')
 333             for iconpath in glob.glob(icon_glob):
 334                 resize_icon(iconpath, density)
 335
 336
 337 # A signature block file with a .DSA, .RSA, or .EC extension
 338 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 339
 340
 341 def getsig(apkpath):
 342     """ Get the signing certificate of an apk. To get the same md5 has that
 343     Android gets, we encode the .RSA certificate in a specific format and pass
 344     it hex-encoded to the md5 digest algorithm.
 345
 346     :param apkpath: path to the apk
 347     :returns: A string containing the md5 of the signature of the apk or None
 348               if an error occurred.
 349     """
 350
 351     cert = None
 352
 353     # verify the jar signature is correct
 354     args = ['jarsigner', '-verify', apkpath]
 355     p = FDroidPopen(args)
 356     if p.returncode != 0:
 357         logging.critical(apkpath + " has a bad signature!")
 358         return None
 359
 360     with zipfile.ZipFile(apkpath, 'r') as apk:
 361
 362         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 363
 364         if len(certs) < 1:
 365             logging.error("Found no signing certificates on %s" % apkpath)
 366             return None
 367         if len(certs) > 1:
 368             logging.error("Found multiple signing certificates on %s" % apkpath)
 369             return None
 370
 371         cert = apk.read(certs[0])
 372
 373     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 374     if content.getComponentByName('contentType') != rfc2315.signedData:
 375         logging.error("Unexpected format.")
 376         return None
 377
 378     content = decoder.decode(content.getComponentByName('content'),
 379                              asn1Spec=rfc2315.SignedData())[0]
 380     try:
 381         certificates = content.getComponentByName('certificates')
 382     except PyAsn1Error:
 383         logging.error("Certificates not found.")
 384         return None
 385
 386     cert_encoded = encoder.encode(certificates)[4:]
 387
 388     return md5(cert_encoded.encode('hex')).hexdigest()
 389
 390
 391 def scan_apks(apps, apkcache, repodir, knownapks):
 392     """Scan the apks in the given repo directory.
 393
 394     This also extracts the icons.
 395
 396     :param apps: list of all applications, as per metadata.read_metadata
 397     :param apkcache: current apk cache information
 398     :param repodir: repo directory to scan
 399     :param knownapks: known apks info
 400     :returns: (apks, cachechanged) where apks is a list of apk information,
 401               and cachechanged is True if the apkcache got changed.
 402     """
 403
 404     cachechanged = False
 405
 406     icon_dirs = get_icon_dirs(repodir)
 407     for icon_dir in icon_dirs:
 408         if os.path.exists(icon_dir):
 409             if options.clean:
 410                 shutil.rmtree(icon_dir)
 411                 os.makedirs(icon_dir)
 412         else:
 413             os.makedirs(icon_dir)
 414
 415     apks = []
 416     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 417     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 418     vername_pat = re.compile(".*versionName='([^']*)'.*")
 419     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 420     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 421     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 422     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 423     string_pat = re.compile(".*'([^']*)'.*")
 424     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 425
 426         apkfilename = apkfile[len(repodir) + 1:]
 427         if ' ' in apkfilename:
 428             logging.critical("Spaces in filenames are not allowed.")
 429             sys.exit(1)
 430
 431         # Calculate the sha256...
 432         sha = hashlib.sha256()
 433         with open(apkfile, 'rb') as f:
 434             while True:
 435                 t = f.read(16384)
 436                 if len(t) == 0:
 437                     break
 438                 sha.update(t)
 439             shasum = sha.hexdigest()
 440
 441         usecache = False
 442         if apkfilename in apkcache:
 443             thisinfo = apkcache[apkfilename]
 444             if thisinfo['sha256'] == shasum:
 445                 logging.debug("Reading " + apkfilename + " from cache")
 446                 usecache = True
 447             else:
 448                 logging.debug("Ignoring stale cache data for " + apkfilename)
 449
 450         if not usecache:
 451             logging.debug("Processing " + apkfilename)
 452             thisinfo = {}
 453             thisinfo['apkname'] = apkfilename
 454             thisinfo['sha256'] = shasum
 455             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 456             if os.path.exists(os.path.join(repodir, srcfilename)):
 457                 thisinfo['srcname'] = srcfilename
 458             thisinfo['size'] = os.path.getsize(apkfile)
 459             thisinfo['permissions'] = set()
 460             thisinfo['features'] = set()
 461             thisinfo['icons_src'] = {}
 462             thisinfo['icons'] = {}
 463             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 464             if p.returncode != 0:
 465                 if options.delete_unknown:
 466                     if os.path.exists(apkfile):
 467                         logging.error("Failed to get apk information, deleting " + apkfile)
 468                         os.remove(apkfile)
 469                     else:
 470                         logging.error("Could not find {0} to remove it".format(apkfile))
 471                 else:
 472                     logging.error("Failed to get apk information, skipping " + apkfile)
 473                 continue
 474             for line in p.output.splitlines():
 475                 if line.startswith("package:"):
 476                     try:
 477                         thisinfo['id'] = re.match(name_pat, line).group(1)
 478                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 479                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 480                     except Exception, e:
 481                         logging.error("Package matching failed: " + str(e))
 482                         logging.info("Line was: " + line)
 483                         sys.exit(1)
 484                 elif line.startswith("application:"):
 485                     thisinfo['name'] = re.match(label_pat, line).group(1)
 486                     # Keep path to non-dpi icon in case we need it
 487                     match = re.match(icon_pat_nodpi, line)
 488                     if match:
 489                         thisinfo['icons_src']['-1'] = match.group(1)
 490                 elif line.startswith("launchable-activity:"):
 491                     # Only use launchable-activity as fallback to application
 492                     if not thisinfo['name']:
 493                         thisinfo['name'] = re.match(label_pat, line).group(1)
 494                     if '-1' not in thisinfo['icons_src']:
 495                         match = re.match(icon_pat_nodpi, line)
 496                         if match:
 497                             thisinfo['icons_src']['-1'] = match.group(1)
 498                 elif line.startswith("application-icon-"):
 499                     match = re.match(icon_pat, line)
 500                     if match:
 501                         density = match.group(1)
 502                         path = match.group(2)
 503                         thisinfo['icons_src'][density] = path
 504                 elif line.startswith("sdkVersion:"):
 505                     m = re.match(sdkversion_pat, line)
 506                     if m is None:
 507                         logging.error(line.replace('sdkVersion:', '')
 508                                       + ' is not a valid minSdkVersion!')
 509                     else:
 510                         thisinfo['sdkversion'] = m.group(1)
 511                 elif line.startswith("maxSdkVersion:"):
 512                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 513                 elif line.startswith("native-code:"):
 514                     thisinfo['nativecode'] = []
 515                     for arch in line[13:].split(' '):
 516                         thisinfo['nativecode'].append(arch[1:-1])
 517                 elif line.startswith("uses-permission:"):
 518                     perm = re.match(string_pat, line).group(1)
 519                     if perm.startswith("android.permission."):
 520                         perm = perm[19:]
 521                     thisinfo['permissions'].add(perm)
 522                 elif line.startswith("uses-feature:"):
 523                     perm = re.match(string_pat, line).group(1)
 524                     # Filter out this, it's only added with the latest SDK tools and
 525                     # causes problems for lots of apps.
 526                     if perm != "android.hardware.screen.portrait" \
 527                             and perm != "android.hardware.screen.landscape":
 528                         if perm.startswith("android.feature."):
 529                             perm = perm[16:]
 530                         thisinfo['features'].add(perm)
 531
 532             if 'sdkversion' not in thisinfo:
 533                 logging.warn("No SDK version information found in {0}".format(apkfile))
 534                 thisinfo['sdkversion'] = 0
 535
 536             # Check for debuggable apks...
 537             if common.isApkDebuggable(apkfile, config):
 538                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 539
 540             # Get the signature (or md5 of, to be precise)...
 541             logging.debug('Getting signature of {0}'.format(apkfile))
 542             thisinfo['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 543             if not thisinfo['sig']:
 544                 logging.critical("Failed to get apk signature")
 545                 sys.exit(1)
 546
 547             apk = zipfile.ZipFile(apkfile, 'r')
 548
 549             # if an APK has files newer than the system time, suggest updating
 550             # the system clock.  This is useful for offline systems, used for
 551             # signing, which do not have another source of clock sync info. It
 552             # has to be more than 24 hours newer because ZIP/APK files do not
 553             # store timezone info
 554             info = apk.getinfo('AndroidManifest.xml')
 555             dt_obj = datetime(*info.date_time)
 556             checkdt = dt_obj - timedelta(1)
 557             if datetime.today() < checkdt:
 558                 logging.warn('System clock is older than manifest in: '
 559                              + apkfilename + '\nSet clock to that time using:\n'
 560                              + 'sudo date -s "' + str(dt_obj) + '"')
 561
 562             iconfilename = "%s.%s.png" % (
 563                 thisinfo['id'],
 564                 thisinfo['versioncode'])
 565
 566             # Extract the icon file...
 567             densities = get_densities()
 568             empty_densities = []
 569             for density in densities:
 570                 if density not in thisinfo['icons_src']:
 571                     empty_densities.append(density)
 572                     continue
 573                 iconsrc = thisinfo['icons_src'][density]
 574                 icon_dir = get_icon_dir(repodir, density)
 575                 icondest = os.path.join(icon_dir, iconfilename)
 576
 577                 try:
 578                     with open(icondest, 'wb') as f:
 579                         f.write(apk.read(iconsrc))
 580                     thisinfo['icons'][density] = iconfilename
 581
 582                 except:
 583                     logging.warn("Error retrieving icon file")
 584                     del thisinfo['icons'][density]
 585                     del thisinfo['icons_src'][density]
 586                     empty_densities.append(density)
 587
 588             if '-1' in thisinfo['icons_src']:
 589                 iconsrc = thisinfo['icons_src']['-1']
 590                 iconpath = os.path.join(
 591                     get_icon_dir(repodir, None), iconfilename)
 592                 with open(iconpath, 'wb') as f:
 593                     f.write(apk.read(iconsrc))
 594                 try:
 595                     im = Image.open(iconpath)
 596                     dpi = px_to_dpi(im.size[0])
 597                     for density in densities:
 598                         if density in thisinfo['icons']:
 599                             break
 600                         if density == densities[-1] or dpi >= int(density):
 601                             thisinfo['icons'][density] = iconfilename
 602                             shutil.move(iconpath,
 603                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 604                             empty_densities.remove(density)
 605                             break
 606                 except Exception, e:
 607                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 608
 609             if thisinfo['icons']:
 610                 thisinfo['icon'] = iconfilename
 611
 612             apk.close()
 613
 614             # First try resizing down to not lose quality
 615             last_density = None
 616             for density in densities:
 617                 if density not in empty_densities:
 618                     last_density = density
 619                     continue
 620                 if last_density is None:
 621                     continue
 622                 logging.debug("Density %s not available, resizing down from %s"
 623                               % (density, last_density))
 624
 625                 last_iconpath = os.path.join(
 626                     get_icon_dir(repodir, last_density), iconfilename)
 627                 iconpath = os.path.join(
 628                     get_icon_dir(repodir, density), iconfilename)
 629                 try:
 630                     im = Image.open(last_iconpath)
 631                 except:
 632                     logging.warn("Invalid image file at %s" % last_iconpath)
 633                     continue
 634
 635                 size = dpi_to_px(density)
 636
 637                 im.thumbnail((size, size), Image.ANTIALIAS)
 638                 im.save(iconpath, "PNG")
 639                 empty_densities.remove(density)
 640
 641             # Then just copy from the highest resolution available
 642             last_density = None
 643             for density in reversed(densities):
 644                 if density not in empty_densities:
 645                     last_density = density
 646                     continue
 647                 if last_density is None:
 648                     continue
 649                 logging.debug("Density %s not available, copying from lower density %s"
 650                               % (density, last_density))
 651
 652                 shutil.copyfile(
 653                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 654                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 655
 656                 empty_densities.remove(density)
 657
 658             for density in densities:
 659                 icon_dir = get_icon_dir(repodir, density)
 660                 icondest = os.path.join(icon_dir, iconfilename)
 661                 resize_icon(icondest, density)
 662
 663             # Copy from icons-mdpi to icons since mdpi is the baseline density
 664             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 665             if os.path.isfile(baseline):
 666                 shutil.copyfile(baseline,
 667                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 668
 669             # Record in known apks, getting the added date at the same time..
 670             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 671             if added:
 672                 thisinfo['added'] = added
 673
 674             apkcache[apkfilename] = thisinfo
 675             cachechanged = True
 676
 677         apks.append(thisinfo)
 678
 679     return apks, cachechanged
 680
 681
 682 repo_pubkey_fingerprint = None
 683
 684
 685 # Generate a certificate fingerprint the same way keytool does it
 686 # (but with slightly different formatting)
 687 def cert_fingerprint(data):
 688     digest = hashlib.sha256(data).digest()
 689     ret = []
 690     ret.append(' '.join("%02X" % ord(b) for b in digest))
 691     return " ".join(ret)
 692
 693
 694 def extract_pubkey():
 695     global repo_pubkey_fingerprint
 696     if 'repo_pubkey' in config:
 697         pubkey = unhexlify(config['repo_pubkey'])
 698     else:
 699         p = FDroidPopen(['keytool', '-exportcert',
 700                          '-alias', config['repo_keyalias'],
 701                          '-keystore', config['keystore'],
 702                          '-storepass:file', config['keystorepassfile']]
 703                         + config['smartcardoptions'], output=False)
 704         if p.returncode != 0 or len(p.output) < 20:
 705             msg = "Failed to get repo pubkey!"
 706             if config['keystore'] == 'NONE':
 707                 msg += ' Is your crypto smartcard plugged in?'
 708             logging.critical(msg)
 709             sys.exit(1)
 710         pubkey = p.output
 711     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 712     return hexlify(pubkey)
 713
 714
 715 def make_index(apps, sortedids, apks, repodir, archive, categories):
 716     """Make a repo index.
 717
 718     :param apps: fully populated apps list
 719     :param apks: full populated apks list
 720     :param repodir: the repo directory
 721     :param archive: True if this is the archive repo, False if it's the
 722                     main one.
 723     :param categories: list of categories
 724     """
 725
 726     doc = Document()
 727
 728     def addElement(name, value, doc, parent):
 729         el = doc.createElement(name)
 730         el.appendChild(doc.createTextNode(value))
 731         parent.appendChild(el)
 732
 733     def addElementNonEmpty(name, value, doc, parent):
 734         if not value:
 735             return
 736         addElement(name, value, doc, parent)
 737
 738     def addElementCDATA(name, value, doc, parent):
 739         el = doc.createElement(name)
 740         el.appendChild(doc.createCDATASection(value))
 741         parent.appendChild(el)
 742
 743     root = doc.createElement("fdroid")
 744     doc.appendChild(root)
 745
 746     repoel = doc.createElement("repo")
 747
 748     if archive:
 749         repoel.setAttribute("name", config['archive_name'])
 750         if config['repo_maxage'] != 0:
 751             repoel.setAttribute("maxage", str(config['repo_maxage']))
 752         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 753         repoel.setAttribute("url", config['archive_url'])
 754         addElement('description', config['archive_description'], doc, repoel)
 755
 756     else:
 757         repoel.setAttribute("name", config['repo_name'])
 758         if config['repo_maxage'] != 0:
 759             repoel.setAttribute("maxage", str(config['repo_maxage']))
 760         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 761         repoel.setAttribute("url", config['repo_url'])
 762         addElement('description', config['repo_description'], doc, repoel)
 763
 764     repoel.setAttribute("version", "14")
 765     repoel.setAttribute("timestamp", str(int(time.time())))
 766
 767     nosigningkey = False
 768     if not options.nosign:
 769         if 'repo_keyalias' not in config:
 770             nosigningkey = True
 771             logging.critical("'repo_keyalias' not found in config.py!")
 772         if 'keystore' not in config:
 773             nosigningkey = True
 774             logging.critical("'keystore' not found in config.py!")
 775         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 776             nosigningkey = True
 777             logging.critical("'keystorepass' not found in config.py!")
 778         if 'keypass' not in config and 'keypassfile' not in config:
 779             nosigningkey = True
 780             logging.critical("'keypass' not found in config.py!")
 781         if not os.path.exists(config['keystore']):
 782             nosigningkey = True
 783             logging.critical("'" + config['keystore'] + "' does not exist!")
 784         if nosigningkey:
 785             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 786             logging.warning("\tfdroid update --create-key")
 787             sys.exit(1)
 788
 789     repoel.setAttribute("pubkey", extract_pubkey())
 790     root.appendChild(repoel)
 791
 792     for appid in sortedids:
 793         app = apps[appid]
 794
 795         if app['Disabled'] is not None:
 796             continue
 797
 798         # Get a list of the apks for this app...
 799         apklist = []
 800         for apk in apks:
 801             if apk['id'] == appid:
 802                 apklist.append(apk)
 803
 804         if len(apklist) == 0:
 805             continue
 806
 807         apel = doc.createElement("application")
 808         apel.setAttribute("id", app['id'])
 809         root.appendChild(apel)
 810
 811         addElement('id', app['id'], doc, apel)
 812         if 'added' in app:
 813             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 814         if 'lastupdated' in app:
 815             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 816         addElement('name', app['Name'], doc, apel)
 817         addElement('summary', app['Summary'], doc, apel)
 818         if app['icon']:
 819             addElement('icon', app['icon'], doc, apel)
 820
 821         def linkres(appid):
 822             if appid in apps:
 823                 return ("fdroid.app:" + appid, apps[appid]['Name'])
 824             raise MetaDataException("Cannot resolve app id " + appid)
 825
 826         addElement('desc',
 827                    metadata.description_html(app['Description'], linkres),
 828                    doc, apel)
 829         addElement('license', app['License'], doc, apel)
 830         if 'Categories' in app and app['Categories']:
 831             addElement('categories', ','.join(app["Categories"]), doc, apel)
 832             # We put the first (primary) category in LAST, which will have
 833             # the desired effect of making clients that only understand one
 834             # category see that one.
 835             addElement('category', app["Categories"][0], doc, apel)
 836         addElement('web', app['Web Site'], doc, apel)
 837         addElement('source', app['Source Code'], doc, apel)
 838         addElement('tracker', app['Issue Tracker'], doc, apel)
 839         addElementNonEmpty('changelog', app['Changelog'], doc, apel)
 840         addElementNonEmpty('donate', app['Donate'], doc, apel)
 841         addElementNonEmpty('bitcoin', app['Bitcoin'], doc, apel)
 842         addElementNonEmpty('litecoin', app['Litecoin'], doc, apel)
 843         addElementNonEmpty('flattr', app['FlattrID'], doc, apel)
 844
 845         # These elements actually refer to the current version (i.e. which
 846         # one is recommended. They are historically mis-named, and need
 847         # changing, but stay like this for now to support existing clients.
 848         addElement('marketversion', app['Current Version'], doc, apel)
 849         addElement('marketvercode', app['Current Version Code'], doc, apel)
 850
 851         if app['AntiFeatures']:
 852             af = app['AntiFeatures']
 853             if af:
 854                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 855         if app['Provides']:
 856             pv = app['Provides'].split(',')
 857             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 858         if app['Requires Root']:
 859             addElement('requirements', 'root', doc, apel)
 860
 861         # Sort the apk list into version order, just so the web site
 862         # doesn't have to do any work by default...
 863         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 864
 865         # Check for duplicates - they will make the client unhappy...
 866         for i in range(len(apklist) - 1):
 867             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 868                 logging.critical("duplicate versions: '%s' - '%s'" % (
 869                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 870                 sys.exit(1)
 871
 872         current_version_code = 0
 873         current_version_file = None
 874         for apk in apklist:
 875             # find the APK for the "Current Version"
 876             if current_version_code < apk['versioncode']:
 877                 current_version_code = apk['versioncode']
 878             if current_version_code < int(app['Current Version Code']):
 879                 current_version_file = apk['apkname']
 880
 881             apkel = doc.createElement("package")
 882             apel.appendChild(apkel)
 883             addElement('version', apk['version'], doc, apkel)
 884             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 885             addElement('apkname', apk['apkname'], doc, apkel)
 886             if 'srcname' in apk:
 887                 addElement('srcname', apk['srcname'], doc, apkel)
 888             for hash_type in ['sha256']:
 889                 if hash_type not in apk:
 890                     continue
 891                 hashel = doc.createElement("hash")
 892                 hashel.setAttribute("type", hash_type)
 893                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 894                 apkel.appendChild(hashel)
 895             addElement('sig', apk['sig'], doc, apkel)
 896             addElement('size', str(apk['size']), doc, apkel)
 897             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 898             if 'maxsdkversion' in apk:
 899                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 900             if 'added' in apk:
 901                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 902             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
 903             if 'nativecode' in apk:
 904                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 905             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
 906
 907         if current_version_file is not None \
 908                 and config['make_current_version_link'] \
 909                 and repodir == 'repo':  # only create these
 910             sanitized_name = re.sub('''[ '"&%?+=/]''', '',
 911                                     app[config['current_version_name_source']])
 912             apklinkname = sanitized_name + '.apk'
 913             current_version_path = os.path.join(repodir, current_version_file)
 914             if os.path.islink(apklinkname):
 915                 os.remove(apklinkname)
 916             os.symlink(current_version_path, apklinkname)
 917             # also symlink gpg signature, if it exists
 918             for extension in ('.asc', '.sig'):
 919                 sigfile_path = current_version_path + extension
 920                 if os.path.exists(sigfile_path):
 921                     siglinkname = apklinkname + extension
 922                     if os.path.islink(siglinkname):
 923                         os.remove(siglinkname)
 924                     os.symlink(sigfile_path, siglinkname)
 925
 926     if options.pretty:
 927         output = doc.toprettyxml()
 928     else:
 929         output = doc.toxml()
 930
 931     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 932         f.write(output)
 933
 934     if 'repo_keyalias' in config:
 935
 936         if options.nosign:
 937             logging.info("Creating unsigned index in preparation for signing")
 938         else:
 939             logging.info("Creating signed index with this key (SHA256):")
 940             logging.info("%s" % repo_pubkey_fingerprint)
 941
 942         # Create a jar of the index...
 943         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
 944         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 945         if p.returncode != 0:
 946             logging.critical("Failed to create {0}".format(jar_output))
 947             sys.exit(1)
 948
 949         # Sign the index...
 950         signed = os.path.join(repodir, 'index.jar')
 951         if options.nosign:
 952             # Remove old signed index if not signing
 953             if os.path.exists(signed):
 954                 os.remove(signed)
 955         else:
 956             args = ['jarsigner', '-keystore', config['keystore'],
 957                     '-storepass:file', config['keystorepassfile'],
 958                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
 959                     signed, config['repo_keyalias']]
 960             if config['keystore'] == 'NONE':
 961                 args += config['smartcardoptions']
 962             else:  # smardcards never use -keypass
 963                 args += ['-keypass:file', config['keypassfile']]
 964             p = FDroidPopen(args)
 965             if p.returncode != 0:
 966                 logging.critical("Failed to sign index")
 967                 sys.exit(1)
 968
 969     # Copy the repo icon into the repo directory...
 970     icon_dir = os.path.join(repodir, 'icons')
 971     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 972     shutil.copyfile(config['repo_icon'], iconfilename)
 973
 974     # Write a category list in the repo to allow quick access...
 975     catdata = ''
 976     for cat in categories:
 977         catdata += cat + '\n'
 978     with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
 979         f.write(catdata)
 980
 981
 982 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 983
 984     for appid, app in apps.iteritems():
 985
 986         if app['Archive Policy']:
 987             keepversions = int(app['Archive Policy'][:-9])
 988         else:
 989             keepversions = defaultkeepversions
 990
 991         def filter_apk_list_sorted(apk_list):
 992             res = []
 993             for apk in apk_list:
 994                 if apk['id'] == appid:
 995                     res.append(apk)
 996
 997             # Sort the apk list by version code. First is highest/newest.
 998             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
 999
1000         def move_file(from_dir, to_dir, filename, ignore_missing):
1001             from_path = os.path.join(from_dir, filename)
1002             if ignore_missing and not os.path.exists(from_path):
1003                 return
1004             to_path = os.path.join(to_dir, filename)
1005             shutil.move(from_path, to_path)
1006
1007         if len(apks) > keepversions:
1008             apklist = filter_apk_list_sorted(apks)
1009             # Move back the ones we don't want.
1010             for apk in apklist[keepversions:]:
1011                 logging.info("Moving " + apk['apkname'] + " to archive")
1012                 move_file(repodir, archivedir, apk['apkname'], False)
1013                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1014                 if 'srcname' in apk:
1015                     move_file(repodir, archivedir, apk['srcname'], False)
1016                 archapks.append(apk)
1017                 apks.remove(apk)
1018         elif len(apks) < keepversions and len(archapks) > 0:
1019             required = keepversions - len(apks)
1020             archapklist = filter_apk_list_sorted(archapks)
1021             # Move forward the ones we want again.
1022             for apk in archapklist[:required]:
1023                 logging.info("Moving " + apk['apkname'] + " from archive")
1024                 move_file(archivedir, repodir, apk['apkname'], False)
1025                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1026                 if 'srcname' in apk:
1027                     move_file(archivedir, repodir, apk['srcname'], False)
1028                 archapks.remove(apk)
1029                 apks.append(apk)
1030
1031
1032 def add_apks_to_per_app_repos(repodir, apks):
1033     apks_per_app = dict()
1034     for apk in apks:
1035         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1036         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1037         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1038         apks_per_app[apk['id']] = apk
1039
1040         if not os.path.exists(apk['per_app_icons']):
1041             logging.info('Adding new repo for only ' + apk['id'])
1042             os.makedirs(apk['per_app_icons'])
1043
1044         apkpath = os.path.join(repodir, apk['apkname'])
1045         shutil.copy(apkpath, apk['per_app_repo'])
1046         apksigpath = apkpath + '.sig'
1047         if os.path.exists(apksigpath):
1048             shutil.copy(apksigpath, apk['per_app_repo'])
1049         apkascpath = apkpath + '.asc'
1050         if os.path.exists(apkascpath):
1051             shutil.copy(apkascpath, apk['per_app_repo'])
1052
1053
1054 config = None
1055 options = None
1056
1057
1058 def main():
1059
1060     global config, options
1061
1062     # Parse command line...
1063     parser = ArgumentParser()
1064     common.setup_global_opts(parser)
1065     parser.add_argument("--create-key", action="store_true", default=False,
1066                         help="Create a repo signing key in a keystore")
1067     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1068                         help="Create skeleton metadata files that are missing")
1069     parser.add_argument("--delete-unknown", action="store_true", default=False,
1070                         help="Delete APKs without metadata from the repo")
1071     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1072                         help="Report on build data status")
1073     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1074                         help="Interactively ask about things that need updating.")
1075     parser.add_argument("-I", "--icons", action="store_true", default=False,
1076                         help="Resize all the icons exceeding the max pixel size and exit")
1077     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1078                         help="Specify editor to use in interactive mode. Default " +
1079                         "is /etc/alternatives/editor")
1080     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1081                         help="Update the wiki")
1082     parser.add_argument("--pretty", action="store_true", default=False,
1083                         help="Produce human-readable index.xml")
1084     parser.add_argument("--clean", action="store_true", default=False,
1085                         help="Clean update - don't uses caches, reprocess all apks")
1086     parser.add_argument("--nosign", action="store_true", default=False,
1087                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1088     options = parser.parse_args()
1089
1090     config = common.read_config(options)
1091
1092     repodirs = ['repo']
1093     if config['archive_older'] != 0:
1094         repodirs.append('archive')
1095         if not os.path.exists('archive'):
1096             os.mkdir('archive')
1097
1098     if options.icons:
1099         resize_all_icons(repodirs)
1100         sys.exit(0)
1101
1102     # check that icons exist now, rather than fail at the end of `fdroid update`
1103     for k in ['repo_icon', 'archive_icon']:
1104         if k in config:
1105             if not os.path.exists(config[k]):
1106                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1107                 sys.exit(1)
1108
1109     # if the user asks to create a keystore, do it now, reusing whatever it can
1110     if options.create_key:
1111         if os.path.exists(config['keystore']):
1112             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1113             logging.critical("\t'" + config['keystore'] + "'")
1114             sys.exit(1)
1115
1116         if 'repo_keyalias' not in config:
1117             config['repo_keyalias'] = socket.getfqdn()
1118             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1119         if 'keydname' not in config:
1120             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1121             common.write_to_config(config, 'keydname', config['keydname'])
1122         if 'keystore' not in config:
1123             config['keystore'] = common.default_config.keystore
1124             common.write_to_config(config, 'keystore', config['keystore'])
1125
1126         password = common.genpassword()
1127         if 'keystorepass' not in config:
1128             config['keystorepass'] = password
1129             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1130         if 'keypass' not in config:
1131             config['keypass'] = password
1132             common.write_to_config(config, 'keypass', config['keypass'])
1133         common.genkeystore(config)
1134
1135     # Get all apps...
1136     apps = metadata.read_metadata()
1137
1138     # Generate a list of categories...
1139     categories = set()
1140     for app in apps.itervalues():
1141         categories.update(app['Categories'])
1142
1143     # Read known apks data (will be updated and written back when we've finished)
1144     knownapks = common.KnownApks()
1145
1146     # Gather information about all the apk files in the repo directory, using
1147     # cached data if possible.
1148     apkcachefile = os.path.join('tmp', 'apkcache')
1149     if not options.clean and os.path.exists(apkcachefile):
1150         with open(apkcachefile, 'rb') as cf:
1151             apkcache = pickle.load(cf)
1152     else:
1153         apkcache = {}
1154
1155     delete_disabled_builds(apps, apkcache, repodirs)
1156
1157     # Scan all apks in the main repo
1158     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks)
1159
1160     # Generate warnings for apk's with no metadata (or create skeleton
1161     # metadata files, if requested on the command line)
1162     newmetadata = False
1163     for apk in apks:
1164         if apk['id'] not in apps:
1165             if options.create_metadata:
1166                 if 'name' not in apk:
1167                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1168                     continue
1169                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1170                 f.write("License:Unknown\n")
1171                 f.write("Web Site:\n")
1172                 f.write("Source Code:\n")
1173                 f.write("Issue Tracker:\n")
1174                 f.write("Changelog:\n")
1175                 f.write("Summary:" + apk['name'] + "\n")
1176                 f.write("Description:\n")
1177                 f.write(apk['name'] + "\n")
1178                 f.write(".\n")
1179                 f.close()
1180                 logging.info("Generated skeleton metadata for " + apk['id'])
1181                 newmetadata = True
1182             else:
1183                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1184                 if options.delete_unknown:
1185                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1186                     rmf = os.path.join(repodirs[0], apk['apkname'])
1187                     if not os.path.exists(rmf):
1188                         logging.error("Could not find {0} to remove it".format(rmf))
1189                     else:
1190                         os.remove(rmf)
1191                 else:
1192                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1193
1194     # update the metadata with the newly created ones included
1195     if newmetadata:
1196         apps = metadata.read_metadata()
1197
1198     # Scan the archive repo for apks as well
1199     if len(repodirs) > 1:
1200         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1201         if cc:
1202             cachechanged = True
1203     else:
1204         archapks = []
1205
1206     # Some information from the apks needs to be applied up to the application
1207     # level. When doing this, we use the info from the most recent version's apk.
1208     # We deal with figuring out when the app was added and last updated at the
1209     # same time.
1210     for appid, app in apps.iteritems():
1211         bestver = 0
1212         added = None
1213         lastupdated = None
1214         for apk in apks + archapks:
1215             if apk['id'] == appid:
1216                 if apk['versioncode'] > bestver:
1217                     bestver = apk['versioncode']
1218                     bestapk = apk
1219
1220                 if 'added' in apk:
1221                     if not added or apk['added'] < added:
1222                         added = apk['added']
1223                     if not lastupdated or apk['added'] > lastupdated:
1224                         lastupdated = apk['added']
1225
1226         if added:
1227             app['added'] = added
1228         else:
1229             logging.warn("Don't know when " + appid + " was added")
1230         if lastupdated:
1231             app['lastupdated'] = lastupdated
1232         else:
1233             logging.warn("Don't know when " + appid + " was last updated")
1234
1235         if bestver == 0:
1236             if app['Name'] is None:
1237                 app['Name'] = app['Auto Name'] or appid
1238             app['icon'] = None
1239             logging.warn("Application " + appid + " has no packages")
1240         else:
1241             if app['Name'] is None:
1242                 app['Name'] = bestapk['name']
1243             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1244
1245     # Sort the app list by name, then the web site doesn't have to by default.
1246     # (we had to wait until we'd scanned the apks to do this, because mostly the
1247     # name comes from there!)
1248     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid]['Name'].upper())
1249
1250     # APKs are placed into multiple repos based on the app package, providing
1251     # per-app subscription feeds for nightly builds and things like it
1252     if config['per_app_repos']:
1253         add_apks_to_per_app_repos(repodirs[0], apks)
1254         for appid, app in apps.iteritems():
1255             repodir = os.path.join(appid, 'fdroid', 'repo')
1256             appdict = dict()
1257             appdict[appid] = app
1258             if os.path.isdir(repodir):
1259                 make_index(appdict, [appid], apks, repodir, False, categories)
1260             else:
1261                 logging.info('Skipping index generation for ' + appid)
1262         return
1263
1264     if len(repodirs) > 1:
1265         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1266
1267     # Make the index for the main repo...
1268     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1269
1270     # If there's an archive repo,  make the index for it. We already scanned it
1271     # earlier on.
1272     if len(repodirs) > 1:
1273         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1274
1275     if config['update_stats']:
1276
1277         # Update known apks info...
1278         knownapks.writeifchanged()
1279
1280         # Generate latest apps data for widget
1281         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1282             data = ''
1283             for line in file(os.path.join('stats', 'latestapps.txt')):
1284                 appid = line.rstrip()
1285                 data += appid + "\t"
1286                 app = apps[appid]
1287                 data += app['Name'] + "\t"
1288                 if app['icon'] is not None:
1289                     data += app['icon'] + "\t"
1290                 data += app['License'] + "\n"
1291             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
1292                 f.write(data)
1293
1294     if cachechanged:
1295         with open(apkcachefile, 'wb') as cf:
1296             pickle.dump(apkcache, cf)
1297
1298     # Update the wiki...
1299     if options.wiki:
1300         update_wiki(apps, sortedids, apks + archapks)
1301
1302     logging.info("Finished.")
1303
1304 if __name__ == "__main__":
1305     main()