fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import socket
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from datetime import datetime, timedelta
  31 from xml.dom.minidom import Document
  32 from optparse import OptionParser
  33 import time
  34 from pyasn1.error import PyAsn1Error
  35 from pyasn1.codec.der import decoder, encoder
  36 from pyasn1_modules import rfc2315
  37 from hashlib import md5
  38 from binascii import hexlify, unhexlify
  39
  40 from PIL import Image
  41 import logging
  42
  43 import common
  44 import metadata
  45 from common import FDroidPopen, SdkToolsPopen
  46 from metadata import MetaDataException
  47
  48
  49 def get_densities():
  50     return ['640', '480', '320', '240', '160', '120']
  51
  52
  53 def dpi_to_px(density):
  54     return (int(density) * 48) / 160
  55
  56
  57 def px_to_dpi(px):
  58     return (int(px) * 160) / 48
  59
  60
  61 def get_icon_dir(repodir, density):
  62     if density is None:
  63         return os.path.join(repodir, "icons")
  64     return os.path.join(repodir, "icons-%s" % density)
  65
  66
  67 def get_icon_dirs(repodir):
  68     for density in get_densities():
  69         yield get_icon_dir(repodir, density)
  70     yield os.path.join(repodir, "icons")
  71
  72
  73 def update_wiki(apps, sortedids, apks):
  74     """Update the wiki
  75
  76     :param apps: fully populated list of all applications
  77     :param apks: all apks, except...
  78     """
  79     logging.info("Updating wiki")
  80     wikicat = 'Apps'
  81     wikiredircat = 'App Redirects'
  82     import mwclient
  83     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  84                          path=config['wiki_path'])
  85     site.login(config['wiki_user'], config['wiki_password'])
  86     generated_pages = {}
  87     generated_redirects = {}
  88
  89     for appid in sortedids:
  90         app = apps[appid]
  91
  92         wikidata = ''
  93         if app['Disabled']:
  94             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  95         if app['AntiFeatures']:
  96             for af in app['AntiFeatures'].split(','):
  97                 wikidata += '{{AntiFeature|' + af + '}}\n'
  98         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
  99             appid,
 100             app['Name'],
 101             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
 102             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
 103             app['Source Code'],
 104             app['Issue Tracker'],
 105             app['Web Site'],
 106             app['Changelog'],
 107             app['Donate'],
 108             app['FlattrID'],
 109             app['Bitcoin'],
 110             app['Litecoin'],
 111             app['Dogecoin'],
 112             app['License'],
 113             app.get('Requires Root', 'No'))
 114
 115         if app['Provides']:
 116             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 117
 118         wikidata += app['Summary']
 119         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 120
 121         wikidata += "=Description=\n"
 122         wikidata += metadata.description_wiki(app['Description']) + "\n"
 123
 124         wikidata += "=Maintainer Notes=\n"
 125         if 'Maintainer Notes' in app:
 126             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 127         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 128
 129         # Get a list of all packages for this application...
 130         apklist = []
 131         gotcurrentver = False
 132         cantupdate = False
 133         buildfails = False
 134         for apk in apks:
 135             if apk['id'] == appid:
 136                 if str(apk['versioncode']) == app['Current Version Code']:
 137                     gotcurrentver = True
 138                 apklist.append(apk)
 139         # Include ones we can't build, as a special case...
 140         for thisbuild in app['builds']:
 141             if thisbuild['disable']:
 142                 if thisbuild['vercode'] == app['Current Version Code']:
 143                     cantupdate = True
 144                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 145                 apklist.append({'versioncode': int(thisbuild['vercode']),
 146                                 'version': thisbuild['version'],
 147                                 'buildproblem': thisbuild['disable']
 148                                 })
 149             else:
 150                 builtit = False
 151                 for apk in apklist:
 152                     if apk['versioncode'] == int(thisbuild['vercode']):
 153                         builtit = True
 154                         break
 155                 if not builtit:
 156                     buildfails = True
 157                     apklist.append({'versioncode': int(thisbuild['vercode']),
 158                                     'version': thisbuild['version'],
 159                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, thisbuild['vercode'])
 160                                     })
 161         if app['Current Version Code'] == '0':
 162             cantupdate = True
 163         # Sort with most recent first...
 164         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 165
 166         wikidata += "=Versions=\n"
 167         if len(apklist) == 0:
 168             wikidata += "We currently have no versions of this app available."
 169         elif not gotcurrentver:
 170             wikidata += "We don't have the current version of this app."
 171         else:
 172             wikidata += "We have the current version of this app."
 173         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 174         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 175         if len(app['No Source Since']) > 0:
 176             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 177         if len(app['Current Version']) > 0:
 178             wikidata += "The current (recommended) version is " + app['Current Version']
 179             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 180         validapks = 0
 181         for apk in apklist:
 182             wikidata += "==" + apk['version'] + "==\n"
 183
 184             if 'buildproblem' in apk:
 185                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 186             else:
 187                 validapks += 1
 188                 wikidata += "This version is built and signed by "
 189                 if 'srcname' in apk:
 190                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 191                 else:
 192                     wikidata += "the original developer.\n\n"
 193             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 194
 195         wikidata += '\n[[Category:' + wikicat + ']]\n'
 196         if len(app['No Source Since']) > 0:
 197             wikidata += '\n[[Category:Apps missing source code]]\n'
 198         if validapks == 0 and not app['Disabled']:
 199             wikidata += '\n[[Category:Apps with no packages]]\n'
 200         if cantupdate and not app['Disabled']:
 201             wikidata += "\n[[Category:Apps we can't update]]\n"
 202         if buildfails and not app['Disabled']:
 203             wikidata += "\n[[Category:Apps with failing builds]]\n"
 204         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 205             wikidata += '\n[[Category:Apps to Update]]\n'
 206         if app['Disabled']:
 207             wikidata += '\n[[Category:Apps that are disabled]]\n'
 208         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 209             wikidata += '\n[[Category:Apps with no update check]]\n'
 210         for appcat in app['Categories']:
 211             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 212
 213         # We can't have underscores in the page name, even if they're in
 214         # the package ID, because MediaWiki messes with them...
 215         pagename = appid.replace('_', ' ')
 216
 217         # Drop a trailing newline, because mediawiki is going to drop it anyway
 218         # and it we don't we'll think the page has changed when it hasn't...
 219         if wikidata.endswith('\n'):
 220             wikidata = wikidata[:-1]
 221
 222         generated_pages[pagename] = wikidata
 223
 224         # Make a redirect from the name to the ID too, unless there's
 225         # already an existing page with the name and it isn't a redirect.
 226         noclobber = False
 227         apppagename = app['Name'].replace('_', ' ')
 228         apppagename = apppagename.replace('{', '')
 229         apppagename = apppagename.replace('}', ' ')
 230         apppagename = apppagename.replace(':', ' ')
 231         # Drop double spaces caused mostly by replacing ':' above
 232         apppagename = apppagename.replace('  ', ' ')
 233         for expagename in site.allpages(prefix=apppagename,
 234                                         filterredir='nonredirects',
 235                                         generator=False):
 236             if expagename == apppagename:
 237                 noclobber = True
 238         # Another reason not to make the redirect page is if the app name
 239         # is the same as it's ID, because that will overwrite the real page
 240         # with an redirect to itself! (Although it seems like an odd
 241         # scenario this happens a lot, e.g. where there is metadata but no
 242         # builds or binaries to extract a name from.
 243         if apppagename == pagename:
 244             noclobber = True
 245         if not noclobber:
 246             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 247
 248     for tcat, genp in [(wikicat, generated_pages),
 249                        (wikiredircat, generated_redirects)]:
 250         catpages = site.Pages['Category:' + tcat]
 251         existingpages = []
 252         for page in catpages:
 253             existingpages.append(page.name)
 254             if page.name in genp:
 255                 pagetxt = page.edit()
 256                 if pagetxt != genp[page.name]:
 257                     logging.debug("Updating modified page " + page.name)
 258                     page.save(genp[page.name], summary='Auto-updated')
 259                 else:
 260                     logging.debug("Page " + page.name + " is unchanged")
 261             else:
 262                 logging.warn("Deleting page " + page.name)
 263                 page.delete('No longer published')
 264         for pagename, text in genp.items():
 265             logging.debug("Checking " + pagename)
 266             if pagename not in existingpages:
 267                 logging.debug("Creating page " + pagename)
 268                 try:
 269                     newpage = site.Pages[pagename]
 270                     newpage.save(text, summary='Auto-created')
 271                 except:
 272                     logging.error("...FAILED to create page '{0}'".format(pagename))
 273
 274     # Purge server cache to ensure counts are up to date
 275     site.pages['Repository Maintenance'].purge()
 276
 277
 278 def delete_disabled_builds(apps, apkcache, repodirs):
 279     """Delete disabled build outputs.
 280
 281     :param apps: list of all applications, as per metadata.read_metadata
 282     :param apkcache: current apk cache information
 283     :param repodirs: the repo directories to process
 284     """
 285     for appid, app in apps.iteritems():
 286         for build in app['builds']:
 287             if build['disable']:
 288                 apkfilename = appid + '_' + str(build['vercode']) + '.apk'
 289                 for repodir in repodirs:
 290                     apkpath = os.path.join(repodir, apkfilename)
 291                     ascpath = apkpath + ".asc"
 292                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 293                     for name in [apkpath, srcpath, ascpath]:
 294                         if os.path.exists(name):
 295                             logging.warn("Deleting disabled build output " + apkfilename)
 296                             os.remove(name)
 297                 if apkfilename in apkcache:
 298                     del apkcache[apkfilename]
 299
 300
 301 def resize_icon(iconpath, density):
 302
 303     if not os.path.isfile(iconpath):
 304         return
 305
 306     try:
 307         im = Image.open(iconpath)
 308         size = dpi_to_px(density)
 309
 310         if any(length > size for length in im.size):
 311             oldsize = im.size
 312             im.thumbnail((size, size), Image.ANTIALIAS)
 313             logging.debug("%s was too large at %s - new size is %s" % (
 314                 iconpath, oldsize, im.size))
 315             im.save(iconpath, "PNG")
 316
 317     except Exception, e:
 318         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 319
 320
 321 def resize_all_icons(repodirs):
 322     """Resize all icons that exceed the max size
 323
 324     :param repodirs: the repo directories to process
 325     """
 326     for repodir in repodirs:
 327         for density in get_densities():
 328             icon_dir = get_icon_dir(repodir, density)
 329             icon_glob = os.path.join(icon_dir, '*.png')
 330             for iconpath in glob.glob(icon_glob):
 331                 resize_icon(iconpath, density)
 332
 333
 334 # A signature block file with a .DSA, .RSA, or .EC extension
 335 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 336
 337
 338 def getsig(apkpath):
 339     """ Get the signing certificate of an apk. To get the same md5 has that
 340     Android gets, we encode the .RSA certificate in a specific format and pass
 341     it hex-encoded to the md5 digest algorithm.
 342
 343     :param apkpath: path to the apk
 344     :returns: A string containing the md5 of the signature of the apk or None
 345               if an error occurred.
 346     """
 347
 348     cert = None
 349
 350     # verify the jar signature is correct
 351     args = ['jarsigner', '-verify', apkpath]
 352     p = FDroidPopen(args)
 353     if p.returncode != 0:
 354         logging.critical(apkpath + " has a bad signature!")
 355         return None
 356
 357     with zipfile.ZipFile(apkpath, 'r') as apk:
 358
 359         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 360
 361         if len(certs) < 1:
 362             logging.error("Found no signing certificates on %s" % apkpath)
 363             return None
 364         if len(certs) > 1:
 365             logging.error("Found multiple signing certificates on %s" % apkpath)
 366             return None
 367
 368         cert = apk.read(certs[0])
 369
 370     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 371     if content.getComponentByName('contentType') != rfc2315.signedData:
 372         logging.error("Unexpected format.")
 373         return None
 374
 375     content = decoder.decode(content.getComponentByName('content'),
 376                              asn1Spec=rfc2315.SignedData())[0]
 377     try:
 378         certificates = content.getComponentByName('certificates')
 379     except PyAsn1Error:
 380         logging.error("Certificates not found.")
 381         return None
 382
 383     cert_encoded = encoder.encode(certificates)[4:]
 384
 385     return md5(cert_encoded.encode('hex')).hexdigest()
 386
 387
 388 def scan_apks(apps, apkcache, repodir, knownapks):
 389     """Scan the apks in the given repo directory.
 390
 391     This also extracts the icons.
 392
 393     :param apps: list of all applications, as per metadata.read_metadata
 394     :param apkcache: current apk cache information
 395     :param repodir: repo directory to scan
 396     :param knownapks: known apks info
 397     :returns: (apks, cachechanged) where apks is a list of apk information,
 398               and cachechanged is True if the apkcache got changed.
 399     """
 400
 401     cachechanged = False
 402
 403     icon_dirs = get_icon_dirs(repodir)
 404     for icon_dir in icon_dirs:
 405         if os.path.exists(icon_dir):
 406             if options.clean:
 407                 shutil.rmtree(icon_dir)
 408                 os.makedirs(icon_dir)
 409         else:
 410             os.makedirs(icon_dir)
 411
 412     apks = []
 413     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 414     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 415     vername_pat = re.compile(".*versionName='([^']*)'.*")
 416     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 417     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 418     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 419     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 420     string_pat = re.compile(".*'([^']*)'.*")
 421     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 422
 423         apkfilename = apkfile[len(repodir) + 1:]
 424         if ' ' in apkfilename:
 425             logging.critical("Spaces in filenames are not allowed.")
 426             sys.exit(1)
 427
 428         # Calculate the sha256...
 429         sha = hashlib.sha256()
 430         with open(apkfile, 'rb') as f:
 431             while True:
 432                 t = f.read(16384)
 433                 if len(t) == 0:
 434                     break
 435                 sha.update(t)
 436             shasum = sha.hexdigest()
 437
 438         usecache = False
 439         if apkfilename in apkcache:
 440             thisinfo = apkcache[apkfilename]
 441             if thisinfo['sha256'] == shasum:
 442                 logging.debug("Reading " + apkfilename + " from cache")
 443                 usecache = True
 444             else:
 445                 logging.debug("Ignoring stale cache data for " + apkfilename)
 446
 447         if not usecache:
 448             logging.debug("Processing " + apkfilename)
 449             thisinfo = {}
 450             thisinfo['apkname'] = apkfilename
 451             thisinfo['sha256'] = shasum
 452             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 453             if os.path.exists(os.path.join(repodir, srcfilename)):
 454                 thisinfo['srcname'] = srcfilename
 455             thisinfo['size'] = os.path.getsize(apkfile)
 456             thisinfo['permissions'] = set()
 457             thisinfo['features'] = set()
 458             thisinfo['icons_src'] = {}
 459             thisinfo['icons'] = {}
 460             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 461             if p.returncode != 0:
 462                 if options.delete_unknown:
 463                     if os.path.exists(apkfile):
 464                         logging.error("Failed to get apk information, deleting " + apkfile)
 465                         os.remove(apkfile)
 466                     else:
 467                         logging.error("Could not find {0} to remove it".format(apkfile))
 468                 else:
 469                     logging.error("Failed to get apk information, skipping " + apkfile)
 470                 continue
 471             for line in p.output.splitlines():
 472                 if line.startswith("package:"):
 473                     try:
 474                         thisinfo['id'] = re.match(name_pat, line).group(1)
 475                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 476                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 477                     except Exception, e:
 478                         logging.error("Package matching failed: " + str(e))
 479                         logging.info("Line was: " + line)
 480                         sys.exit(1)
 481                 elif line.startswith("application:"):
 482                     thisinfo['name'] = re.match(label_pat, line).group(1)
 483                     # Keep path to non-dpi icon in case we need it
 484                     match = re.match(icon_pat_nodpi, line)
 485                     if match:
 486                         thisinfo['icons_src']['-1'] = match.group(1)
 487                 elif line.startswith("launchable-activity:"):
 488                     # Only use launchable-activity as fallback to application
 489                     if not thisinfo['name']:
 490                         thisinfo['name'] = re.match(label_pat, line).group(1)
 491                     if '-1' not in thisinfo['icons_src']:
 492                         match = re.match(icon_pat_nodpi, line)
 493                         if match:
 494                             thisinfo['icons_src']['-1'] = match.group(1)
 495                 elif line.startswith("application-icon-"):
 496                     match = re.match(icon_pat, line)
 497                     if match:
 498                         density = match.group(1)
 499                         path = match.group(2)
 500                         thisinfo['icons_src'][density] = path
 501                 elif line.startswith("sdkVersion:"):
 502                     m = re.match(sdkversion_pat, line)
 503                     if m is None:
 504                         logging.error(line.replace('sdkVersion:', '')
 505                                       + ' is not a valid minSdkVersion!')
 506                     else:
 507                         thisinfo['sdkversion'] = m.group(1)
 508                 elif line.startswith("maxSdkVersion:"):
 509                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 510                 elif line.startswith("native-code:"):
 511                     thisinfo['nativecode'] = []
 512                     for arch in line[13:].split(' '):
 513                         thisinfo['nativecode'].append(arch[1:-1])
 514                 elif line.startswith("uses-permission:"):
 515                     perm = re.match(string_pat, line).group(1)
 516                     if perm.startswith("android.permission."):
 517                         perm = perm[19:]
 518                     thisinfo['permissions'].add(perm)
 519                 elif line.startswith("uses-feature:"):
 520                     perm = re.match(string_pat, line).group(1)
 521                     # Filter out this, it's only added with the latest SDK tools and
 522                     # causes problems for lots of apps.
 523                     if perm != "android.hardware.screen.portrait" \
 524                             and perm != "android.hardware.screen.landscape":
 525                         if perm.startswith("android.feature."):
 526                             perm = perm[16:]
 527                         thisinfo['features'].add(perm)
 528
 529             if 'sdkversion' not in thisinfo:
 530                 logging.warn("No SDK version information found in {0}".format(apkfile))
 531                 thisinfo['sdkversion'] = 0
 532
 533             # Check for debuggable apks...
 534             if common.isApkDebuggable(apkfile, config):
 535                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 536
 537             # Get the signature (or md5 of, to be precise)...
 538             logging.debug('Getting signature of {0}'.format(apkfile))
 539             thisinfo['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 540             if not thisinfo['sig']:
 541                 logging.critical("Failed to get apk signature")
 542                 sys.exit(1)
 543
 544             apk = zipfile.ZipFile(apkfile, 'r')
 545
 546             # if an APK has files newer than the system time, suggest updating
 547             # the system clock.  This is useful for offline systems, used for
 548             # signing, which do not have another source of clock sync info. It
 549             # has to be more than 24 hours newer because ZIP/APK files do not
 550             # store timezone info
 551             info = apk.getinfo('AndroidManifest.xml')
 552             dt_obj = datetime(*info.date_time)
 553             checkdt = dt_obj - timedelta(1)
 554             if datetime.today() < checkdt:
 555                 logging.warn('System clock is older than manifest in: '
 556                              + apkfilename + '\nSet clock to that time using:\n'
 557                              + 'sudo date -s "' + str(dt_obj) + '"')
 558
 559             iconfilename = "%s.%s.png" % (
 560                 thisinfo['id'],
 561                 thisinfo['versioncode'])
 562
 563             # Extract the icon file...
 564             densities = get_densities()
 565             empty_densities = []
 566             for density in densities:
 567                 if density not in thisinfo['icons_src']:
 568                     empty_densities.append(density)
 569                     continue
 570                 iconsrc = thisinfo['icons_src'][density]
 571                 icon_dir = get_icon_dir(repodir, density)
 572                 icondest = os.path.join(icon_dir, iconfilename)
 573
 574                 try:
 575                     iconfile = open(icondest, 'wb')
 576                     iconfile.write(apk.read(iconsrc))
 577                     iconfile.close()
 578                     thisinfo['icons'][density] = iconfilename
 579
 580                 except:
 581                     logging.warn("Error retrieving icon file")
 582                     del thisinfo['icons'][density]
 583                     del thisinfo['icons_src'][density]
 584                     empty_densities.append(density)
 585
 586             if '-1' in thisinfo['icons_src']:
 587                 iconsrc = thisinfo['icons_src']['-1']
 588                 iconpath = os.path.join(
 589                     get_icon_dir(repodir, None), iconfilename)
 590                 iconfile = open(iconpath, 'wb')
 591                 iconfile.write(apk.read(iconsrc))
 592                 iconfile.close()
 593                 try:
 594                     im = Image.open(iconpath)
 595                     dpi = px_to_dpi(im.size[0])
 596                     for density in densities:
 597                         if density in thisinfo['icons']:
 598                             break
 599                         if density == densities[-1] or dpi >= int(density):
 600                             thisinfo['icons'][density] = iconfilename
 601                             shutil.move(iconpath,
 602                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 603                             empty_densities.remove(density)
 604                             break
 605                 except Exception, e:
 606                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 607
 608             if thisinfo['icons']:
 609                 thisinfo['icon'] = iconfilename
 610
 611             apk.close()
 612
 613             # First try resizing down to not lose quality
 614             last_density = None
 615             for density in densities:
 616                 if density not in empty_densities:
 617                     last_density = density
 618                     continue
 619                 if last_density is None:
 620                     continue
 621                 logging.debug("Density %s not available, resizing down from %s"
 622                               % (density, last_density))
 623
 624                 last_iconpath = os.path.join(
 625                     get_icon_dir(repodir, last_density), iconfilename)
 626                 iconpath = os.path.join(
 627                     get_icon_dir(repodir, density), iconfilename)
 628                 try:
 629                     im = Image.open(last_iconpath)
 630                 except:
 631                     logging.warn("Invalid image file at %s" % last_iconpath)
 632                     continue
 633
 634                 size = dpi_to_px(density)
 635
 636                 im.thumbnail((size, size), Image.ANTIALIAS)
 637                 im.save(iconpath, "PNG")
 638                 empty_densities.remove(density)
 639
 640             # Then just copy from the highest resolution available
 641             last_density = None
 642             for density in reversed(densities):
 643                 if density not in empty_densities:
 644                     last_density = density
 645                     continue
 646                 if last_density is None:
 647                     continue
 648                 logging.debug("Density %s not available, copying from lower density %s"
 649                               % (density, last_density))
 650
 651                 shutil.copyfile(
 652                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 653                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 654
 655                 empty_densities.remove(density)
 656
 657             for density in densities:
 658                 icon_dir = get_icon_dir(repodir, density)
 659                 icondest = os.path.join(icon_dir, iconfilename)
 660                 resize_icon(icondest, density)
 661
 662             # Copy from icons-mdpi to icons since mdpi is the baseline density
 663             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 664             if os.path.isfile(baseline):
 665                 shutil.copyfile(baseline,
 666                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 667
 668             # Record in known apks, getting the added date at the same time..
 669             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 670             if added:
 671                 thisinfo['added'] = added
 672
 673             apkcache[apkfilename] = thisinfo
 674             cachechanged = True
 675
 676         apks.append(thisinfo)
 677
 678     return apks, cachechanged
 679
 680
 681 repo_pubkey_fingerprint = None
 682
 683
 684 # Generate a certificate fingerprint the same way keytool does it
 685 # (but with slightly different formatting)
 686 def cert_fingerprint(data):
 687     digest = hashlib.sha256(data).digest()
 688     ret = []
 689     ret.append(' '.join("%02X" % ord(b) for b in digest))
 690     return " ".join(ret)
 691
 692
 693 def extract_pubkey():
 694     global repo_pubkey_fingerprint
 695     if 'repo_pubkey' in config:
 696         pubkey = unhexlify(config['repo_pubkey'])
 697     else:
 698         p = FDroidPopen(['keytool', '-exportcert',
 699                          '-alias', config['repo_keyalias'],
 700                          '-keystore', config['keystore'],
 701                          '-storepass:file', config['keystorepassfile']]
 702                         + config['smartcardoptions'], output=False)
 703         if p.returncode != 0 or len(p.output) < 20:
 704             msg = "Failed to get repo pubkey!"
 705             if config['keystore'] == 'NONE':
 706                 msg += ' Is your crypto smartcard plugged in?'
 707             logging.critical(msg)
 708             sys.exit(1)
 709         pubkey = p.output
 710     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 711     return hexlify(pubkey)
 712
 713
 714 def make_index(apps, sortedids, apks, repodir, archive, categories):
 715     """Make a repo index.
 716
 717     :param apps: fully populated apps list
 718     :param apks: full populated apks list
 719     :param repodir: the repo directory
 720     :param archive: True if this is the archive repo, False if it's the
 721                     main one.
 722     :param categories: list of categories
 723     """
 724
 725     doc = Document()
 726
 727     def addElement(name, value, doc, parent):
 728         el = doc.createElement(name)
 729         el.appendChild(doc.createTextNode(value))
 730         parent.appendChild(el)
 731
 732     def addElementNonEmpty(name, value, doc, parent):
 733         if not value:
 734             return
 735         addElement(name, value, doc, parent)
 736
 737     def addElementCDATA(name, value, doc, parent):
 738         el = doc.createElement(name)
 739         el.appendChild(doc.createCDATASection(value))
 740         parent.appendChild(el)
 741
 742     root = doc.createElement("fdroid")
 743     doc.appendChild(root)
 744
 745     repoel = doc.createElement("repo")
 746
 747     if archive:
 748         repoel.setAttribute("name", config['archive_name'])
 749         if config['repo_maxage'] != 0:
 750             repoel.setAttribute("maxage", str(config['repo_maxage']))
 751         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 752         repoel.setAttribute("url", config['archive_url'])
 753         addElement('description', config['archive_description'], doc, repoel)
 754
 755     else:
 756         repoel.setAttribute("name", config['repo_name'])
 757         if config['repo_maxage'] != 0:
 758             repoel.setAttribute("maxage", str(config['repo_maxage']))
 759         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 760         repoel.setAttribute("url", config['repo_url'])
 761         addElement('description', config['repo_description'], doc, repoel)
 762
 763     repoel.setAttribute("version", "13")
 764     repoel.setAttribute("timestamp", str(int(time.time())))
 765
 766     nosigningkey = False
 767     if not options.nosign:
 768         if 'repo_keyalias' not in config:
 769             nosigningkey = True
 770             logging.critical("'repo_keyalias' not found in config.py!")
 771         if 'keystore' not in config:
 772             nosigningkey = True
 773             logging.critical("'keystore' not found in config.py!")
 774         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 775             nosigningkey = True
 776             logging.critical("'keystorepass' not found in config.py!")
 777         if 'keypass' not in config and 'keypassfile' not in config:
 778             nosigningkey = True
 779             logging.critical("'keypass' not found in config.py!")
 780         if not os.path.exists(config['keystore']):
 781             nosigningkey = True
 782             logging.critical("'" + config['keystore'] + "' does not exist!")
 783         if nosigningkey:
 784             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 785             logging.warning("\tfdroid update --create-key")
 786             sys.exit(1)
 787
 788     repoel.setAttribute("pubkey", extract_pubkey())
 789     root.appendChild(repoel)
 790
 791     for appid in sortedids:
 792         app = apps[appid]
 793
 794         if app['Disabled'] is not None:
 795             continue
 796
 797         # Get a list of the apks for this app...
 798         apklist = []
 799         for apk in apks:
 800             if apk['id'] == appid:
 801                 apklist.append(apk)
 802
 803         if len(apklist) == 0:
 804             continue
 805
 806         apel = doc.createElement("application")
 807         apel.setAttribute("id", app['id'])
 808         root.appendChild(apel)
 809
 810         addElement('id', app['id'], doc, apel)
 811         if 'added' in app:
 812             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 813         if 'lastupdated' in app:
 814             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 815         addElement('name', app['Name'], doc, apel)
 816         addElement('summary', app['Summary'], doc, apel)
 817         if app['icon']:
 818             addElement('icon', app['icon'], doc, apel)
 819
 820         def linkres(appid):
 821             if appid in apps:
 822                 return ("fdroid.app:" + appid, apps[appid]['Name'])
 823             raise MetaDataException("Cannot resolve app id " + appid)
 824
 825         addElement('desc',
 826                    metadata.description_html(app['Description'], linkres),
 827                    doc, apel)
 828         addElement('license', app['License'], doc, apel)
 829         if 'Categories' in app and app['Categories']:
 830             addElement('categories', ','.join(app["Categories"]), doc, apel)
 831             # We put the first (primary) category in LAST, which will have
 832             # the desired effect of making clients that only understand one
 833             # category see that one.
 834             addElement('category', app["Categories"][0], doc, apel)
 835         addElement('web', app['Web Site'], doc, apel)
 836         addElement('source', app['Source Code'], doc, apel)
 837         addElement('tracker', app['Issue Tracker'], doc, apel)
 838         addElementNonEmpty('changelog', app['Changelog'], doc, apel)
 839         addElementNonEmpty('donate', app['Donate'], doc, apel)
 840         addElementNonEmpty('bitcoin', app['Bitcoin'], doc, apel)
 841         addElementNonEmpty('litecoin', app['Litecoin'], doc, apel)
 842         addElementNonEmpty('dogecoin', app['Dogecoin'], doc, apel)
 843         addElementNonEmpty('flattr', app['FlattrID'], doc, apel)
 844
 845         # These elements actually refer to the current version (i.e. which
 846         # one is recommended. They are historically mis-named, and need
 847         # changing, but stay like this for now to support existing clients.
 848         addElement('marketversion', app['Current Version'], doc, apel)
 849         addElement('marketvercode', app['Current Version Code'], doc, apel)
 850
 851         if app['AntiFeatures']:
 852             af = app['AntiFeatures'].split(',')
 853             if af:
 854                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 855         if app['Provides']:
 856             pv = app['Provides'].split(',')
 857             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 858         if app['Requires Root']:
 859             addElement('requirements', 'root', doc, apel)
 860
 861         # Sort the apk list into version order, just so the web site
 862         # doesn't have to do any work by default...
 863         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 864
 865         # Check for duplicates - they will make the client unhappy...
 866         for i in range(len(apklist) - 1):
 867             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 868                 logging.critical("duplicate versions: '%s' - '%s'" % (
 869                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 870                 sys.exit(1)
 871
 872         current_version_code = 0
 873         current_version_file = None
 874         for apk in apklist:
 875             # find the APK for the "Current Version"
 876             if current_version_code < apk['versioncode']:
 877                 current_version_code = apk['versioncode']
 878             if current_version_code < int(app['Current Version Code']):
 879                 current_version_file = apk['apkname']
 880
 881             apkel = doc.createElement("package")
 882             apel.appendChild(apkel)
 883             addElement('version', apk['version'], doc, apkel)
 884             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 885             addElement('apkname', apk['apkname'], doc, apkel)
 886             if 'srcname' in apk:
 887                 addElement('srcname', apk['srcname'], doc, apkel)
 888             for hash_type in ['sha256']:
 889                 if hash_type not in apk:
 890                     continue
 891                 hashel = doc.createElement("hash")
 892                 hashel.setAttribute("type", hash_type)
 893                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 894                 apkel.appendChild(hashel)
 895             addElement('sig', apk['sig'], doc, apkel)
 896             addElement('size', str(apk['size']), doc, apkel)
 897             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 898             if 'maxsdkversion' in apk:
 899                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 900             if 'added' in apk:
 901                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 902             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
 903             if 'nativecode' in apk:
 904                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 905             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
 906
 907         if current_version_file is not None \
 908                 and config['make_current_version_link'] \
 909                 and repodir == 'repo':  # only create these
 910             sanitized_name = re.sub('''[ '"&%?+=/]''', '',
 911                                     app[config['current_version_name_source']])
 912             apklinkname = sanitized_name + '.apk'
 913             current_version_path = os.path.join(repodir, current_version_file)
 914             if os.path.exists(apklinkname):
 915                 os.remove(apklinkname)
 916             os.symlink(current_version_path, apklinkname)
 917             # also symlink gpg signature, if it exists
 918             for extension in ('.asc', '.sig'):
 919                 sigfile_path = current_version_path + extension
 920                 if os.path.exists(sigfile_path):
 921                     siglinkname = apklinkname + extension
 922                     if os.path.exists(siglinkname):
 923                         os.remove(siglinkname)
 924                     os.symlink(sigfile_path, siglinkname)
 925
 926     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 927     if options.pretty:
 928         output = doc.toprettyxml()
 929     else:
 930         output = doc.toxml()
 931     of.write(output)
 932     of.close()
 933
 934     if 'repo_keyalias' in config:
 935
 936         if options.nosign:
 937             logging.info("Creating unsigned index in preparation for signing")
 938         else:
 939             logging.info("Creating signed index with this key (SHA256):")
 940             logging.info("%s" % repo_pubkey_fingerprint)
 941
 942         # Create a jar of the index...
 943         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
 944         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 945         if p.returncode != 0:
 946             logging.critical("Failed to create {0}".format(jar_output))
 947             sys.exit(1)
 948
 949         # Sign the index...
 950         signed = os.path.join(repodir, 'index.jar')
 951         if options.nosign:
 952             # Remove old signed index if not signing
 953             if os.path.exists(signed):
 954                 os.remove(signed)
 955         else:
 956             args = ['jarsigner', '-keystore', config['keystore'],
 957                     '-storepass:file', config['keystorepassfile'],
 958                     '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 959                     signed, config['repo_keyalias']]
 960             if config['keystore'] == 'NONE':
 961                 args += config['smartcardoptions']
 962             else:  # smardcards never use -keypass
 963                 args += ['-keypass:file', config['keypassfile']]
 964             p = FDroidPopen(args)
 965             if p.returncode != 0:
 966                 logging.critical("Failed to sign index")
 967                 sys.exit(1)
 968
 969     # Copy the repo icon into the repo directory...
 970     icon_dir = os.path.join(repodir, 'icons')
 971     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 972     shutil.copyfile(config['repo_icon'], iconfilename)
 973
 974     # Write a category list in the repo to allow quick access...
 975     catdata = ''
 976     for cat in categories:
 977         catdata += cat + '\n'
 978     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 979     f.write(catdata)
 980     f.close()
 981
 982
 983 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 984
 985     for appid, app in apps.iteritems():
 986
 987         # Get a list of the apks for this app...
 988         apklist = []
 989         for apk in apks:
 990             if apk['id'] == appid:
 991                 apklist.append(apk)
 992
 993         # Sort the apk list into version order...
 994         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 995
 996         if app['Archive Policy']:
 997             keepversions = int(app['Archive Policy'][:-9])
 998         else:
 999             keepversions = defaultkeepversions
1000
1001         if len(apklist) > keepversions:
1002             for apk in apklist[keepversions:]:
1003                 logging.info("Moving " + apk['apkname'] + " to archive")
1004                 shutil.move(os.path.join(repodir, apk['apkname']),
1005                             os.path.join(archivedir, apk['apkname']))
1006                 if 'srcname' in apk:
1007                     shutil.move(os.path.join(repodir, apk['srcname']),
1008                                 os.path.join(archivedir, apk['srcname']))
1009                     # Move GPG signature too...
1010                     sigfile = apk['srcname'] + '.asc'
1011                     sigsrc = os.path.join(repodir, sigfile)
1012                     if os.path.exists(sigsrc):
1013                         shutil.move(sigsrc, os.path.join(archivedir, sigfile))
1014
1015                 archapks.append(apk)
1016                 apks.remove(apk)
1017
1018
1019 def add_apks_to_per_app_repos(repodir, apks):
1020     apks_per_app = dict()
1021     for apk in apks:
1022         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1023         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1024         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1025         apks_per_app[apk['id']] = apk
1026
1027         if not os.path.exists(apk['per_app_icons']):
1028             logging.info('Adding new repo for only ' + apk['id'])
1029             os.makedirs(apk['per_app_icons'])
1030
1031         apkpath = os.path.join(repodir, apk['apkname'])
1032         shutil.copy(apkpath, apk['per_app_repo'])
1033         apksigpath = apkpath + '.sig'
1034         if os.path.exists(apksigpath):
1035             shutil.copy(apksigpath, apk['per_app_repo'])
1036         apkascpath = apkpath + '.asc'
1037         if os.path.exists(apkascpath):
1038             shutil.copy(apkascpath, apk['per_app_repo'])
1039
1040
1041 config = None
1042 options = None
1043
1044
1045 def main():
1046
1047     global config, options
1048
1049     # Parse command line...
1050     parser = OptionParser()
1051     parser.add_option("--create-key", action="store_true", default=False,
1052                       help="Create a repo signing key in a keystore")
1053     parser.add_option("-c", "--create-metadata", action="store_true", default=False,
1054                       help="Create skeleton metadata files that are missing")
1055     parser.add_option("--delete-unknown", action="store_true", default=False,
1056                       help="Delete APKs without metadata from the repo")
1057     parser.add_option("-v", "--verbose", action="store_true", default=False,
1058                       help="Spew out even more information than normal")
1059     parser.add_option("-q", "--quiet", action="store_true", default=False,
1060                       help="Restrict output to warnings and errors")
1061     parser.add_option("-b", "--buildreport", action="store_true", default=False,
1062                       help="Report on build data status")
1063     parser.add_option("-i", "--interactive", default=False, action="store_true",
1064                       help="Interactively ask about things that need updating.")
1065     parser.add_option("-I", "--icons", action="store_true", default=False,
1066                       help="Resize all the icons exceeding the max pixel size and exit")
1067     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
1068                       help="Specify editor to use in interactive mode. Default " +
1069                       "is /etc/alternatives/editor")
1070     parser.add_option("-w", "--wiki", default=False, action="store_true",
1071                       help="Update the wiki")
1072     parser.add_option("", "--pretty", action="store_true", default=False,
1073                       help="Produce human-readable index.xml")
1074     parser.add_option("--clean", action="store_true", default=False,
1075                       help="Clean update - don't uses caches, reprocess all apks")
1076     parser.add_option("--nosign", action="store_true", default=False,
1077                       help="When configured for signed indexes, create only unsigned indexes at this stage")
1078     (options, args) = parser.parse_args()
1079
1080     config = common.read_config(options)
1081
1082     repodirs = ['repo']
1083     if config['archive_older'] != 0:
1084         repodirs.append('archive')
1085         if not os.path.exists('archive'):
1086             os.mkdir('archive')
1087
1088     if options.icons:
1089         resize_all_icons(repodirs)
1090         sys.exit(0)
1091
1092     # check that icons exist now, rather than fail at the end of `fdroid update`
1093     for k in ['repo_icon', 'archive_icon']:
1094         if k in config:
1095             if not os.path.exists(config[k]):
1096                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1097                 sys.exit(1)
1098
1099     # if the user asks to create a keystore, do it now, reusing whatever it can
1100     if options.create_key:
1101         if os.path.exists(config['keystore']):
1102             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1103             logging.critical("\t'" + config['keystore'] + "'")
1104             sys.exit(1)
1105
1106         if 'repo_keyalias' not in config:
1107             config['repo_keyalias'] = socket.getfqdn()
1108             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1109         if 'keydname' not in config:
1110             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1111             common.write_to_config(config, 'keydname', config['keydname'])
1112         if 'keystore' not in config:
1113             config['keystore'] = common.default_config.keystore
1114             common.write_to_config(config, 'keystore', config['keystore'])
1115
1116         password = common.genpassword()
1117         if 'keystorepass' not in config:
1118             config['keystorepass'] = password
1119             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1120         if 'keypass' not in config:
1121             config['keypass'] = password
1122             common.write_to_config(config, 'keypass', config['keypass'])
1123         common.genkeystore(config)
1124
1125     # Get all apps...
1126     apps = metadata.read_metadata()
1127
1128     # Generate a list of categories...
1129     categories = set()
1130     for app in apps.itervalues():
1131         categories.update(app['Categories'])
1132
1133     # Read known apks data (will be updated and written back when we've finished)
1134     knownapks = common.KnownApks()
1135
1136     # Gather information about all the apk files in the repo directory, using
1137     # cached data if possible.
1138     apkcachefile = os.path.join('tmp', 'apkcache')
1139     if not options.clean and os.path.exists(apkcachefile):
1140         with open(apkcachefile, 'rb') as cf:
1141             apkcache = pickle.load(cf)
1142     else:
1143         apkcache = {}
1144
1145     delete_disabled_builds(apps, apkcache, repodirs)
1146
1147     # Scan all apks in the main repo
1148     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks)
1149
1150     # Generate warnings for apk's with no metadata (or create skeleton
1151     # metadata files, if requested on the command line)
1152     newmetadata = False
1153     for apk in apks:
1154         if apk['id'] not in apps:
1155             if options.create_metadata:
1156                 if 'name' not in apk:
1157                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1158                     continue
1159                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1160                 f.write("License:Unknown\n")
1161                 f.write("Web Site:\n")
1162                 f.write("Source Code:\n")
1163                 f.write("Issue Tracker:\n")
1164                 f.write("Changelog:\n")
1165                 f.write("Summary:" + apk['name'] + "\n")
1166                 f.write("Description:\n")
1167                 f.write(apk['name'] + "\n")
1168                 f.write(".\n")
1169                 f.close()
1170                 logging.info("Generated skeleton metadata for " + apk['id'])
1171                 newmetadata = True
1172             else:
1173                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1174                 if options.delete_unknown:
1175                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1176                     rmf = os.path.join(repodirs[0], apk['apkname'])
1177                     if not os.path.exists(rmf):
1178                         logging.error("Could not find {0} to remove it".format(rmf))
1179                     else:
1180                         os.remove(rmf)
1181                 else:
1182                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1183
1184     # update the metadata with the newly created ones included
1185     if newmetadata:
1186         apps = metadata.read_metadata()
1187
1188     # Scan the archive repo for apks as well
1189     if len(repodirs) > 1:
1190         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1191         if cc:
1192             cachechanged = True
1193     else:
1194         archapks = []
1195
1196     # Some information from the apks needs to be applied up to the application
1197     # level. When doing this, we use the info from the most recent version's apk.
1198     # We deal with figuring out when the app was added and last updated at the
1199     # same time.
1200     for appid, app in apps.iteritems():
1201         bestver = 0
1202         added = None
1203         lastupdated = None
1204         for apk in apks + archapks:
1205             if apk['id'] == appid:
1206                 if apk['versioncode'] > bestver:
1207                     bestver = apk['versioncode']
1208                     bestapk = apk
1209
1210                 if 'added' in apk:
1211                     if not added or apk['added'] < added:
1212                         added = apk['added']
1213                     if not lastupdated or apk['added'] > lastupdated:
1214                         lastupdated = apk['added']
1215
1216         if added:
1217             app['added'] = added
1218         else:
1219             logging.warn("Don't know when " + appid + " was added")
1220         if lastupdated:
1221             app['lastupdated'] = lastupdated
1222         else:
1223             logging.warn("Don't know when " + appid + " was last updated")
1224
1225         if bestver == 0:
1226             if app['Name'] is None:
1227                 app['Name'] = app['Auto Name'] or appid
1228             app['icon'] = None
1229             logging.warn("Application " + appid + " has no packages")
1230         else:
1231             if app['Name'] is None:
1232                 app['Name'] = bestapk['name']
1233             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1234
1235     # Sort the app list by name, then the web site doesn't have to by default.
1236     # (we had to wait until we'd scanned the apks to do this, because mostly the
1237     # name comes from there!)
1238     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid]['Name'].upper())
1239
1240     # APKs are placed into multiple repos based on the app package, providing
1241     # per-app subscription feeds for nightly builds and things like it
1242     if config['per_app_repos']:
1243         add_apks_to_per_app_repos(repodirs[0], apks)
1244         for appid, app in apps.iteritems():
1245             repodir = os.path.join(appid, 'fdroid', 'repo')
1246             appdict = dict()
1247             appdict[appid] = app
1248             if os.path.isdir(repodir):
1249                 make_index(appdict, [appid], apks, repodir, False, categories)
1250             else:
1251                 logging.info('Skipping index generation for ' + appid)
1252         return
1253
1254     if len(repodirs) > 1:
1255         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1256
1257     # Make the index for the main repo...
1258     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1259
1260     # If there's an archive repo,  make the index for it. We already scanned it
1261     # earlier on.
1262     if len(repodirs) > 1:
1263         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1264
1265     if config['update_stats']:
1266
1267         # Update known apks info...
1268         knownapks.writeifchanged()
1269
1270         # Generate latest apps data for widget
1271         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1272             data = ''
1273             for line in file(os.path.join('stats', 'latestapps.txt')):
1274                 appid = line.rstrip()
1275                 data += appid + "\t"
1276                 app = apps[appid]
1277                 data += app['Name'] + "\t"
1278                 if app['icon'] is not None:
1279                     data += app['icon'] + "\t"
1280                 data += app['License'] + "\n"
1281             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1282             f.write(data)
1283             f.close()
1284
1285     if cachechanged:
1286         with open(apkcachefile, 'wb') as cf:
1287             pickle.dump(apkcache, cf)
1288
1289     # Update the wiki...
1290     if options.wiki:
1291         update_wiki(apps, sortedids, apks + archapks)
1292
1293     logging.info("Finished.")
1294
1295 if __name__ == "__main__":
1296     main()