fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import socket
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 from datetime import datetime, timedelta
  31 from xml.dom.minidom import Document
  32 from argparse import ArgumentParser
  33 import time
  34 from pyasn1.error import PyAsn1Error
  35 from pyasn1.codec.der import decoder, encoder
  36 from pyasn1_modules import rfc2315
  37 from hashlib import md5
  38 from binascii import hexlify, unhexlify
  39
  40 from PIL import Image
  41 import logging
  42
  43 import common
  44 import metadata
  45 from common import FDroidPopen, SdkToolsPopen
  46 from metadata import MetaDataException
  47
  48 screen_densities = ['640', '480', '320', '240', '160', '120']
  49
  50 all_screen_densities = ['0'] + screen_densities
  51
  52
  53 def dpi_to_px(density):
  54     return (int(density) * 48) / 160
  55
  56
  57 def px_to_dpi(px):
  58     return (int(px) * 160) / 48
  59
  60
  61 def get_icon_dir(repodir, density):
  62     if density == '0':
  63         return os.path.join(repodir, "icons")
  64     return os.path.join(repodir, "icons-%s" % density)
  65
  66
  67 def get_icon_dirs(repodir):
  68     for density in screen_densities:
  69         yield get_icon_dir(repodir, density)
  70
  71
  72 def get_all_icon_dirs(repodir):
  73     for density in all_screen_densities:
  74         yield get_icon_dir(repodir, density)
  75
  76
  77 def update_wiki(apps, sortedids, apks):
  78     """Update the wiki
  79
  80     :param apps: fully populated list of all applications
  81     :param apks: all apks, except...
  82     """
  83     logging.info("Updating wiki")
  84     wikicat = 'Apps'
  85     wikiredircat = 'App Redirects'
  86     import mwclient
  87     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  88                          path=config['wiki_path'])
  89     site.login(config['wiki_user'], config['wiki_password'])
  90     generated_pages = {}
  91     generated_redirects = {}
  92
  93     for appid in sortedids:
  94         app = apps[appid]
  95
  96         wikidata = ''
  97         if app.Disabled:
  98             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
  99         if app.AntiFeatures:
 100             for af in app.AntiFeatures:
 101                 wikidata += '{{AntiFeature|' + af + '}}\n'
 102         if app.RequiresRoot:
 103             requiresroot = 'Yes'
 104         else:
 105             requiresroot = 'No'
 106         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 107             appid,
 108             app.Name,
 109             time.strftime('%Y-%m-%d', app.added) if app.added else '',
 110             time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
 111             app.SourceCode,
 112             app.IssueTracker,
 113             app.WebSite,
 114             app.Changelog,
 115             app.Donate,
 116             app.FlattrID,
 117             app.Bitcoin,
 118             app.Litecoin,
 119             app.License,
 120             requiresroot,
 121             app.AuthorName,
 122             app.AuthorEmail)
 123
 124         if app.Provides:
 125             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 126
 127         wikidata += app.Summary
 128         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 129
 130         wikidata += "=Description=\n"
 131         wikidata += metadata.description_wiki(app.Description) + "\n"
 132
 133         wikidata += "=Maintainer Notes=\n"
 134         if app.MaintainerNotes:
 135             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 136         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 137
 138         # Get a list of all packages for this application...
 139         apklist = []
 140         gotcurrentver = False
 141         cantupdate = False
 142         buildfails = False
 143         for apk in apks:
 144             if apk['id'] == appid:
 145                 if str(apk['versioncode']) == app.CurrentVersionCode:
 146                     gotcurrentver = True
 147                 apklist.append(apk)
 148         # Include ones we can't build, as a special case...
 149         for build in app.builds:
 150             if build.disable:
 151                 if build.vercode == app.CurrentVersionCode:
 152                     cantupdate = True
 153                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 154                 apklist.append({'versioncode': int(build.vercode),
 155                                 'version': build.version,
 156                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 157                                 })
 158             else:
 159                 builtit = False
 160                 for apk in apklist:
 161                     if apk['versioncode'] == int(build.vercode):
 162                         builtit = True
 163                         break
 164                 if not builtit:
 165                     buildfails = True
 166                     apklist.append({'versioncode': int(build.vercode),
 167                                     'version': build.version,
 168                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
 169                                     })
 170         if app.CurrentVersionCode == '0':
 171             cantupdate = True
 172         # Sort with most recent first...
 173         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 174
 175         wikidata += "=Versions=\n"
 176         if len(apklist) == 0:
 177             wikidata += "We currently have no versions of this app available."
 178         elif not gotcurrentver:
 179             wikidata += "We don't have the current version of this app."
 180         else:
 181             wikidata += "We have the current version of this app."
 182         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 183         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 184         if len(app.NoSourceSince) > 0:
 185             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 186         if len(app.CurrentVersion) > 0:
 187             wikidata += "The current (recommended) version is " + app.CurrentVersion
 188             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 189         validapks = 0
 190         for apk in apklist:
 191             wikidata += "==" + apk['version'] + "==\n"
 192
 193             if 'buildproblem' in apk:
 194                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 195             else:
 196                 validapks += 1
 197                 wikidata += "This version is built and signed by "
 198                 if 'srcname' in apk:
 199                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 200                 else:
 201                     wikidata += "the original developer.\n\n"
 202             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 203
 204         wikidata += '\n[[Category:' + wikicat + ']]\n'
 205         if len(app.NoSourceSince) > 0:
 206             wikidata += '\n[[Category:Apps missing source code]]\n'
 207         if validapks == 0 and not app.Disabled:
 208             wikidata += '\n[[Category:Apps with no packages]]\n'
 209         if cantupdate and not app.Disabled:
 210             wikidata += "\n[[Category:Apps we can't update]]\n"
 211         if buildfails and not app.Disabled:
 212             wikidata += "\n[[Category:Apps with failing builds]]\n"
 213         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 214             wikidata += '\n[[Category:Apps to Update]]\n'
 215         if app.Disabled:
 216             wikidata += '\n[[Category:Apps that are disabled]]\n'
 217         if app.UpdateCheckMode == 'None' and not app.Disabled:
 218             wikidata += '\n[[Category:Apps with no update check]]\n'
 219         for appcat in app.Categories:
 220             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 221
 222         # We can't have underscores in the page name, even if they're in
 223         # the package ID, because MediaWiki messes with them...
 224         pagename = appid.replace('_', ' ')
 225
 226         # Drop a trailing newline, because mediawiki is going to drop it anyway
 227         # and it we don't we'll think the page has changed when it hasn't...
 228         if wikidata.endswith('\n'):
 229             wikidata = wikidata[:-1]
 230
 231         generated_pages[pagename] = wikidata
 232
 233         # Make a redirect from the name to the ID too, unless there's
 234         # already an existing page with the name and it isn't a redirect.
 235         noclobber = False
 236         apppagename = app.Name.replace('_', ' ')
 237         apppagename = apppagename.replace('{', '')
 238         apppagename = apppagename.replace('}', ' ')
 239         apppagename = apppagename.replace(':', ' ')
 240         # Drop double spaces caused mostly by replacing ':' above
 241         apppagename = apppagename.replace('  ', ' ')
 242         for expagename in site.allpages(prefix=apppagename,
 243                                         filterredir='nonredirects',
 244                                         generator=False):
 245             if expagename == apppagename:
 246                 noclobber = True
 247         # Another reason not to make the redirect page is if the app name
 248         # is the same as it's ID, because that will overwrite the real page
 249         # with an redirect to itself! (Although it seems like an odd
 250         # scenario this happens a lot, e.g. where there is metadata but no
 251         # builds or binaries to extract a name from.
 252         if apppagename == pagename:
 253             noclobber = True
 254         if not noclobber:
 255             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 256
 257     for tcat, genp in [(wikicat, generated_pages),
 258                        (wikiredircat, generated_redirects)]:
 259         catpages = site.Pages['Category:' + tcat]
 260         existingpages = []
 261         for page in catpages:
 262             existingpages.append(page.name)
 263             if page.name in genp:
 264                 pagetxt = page.edit()
 265                 if pagetxt != genp[page.name]:
 266                     logging.debug("Updating modified page " + page.name)
 267                     page.save(genp[page.name], summary='Auto-updated')
 268                 else:
 269                     logging.debug("Page " + page.name + " is unchanged")
 270             else:
 271                 logging.warn("Deleting page " + page.name)
 272                 page.delete('No longer published')
 273         for pagename, text in genp.items():
 274             logging.debug("Checking " + pagename)
 275             if pagename not in existingpages:
 276                 logging.debug("Creating page " + pagename)
 277                 try:
 278                     newpage = site.Pages[pagename]
 279                     newpage.save(text, summary='Auto-created')
 280                 except:
 281                     logging.error("...FAILED to create page '{0}'".format(pagename))
 282
 283     # Purge server cache to ensure counts are up to date
 284     site.pages['Repository Maintenance'].purge()
 285
 286
 287 def delete_disabled_builds(apps, apkcache, repodirs):
 288     """Delete disabled build outputs.
 289
 290     :param apps: list of all applications, as per metadata.read_metadata
 291     :param apkcache: current apk cache information
 292     :param repodirs: the repo directories to process
 293     """
 294     for appid, app in apps.iteritems():
 295         for build in app.builds:
 296             if not build.disable:
 297                 continue
 298             apkfilename = appid + '_' + str(build.vercode) + '.apk'
 299             iconfilename = "%s.%s.png" % (
 300                 appid,
 301                 build.vercode)
 302             for repodir in repodirs:
 303                 files = [
 304                     os.path.join(repodir, apkfilename),
 305                     os.path.join(repodir, apkfilename + '.asc'),
 306                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 307                 ]
 308                 for density in all_screen_densities:
 309                     repo_dir = get_icon_dir(repodir, density)
 310                     files.append(os.path.join(repo_dir, iconfilename))
 311
 312                 for f in files:
 313                     if os.path.exists(f):
 314                         logging.info("Deleting disabled build output " + f)
 315                         os.remove(f)
 316             if apkfilename in apkcache:
 317                 del apkcache[apkfilename]
 318
 319
 320 def resize_icon(iconpath, density):
 321
 322     if not os.path.isfile(iconpath):
 323         return
 324
 325     try:
 326         im = Image.open(iconpath)
 327         size = dpi_to_px(density)
 328
 329         if any(length > size for length in im.size):
 330             oldsize = im.size
 331             im.thumbnail((size, size), Image.ANTIALIAS)
 332             logging.debug("%s was too large at %s - new size is %s" % (
 333                 iconpath, oldsize, im.size))
 334             im.save(iconpath, "PNG")
 335
 336     except Exception as e:
 337         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 338
 339
 340 def resize_all_icons(repodirs):
 341     """Resize all icons that exceed the max size
 342
 343     :param repodirs: the repo directories to process
 344     """
 345     for repodir in repodirs:
 346         for density in screen_densities:
 347             icon_dir = get_icon_dir(repodir, density)
 348             icon_glob = os.path.join(icon_dir, '*.png')
 349             for iconpath in glob.glob(icon_glob):
 350                 resize_icon(iconpath, density)
 351
 352
 353 # A signature block file with a .DSA, .RSA, or .EC extension
 354 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 355
 356
 357 def getsig(apkpath):
 358     """ Get the signing certificate of an apk. To get the same md5 has that
 359     Android gets, we encode the .RSA certificate in a specific format and pass
 360     it hex-encoded to the md5 digest algorithm.
 361
 362     :param apkpath: path to the apk
 363     :returns: A string containing the md5 of the signature of the apk or None
 364               if an error occurred.
 365     """
 366
 367     cert = None
 368
 369     # verify the jar signature is correct
 370     args = ['jarsigner', '-verify', apkpath]
 371     p = FDroidPopen(args)
 372     if p.returncode != 0:
 373         logging.critical(apkpath + " has a bad signature!")
 374         return None
 375
 376     with zipfile.ZipFile(apkpath, 'r') as apk:
 377
 378         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 379
 380         if len(certs) < 1:
 381             logging.error("Found no signing certificates on %s" % apkpath)
 382             return None
 383         if len(certs) > 1:
 384             logging.error("Found multiple signing certificates on %s" % apkpath)
 385             return None
 386
 387         cert = apk.read(certs[0])
 388
 389     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 390     if content.getComponentByName('contentType') != rfc2315.signedData:
 391         logging.error("Unexpected format.")
 392         return None
 393
 394     content = decoder.decode(content.getComponentByName('content'),
 395                              asn1Spec=rfc2315.SignedData())[0]
 396     try:
 397         certificates = content.getComponentByName('certificates')
 398     except PyAsn1Error:
 399         logging.error("Certificates not found.")
 400         return None
 401
 402     cert_encoded = encoder.encode(certificates)[4:]
 403
 404     return md5(cert_encoded.encode('hex')).hexdigest()
 405
 406
 407 def scan_apks(apps, apkcache, repodir, knownapks):
 408     """Scan the apks in the given repo directory.
 409
 410     This also extracts the icons.
 411
 412     :param apps: list of all applications, as per metadata.read_metadata
 413     :param apkcache: current apk cache information
 414     :param repodir: repo directory to scan
 415     :param knownapks: known apks info
 416     :returns: (apks, cachechanged) where apks is a list of apk information,
 417               and cachechanged is True if the apkcache got changed.
 418     """
 419
 420     cachechanged = False
 421
 422     for icon_dir in get_all_icon_dirs(repodir):
 423         if os.path.exists(icon_dir):
 424             if options.clean:
 425                 shutil.rmtree(icon_dir)
 426                 os.makedirs(icon_dir)
 427         else:
 428             os.makedirs(icon_dir)
 429
 430     apks = []
 431     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 432     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 433     vername_pat = re.compile(".*versionName='([^']*)'.*")
 434     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 435     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 436     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 437     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 438     string_pat = re.compile(".*'([^']*)'.*")
 439     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 440
 441         apkfilename = apkfile[len(repodir) + 1:]
 442         if ' ' in apkfilename:
 443             logging.critical("Spaces in filenames are not allowed.")
 444             sys.exit(1)
 445
 446         # Calculate the sha256...
 447         sha = hashlib.sha256()
 448         with open(apkfile, 'rb') as f:
 449             while True:
 450                 t = f.read(16384)
 451                 if len(t) == 0:
 452                     break
 453                 sha.update(t)
 454             shasum = sha.hexdigest()
 455
 456         usecache = False
 457         if apkfilename in apkcache:
 458             apk = apkcache[apkfilename]
 459             if apk['sha256'] == shasum:
 460                 logging.debug("Reading " + apkfilename + " from cache")
 461                 usecache = True
 462             else:
 463                 logging.debug("Ignoring stale cache data for " + apkfilename)
 464
 465         if not usecache:
 466             logging.debug("Processing " + apkfilename)
 467             apk = {}
 468             apk['apkname'] = apkfilename
 469             apk['sha256'] = shasum
 470             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 471             if os.path.exists(os.path.join(repodir, srcfilename)):
 472                 apk['srcname'] = srcfilename
 473             apk['size'] = os.path.getsize(apkfile)
 474             apk['permissions'] = set()
 475             apk['features'] = set()
 476             apk['icons_src'] = {}
 477             apk['icons'] = {}
 478             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 479             if p.returncode != 0:
 480                 if options.delete_unknown:
 481                     if os.path.exists(apkfile):
 482                         logging.error("Failed to get apk information, deleting " + apkfile)
 483                         os.remove(apkfile)
 484                     else:
 485                         logging.error("Could not find {0} to remove it".format(apkfile))
 486                 else:
 487                     logging.error("Failed to get apk information, skipping " + apkfile)
 488                 continue
 489             for line in p.output.splitlines():
 490                 if line.startswith("package:"):
 491                     try:
 492                         apk['id'] = re.match(name_pat, line).group(1)
 493                         apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
 494                         apk['version'] = re.match(vername_pat, line).group(1)
 495                     except Exception as e:
 496                         logging.error("Package matching failed: " + str(e))
 497                         logging.info("Line was: " + line)
 498                         sys.exit(1)
 499                 elif line.startswith("application:"):
 500                     apk['name'] = re.match(label_pat, line).group(1)
 501                     # Keep path to non-dpi icon in case we need it
 502                     match = re.match(icon_pat_nodpi, line)
 503                     if match:
 504                         apk['icons_src']['-1'] = match.group(1)
 505                 elif line.startswith("launchable-activity:"):
 506                     # Only use launchable-activity as fallback to application
 507                     if not apk['name']:
 508                         apk['name'] = re.match(label_pat, line).group(1)
 509                     if '-1' not in apk['icons_src']:
 510                         match = re.match(icon_pat_nodpi, line)
 511                         if match:
 512                             apk['icons_src']['-1'] = match.group(1)
 513                 elif line.startswith("application-icon-"):
 514                     match = re.match(icon_pat, line)
 515                     if match:
 516                         density = match.group(1)
 517                         path = match.group(2)
 518                         apk['icons_src'][density] = path
 519                 elif line.startswith("sdkVersion:"):
 520                     m = re.match(sdkversion_pat, line)
 521                     if m is None:
 522                         logging.error(line.replace('sdkVersion:', '')
 523                                       + ' is not a valid minSdkVersion!')
 524                     else:
 525                         apk['sdkversion'] = m.group(1)
 526                 elif line.startswith("maxSdkVersion:"):
 527                     apk['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 528                 elif line.startswith("native-code:"):
 529                     apk['nativecode'] = []
 530                     for arch in line[13:].split(' '):
 531                         apk['nativecode'].append(arch[1:-1])
 532                 elif line.startswith("uses-permission:"):
 533                     perm = re.match(string_pat, line).group(1)
 534                     if perm.startswith("android.permission."):
 535                         perm = perm[19:]
 536                     apk['permissions'].add(perm)
 537                 elif line.startswith("uses-feature:"):
 538                     perm = re.match(string_pat, line).group(1)
 539                     # Filter out this, it's only added with the latest SDK tools and
 540                     # causes problems for lots of apps.
 541                     if perm != "android.hardware.screen.portrait" \
 542                             and perm != "android.hardware.screen.landscape":
 543                         if perm.startswith("android.feature."):
 544                             perm = perm[16:]
 545                         apk['features'].add(perm)
 546
 547             if 'sdkversion' not in apk:
 548                 logging.warn("No SDK version information found in {0}".format(apkfile))
 549                 apk['sdkversion'] = 0
 550
 551             # Check for debuggable apks...
 552             if common.isApkDebuggable(apkfile, config):
 553                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 554
 555             # Get the signature (or md5 of, to be precise)...
 556             logging.debug('Getting signature of {0}'.format(apkfile))
 557             apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 558             if not apk['sig']:
 559                 logging.critical("Failed to get apk signature")
 560                 sys.exit(1)
 561
 562             apkzip = zipfile.ZipFile(apkfile, 'r')
 563
 564             # if an APK has files newer than the system time, suggest updating
 565             # the system clock.  This is useful for offline systems, used for
 566             # signing, which do not have another source of clock sync info. It
 567             # has to be more than 24 hours newer because ZIP/APK files do not
 568             # store timezone info
 569             manifest = apkzip.getinfo('AndroidManifest.xml')
 570             dt_obj = datetime(*manifest.date_time)
 571             checkdt = dt_obj - timedelta(1)
 572             if datetime.today() < checkdt:
 573                 logging.warn('System clock is older than manifest in: '
 574                              + apkfilename + '\nSet clock to that time using:\n'
 575                              + 'sudo date -s "' + str(dt_obj) + '"')
 576
 577             iconfilename = "%s.%s.png" % (
 578                 apk['id'],
 579                 apk['versioncode'])
 580
 581             # Extract the icon file...
 582             empty_densities = []
 583             for density in screen_densities:
 584                 if density not in apk['icons_src']:
 585                     empty_densities.append(density)
 586                     continue
 587                 iconsrc = apk['icons_src'][density]
 588                 icon_dir = get_icon_dir(repodir, density)
 589                 icondest = os.path.join(icon_dir, iconfilename)
 590
 591                 try:
 592                     with open(icondest, 'wb') as f:
 593                         f.write(apkzip.read(iconsrc))
 594                     apk['icons'][density] = iconfilename
 595
 596                 except:
 597                     logging.warn("Error retrieving icon file")
 598                     del apk['icons'][density]
 599                     del apk['icons_src'][density]
 600                     empty_densities.append(density)
 601
 602             if '-1' in apk['icons_src']:
 603                 iconsrc = apk['icons_src']['-1']
 604                 iconpath = os.path.join(
 605                     get_icon_dir(repodir, '0'), iconfilename)
 606                 with open(iconpath, 'wb') as f:
 607                     f.write(apkzip.read(iconsrc))
 608                 try:
 609                     im = Image.open(iconpath)
 610                     dpi = px_to_dpi(im.size[0])
 611                     for density in screen_densities:
 612                         if density in apk['icons']:
 613                             break
 614                         if density == screen_densities[-1] or dpi >= int(density):
 615                             apk['icons'][density] = iconfilename
 616                             shutil.move(iconpath,
 617                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 618                             empty_densities.remove(density)
 619                             break
 620                 except Exception as e:
 621                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 622
 623             if apk['icons']:
 624                 apk['icon'] = iconfilename
 625
 626             apkzip.close()
 627
 628             # First try resizing down to not lose quality
 629             last_density = None
 630             for density in screen_densities:
 631                 if density not in empty_densities:
 632                     last_density = density
 633                     continue
 634                 if last_density is None:
 635                     continue
 636                 logging.debug("Density %s not available, resizing down from %s"
 637                               % (density, last_density))
 638
 639                 last_iconpath = os.path.join(
 640                     get_icon_dir(repodir, last_density), iconfilename)
 641                 iconpath = os.path.join(
 642                     get_icon_dir(repodir, density), iconfilename)
 643                 try:
 644                     im = Image.open(last_iconpath)
 645                 except:
 646                     logging.warn("Invalid image file at %s" % last_iconpath)
 647                     continue
 648
 649                 size = dpi_to_px(density)
 650
 651                 im.thumbnail((size, size), Image.ANTIALIAS)
 652                 im.save(iconpath, "PNG")
 653                 empty_densities.remove(density)
 654
 655             # Then just copy from the highest resolution available
 656             last_density = None
 657             for density in reversed(screen_densities):
 658                 if density not in empty_densities:
 659                     last_density = density
 660                     continue
 661                 if last_density is None:
 662                     continue
 663                 logging.debug("Density %s not available, copying from lower density %s"
 664                               % (density, last_density))
 665
 666                 shutil.copyfile(
 667                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 668                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 669
 670                 empty_densities.remove(density)
 671
 672             for density in screen_densities:
 673                 icon_dir = get_icon_dir(repodir, density)
 674                 icondest = os.path.join(icon_dir, iconfilename)
 675                 resize_icon(icondest, density)
 676
 677             # Copy from icons-mdpi to icons since mdpi is the baseline density
 678             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 679             if os.path.isfile(baseline):
 680                 apk['icons']['0'] = iconfilename
 681                 shutil.copyfile(baseline,
 682                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 683
 684             # Record in known apks, getting the added date at the same time..
 685             added = knownapks.recordapk(apk['apkname'], apk['id'])
 686             if added:
 687                 apk['added'] = added
 688
 689             apkcache[apkfilename] = apk
 690             cachechanged = True
 691
 692         apks.append(apk)
 693
 694     return apks, cachechanged
 695
 696
 697 repo_pubkey_fingerprint = None
 698
 699
 700 # Generate a certificate fingerprint the same way keytool does it
 701 # (but with slightly different formatting)
 702 def cert_fingerprint(data):
 703     digest = hashlib.sha256(data).digest()
 704     ret = []
 705     ret.append(' '.join("%02X" % ord(b) for b in digest))
 706     return " ".join(ret)
 707
 708
 709 def extract_pubkey():
 710     global repo_pubkey_fingerprint
 711     if 'repo_pubkey' in config:
 712         pubkey = unhexlify(config['repo_pubkey'])
 713     else:
 714         p = FDroidPopen(['keytool', '-exportcert',
 715                          '-alias', config['repo_keyalias'],
 716                          '-keystore', config['keystore'],
 717                          '-storepass:file', config['keystorepassfile']]
 718                         + config['smartcardoptions'], output=False)
 719         if p.returncode != 0 or len(p.output) < 20:
 720             msg = "Failed to get repo pubkey!"
 721             if config['keystore'] == 'NONE':
 722                 msg += ' Is your crypto smartcard plugged in?'
 723             logging.critical(msg)
 724             sys.exit(1)
 725         pubkey = p.output
 726     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 727     return hexlify(pubkey)
 728
 729
 730 def make_index(apps, sortedids, apks, repodir, archive, categories):
 731     """Make a repo index.
 732
 733     :param apps: fully populated apps list
 734     :param apks: full populated apks list
 735     :param repodir: the repo directory
 736     :param archive: True if this is the archive repo, False if it's the
 737                     main one.
 738     :param categories: list of categories
 739     """
 740
 741     doc = Document()
 742
 743     def addElement(name, value, doc, parent):
 744         el = doc.createElement(name)
 745         el.appendChild(doc.createTextNode(value))
 746         parent.appendChild(el)
 747
 748     def addElementNonEmpty(name, value, doc, parent):
 749         if not value:
 750             return
 751         addElement(name, value, doc, parent)
 752
 753     def addElementCDATA(name, value, doc, parent):
 754         el = doc.createElement(name)
 755         el.appendChild(doc.createCDATASection(value))
 756         parent.appendChild(el)
 757
 758     root = doc.createElement("fdroid")
 759     doc.appendChild(root)
 760
 761     repoel = doc.createElement("repo")
 762
 763     if archive:
 764         repoel.setAttribute("name", config['archive_name'])
 765         if config['repo_maxage'] != 0:
 766             repoel.setAttribute("maxage", str(config['repo_maxage']))
 767         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 768         repoel.setAttribute("url", config['archive_url'])
 769         addElement('description', config['archive_description'], doc, repoel)
 770
 771     else:
 772         repoel.setAttribute("name", config['repo_name'])
 773         if config['repo_maxage'] != 0:
 774             repoel.setAttribute("maxage", str(config['repo_maxage']))
 775         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 776         repoel.setAttribute("url", config['repo_url'])
 777         addElement('description', config['repo_description'], doc, repoel)
 778
 779     repoel.setAttribute("version", "14")
 780     repoel.setAttribute("timestamp", str(int(time.time())))
 781
 782     nosigningkey = False
 783     if not options.nosign:
 784         if 'repo_keyalias' not in config:
 785             nosigningkey = True
 786             logging.critical("'repo_keyalias' not found in config.py!")
 787         if 'keystore' not in config:
 788             nosigningkey = True
 789             logging.critical("'keystore' not found in config.py!")
 790         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 791             nosigningkey = True
 792             logging.critical("'keystorepass' not found in config.py!")
 793         if 'keypass' not in config and 'keypassfile' not in config:
 794             nosigningkey = True
 795             logging.critical("'keypass' not found in config.py!")
 796         if not os.path.exists(config['keystore']):
 797             nosigningkey = True
 798             logging.critical("'" + config['keystore'] + "' does not exist!")
 799         if nosigningkey:
 800             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 801             logging.warning("\tfdroid update --create-key")
 802             sys.exit(1)
 803
 804     repoel.setAttribute("pubkey", extract_pubkey())
 805     root.appendChild(repoel)
 806
 807     for appid in sortedids:
 808         app = apps[appid]
 809
 810         if app.Disabled is not None:
 811             continue
 812
 813         # Get a list of the apks for this app...
 814         apklist = []
 815         for apk in apks:
 816             if apk['id'] == appid:
 817                 apklist.append(apk)
 818
 819         if len(apklist) == 0:
 820             continue
 821
 822         apel = doc.createElement("application")
 823         apel.setAttribute("id", app.id)
 824         root.appendChild(apel)
 825
 826         addElement('id', app.id, doc, apel)
 827         if app.added:
 828             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
 829         if app.lastupdated:
 830             addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
 831         addElement('name', app.Name, doc, apel)
 832         addElement('summary', app.Summary, doc, apel)
 833         if app.icon:
 834             addElement('icon', app.icon, doc, apel)
 835
 836         def linkres(appid):
 837             if appid in apps:
 838                 return ("fdroid.app:" + appid, apps[appid].Name)
 839             raise MetaDataException("Cannot resolve app id " + appid)
 840
 841         addElement('desc',
 842                    metadata.description_html(app.Description, linkres),
 843                    doc, apel)
 844         addElement('license', app.License, doc, apel)
 845         if app.Categories:
 846             addElement('categories', ','.join(app.Categories), doc, apel)
 847             # We put the first (primary) category in LAST, which will have
 848             # the desired effect of making clients that only understand one
 849             # category see that one.
 850             addElement('category', app.Categories[0], doc, apel)
 851         addElement('web', app.WebSite, doc, apel)
 852         addElement('source', app.SourceCode, doc, apel)
 853         addElement('tracker', app.IssueTracker, doc, apel)
 854         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 855         addElementNonEmpty('author', app.AuthorName, doc, apel)
 856         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 857         addElementNonEmpty('donate', app.Donate, doc, apel)
 858         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 859         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 860         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 861
 862         # These elements actually refer to the current version (i.e. which
 863         # one is recommended. They are historically mis-named, and need
 864         # changing, but stay like this for now to support existing clients.
 865         addElement('marketversion', app.CurrentVersion, doc, apel)
 866         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 867
 868         if app.AntiFeatures:
 869             af = app.AntiFeatures
 870             if af:
 871                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 872         if app.Provides:
 873             pv = app.Provides.split(',')
 874             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 875         if app.RequiresRoot:
 876             addElement('requirements', 'root', doc, apel)
 877
 878         # Sort the apk list into version order, just so the web site
 879         # doesn't have to do any work by default...
 880         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 881
 882         # Check for duplicates - they will make the client unhappy...
 883         for i in range(len(apklist) - 1):
 884             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 885                 logging.critical("duplicate versions: '%s' - '%s'" % (
 886                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 887                 sys.exit(1)
 888
 889         current_version_code = 0
 890         current_version_file = None
 891         for apk in apklist:
 892             # find the APK for the "Current Version"
 893             if current_version_code < apk['versioncode']:
 894                 current_version_code = apk['versioncode']
 895             if current_version_code < int(app.CurrentVersionCode):
 896                 current_version_file = apk['apkname']
 897
 898             apkel = doc.createElement("package")
 899             apel.appendChild(apkel)
 900             addElement('version', apk['version'], doc, apkel)
 901             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 902             addElement('apkname', apk['apkname'], doc, apkel)
 903             if 'srcname' in apk:
 904                 addElement('srcname', apk['srcname'], doc, apkel)
 905             for hash_type in ['sha256']:
 906                 if hash_type not in apk:
 907                     continue
 908                 hashel = doc.createElement("hash")
 909                 hashel.setAttribute("type", hash_type)
 910                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 911                 apkel.appendChild(hashel)
 912             addElement('sig', apk['sig'], doc, apkel)
 913             addElement('size', str(apk['size']), doc, apkel)
 914             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 915             if 'maxsdkversion' in apk:
 916                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 917             if 'added' in apk:
 918                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 919             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
 920             if 'nativecode' in apk:
 921                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 922             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
 923
 924         if current_version_file is not None \
 925                 and config['make_current_version_link'] \
 926                 and repodir == 'repo':  # only create these
 927             namefield = config['current_version_name_source']
 928             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
 929             apklinkname = sanitized_name + '.apk'
 930             current_version_path = os.path.join(repodir, current_version_file)
 931             if os.path.islink(apklinkname):
 932                 os.remove(apklinkname)
 933             os.symlink(current_version_path, apklinkname)
 934             # also symlink gpg signature, if it exists
 935             for extension in ('.asc', '.sig'):
 936                 sigfile_path = current_version_path + extension
 937                 if os.path.exists(sigfile_path):
 938                     siglinkname = apklinkname + extension
 939                     if os.path.islink(siglinkname):
 940                         os.remove(siglinkname)
 941                     os.symlink(sigfile_path, siglinkname)
 942
 943     if options.pretty:
 944         output = doc.toprettyxml()
 945     else:
 946         output = doc.toxml()
 947
 948     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 949         f.write(output)
 950
 951     if 'repo_keyalias' in config:
 952
 953         if options.nosign:
 954             logging.info("Creating unsigned index in preparation for signing")
 955         else:
 956             logging.info("Creating signed index with this key (SHA256):")
 957             logging.info("%s" % repo_pubkey_fingerprint)
 958
 959         # Create a jar of the index...
 960         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
 961         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 962         if p.returncode != 0:
 963             logging.critical("Failed to create {0}".format(jar_output))
 964             sys.exit(1)
 965
 966         # Sign the index...
 967         signed = os.path.join(repodir, 'index.jar')
 968         if options.nosign:
 969             # Remove old signed index if not signing
 970             if os.path.exists(signed):
 971                 os.remove(signed)
 972         else:
 973             args = ['jarsigner', '-keystore', config['keystore'],
 974                     '-storepass:file', config['keystorepassfile'],
 975                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
 976                     signed, config['repo_keyalias']]
 977             if config['keystore'] == 'NONE':
 978                 args += config['smartcardoptions']
 979             else:  # smardcards never use -keypass
 980                 args += ['-keypass:file', config['keypassfile']]
 981             p = FDroidPopen(args)
 982             if p.returncode != 0:
 983                 logging.critical("Failed to sign index")
 984                 sys.exit(1)
 985
 986     # Copy the repo icon into the repo directory...
 987     icon_dir = os.path.join(repodir, 'icons')
 988     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 989     shutil.copyfile(config['repo_icon'], iconfilename)
 990
 991     # Write a category list in the repo to allow quick access...
 992     catdata = ''
 993     for cat in categories:
 994         catdata += cat + '\n'
 995     with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
 996         f.write(catdata)
 997
 998
 999 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1000
1001     for appid, app in apps.iteritems():
1002
1003         if app.ArchivePolicy:
1004             keepversions = int(app.ArchivePolicy[:-9])
1005         else:
1006             keepversions = defaultkeepversions
1007
1008         def filter_apk_list_sorted(apk_list):
1009             res = []
1010             for apk in apk_list:
1011                 if apk['id'] == appid:
1012                     res.append(apk)
1013
1014             # Sort the apk list by version code. First is highest/newest.
1015             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1016
1017         def move_file(from_dir, to_dir, filename, ignore_missing):
1018             from_path = os.path.join(from_dir, filename)
1019             if ignore_missing and not os.path.exists(from_path):
1020                 return
1021             to_path = os.path.join(to_dir, filename)
1022             shutil.move(from_path, to_path)
1023
1024         if len(apks) > keepversions:
1025             apklist = filter_apk_list_sorted(apks)
1026             # Move back the ones we don't want.
1027             for apk in apklist[keepversions:]:
1028                 logging.info("Moving " + apk['apkname'] + " to archive")
1029                 move_file(repodir, archivedir, apk['apkname'], False)
1030                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1031                 for density in all_screen_densities:
1032                     repo_icon_dir = get_icon_dir(repodir, density)
1033                     archive_icon_dir = get_icon_dir(archivedir, density)
1034                     if density not in apk['icons']:
1035                         continue
1036                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1037                 if 'srcname' in apk:
1038                     move_file(repodir, archivedir, apk['srcname'], False)
1039                 archapks.append(apk)
1040                 apks.remove(apk)
1041         elif len(apks) < keepversions and len(archapks) > 0:
1042             required = keepversions - len(apks)
1043             archapklist = filter_apk_list_sorted(archapks)
1044             # Move forward the ones we want again.
1045             for apk in archapklist[:required]:
1046                 logging.info("Moving " + apk['apkname'] + " from archive")
1047                 move_file(archivedir, repodir, apk['apkname'], False)
1048                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1049                 for density in all_screen_densities:
1050                     repo_icon_dir = get_icon_dir(repodir, density)
1051                     archive_icon_dir = get_icon_dir(archivedir, density)
1052                     if density not in apk['icons']:
1053                         continue
1054                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1055                 if 'srcname' in apk:
1056                     move_file(archivedir, repodir, apk['srcname'], False)
1057                 archapks.remove(apk)
1058                 apks.append(apk)
1059
1060
1061 def add_apks_to_per_app_repos(repodir, apks):
1062     apks_per_app = dict()
1063     for apk in apks:
1064         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1065         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1066         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1067         apks_per_app[apk['id']] = apk
1068
1069         if not os.path.exists(apk['per_app_icons']):
1070             logging.info('Adding new repo for only ' + apk['id'])
1071             os.makedirs(apk['per_app_icons'])
1072
1073         apkpath = os.path.join(repodir, apk['apkname'])
1074         shutil.copy(apkpath, apk['per_app_repo'])
1075         apksigpath = apkpath + '.sig'
1076         if os.path.exists(apksigpath):
1077             shutil.copy(apksigpath, apk['per_app_repo'])
1078         apkascpath = apkpath + '.asc'
1079         if os.path.exists(apkascpath):
1080             shutil.copy(apkascpath, apk['per_app_repo'])
1081
1082
1083 config = None
1084 options = None
1085
1086
1087 def main():
1088
1089     global config, options
1090
1091     # Parse command line...
1092     parser = ArgumentParser()
1093     common.setup_global_opts(parser)
1094     parser.add_argument("--create-key", action="store_true", default=False,
1095                         help="Create a repo signing key in a keystore")
1096     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1097                         help="Create skeleton metadata files that are missing")
1098     parser.add_argument("--delete-unknown", action="store_true", default=False,
1099                         help="Delete APKs without metadata from the repo")
1100     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1101                         help="Report on build data status")
1102     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1103                         help="Interactively ask about things that need updating.")
1104     parser.add_argument("-I", "--icons", action="store_true", default=False,
1105                         help="Resize all the icons exceeding the max pixel size and exit")
1106     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1107                         help="Specify editor to use in interactive mode. Default " +
1108                         "is /etc/alternatives/editor")
1109     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1110                         help="Update the wiki")
1111     parser.add_argument("--pretty", action="store_true", default=False,
1112                         help="Produce human-readable index.xml")
1113     parser.add_argument("--clean", action="store_true", default=False,
1114                         help="Clean update - don't uses caches, reprocess all apks")
1115     parser.add_argument("--nosign", action="store_true", default=False,
1116                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1117     options = parser.parse_args()
1118
1119     config = common.read_config(options)
1120
1121     repodirs = ['repo']
1122     if config['archive_older'] != 0:
1123         repodirs.append('archive')
1124         if not os.path.exists('archive'):
1125             os.mkdir('archive')
1126
1127     if options.icons:
1128         resize_all_icons(repodirs)
1129         sys.exit(0)
1130
1131     # check that icons exist now, rather than fail at the end of `fdroid update`
1132     for k in ['repo_icon', 'archive_icon']:
1133         if k in config:
1134             if not os.path.exists(config[k]):
1135                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1136                 sys.exit(1)
1137
1138     # if the user asks to create a keystore, do it now, reusing whatever it can
1139     if options.create_key:
1140         if os.path.exists(config['keystore']):
1141             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1142             logging.critical("\t'" + config['keystore'] + "'")
1143             sys.exit(1)
1144
1145         if 'repo_keyalias' not in config:
1146             config['repo_keyalias'] = socket.getfqdn()
1147             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1148         if 'keydname' not in config:
1149             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1150             common.write_to_config(config, 'keydname', config['keydname'])
1151         if 'keystore' not in config:
1152             config['keystore'] = common.default_config.keystore
1153             common.write_to_config(config, 'keystore', config['keystore'])
1154
1155         password = common.genpassword()
1156         if 'keystorepass' not in config:
1157             config['keystorepass'] = password
1158             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1159         if 'keypass' not in config:
1160             config['keypass'] = password
1161             common.write_to_config(config, 'keypass', config['keypass'])
1162         common.genkeystore(config)
1163
1164     # Get all apps...
1165     apps = metadata.read_metadata()
1166
1167     # Generate a list of categories...
1168     categories = set()
1169     for app in apps.itervalues():
1170         categories.update(app.Categories)
1171
1172     # Read known apks data (will be updated and written back when we've finished)
1173     knownapks = common.KnownApks()
1174
1175     # Gather information about all the apk files in the repo directory, using
1176     # cached data if possible.
1177     apkcachefile = os.path.join('tmp', 'apkcache')
1178     if not options.clean and os.path.exists(apkcachefile):
1179         with open(apkcachefile, 'rb') as cf:
1180             apkcache = pickle.load(cf)
1181     else:
1182         apkcache = {}
1183
1184     delete_disabled_builds(apps, apkcache, repodirs)
1185
1186     # Scan all apks in the main repo
1187     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks)
1188
1189     # Generate warnings for apk's with no metadata (or create skeleton
1190     # metadata files, if requested on the command line)
1191     newmetadata = False
1192     for apk in apks:
1193         if apk['id'] not in apps:
1194             if options.create_metadata:
1195                 if 'name' not in apk:
1196                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1197                     continue
1198                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1199                 f.write("License:Unknown\n")
1200                 f.write("Web Site:\n")
1201                 f.write("Source Code:\n")
1202                 f.write("Issue Tracker:\n")
1203                 f.write("Changelog:\n")
1204                 f.write("Summary:" + apk['name'] + "\n")
1205                 f.write("Description:\n")
1206                 f.write(apk['name'] + "\n")
1207                 f.write(".\n")
1208                 f.close()
1209                 logging.info("Generated skeleton metadata for " + apk['id'])
1210                 newmetadata = True
1211             else:
1212                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1213                 if options.delete_unknown:
1214                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1215                     rmf = os.path.join(repodirs[0], apk['apkname'])
1216                     if not os.path.exists(rmf):
1217                         logging.error("Could not find {0} to remove it".format(rmf))
1218                     else:
1219                         os.remove(rmf)
1220                 else:
1221                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1222
1223     # update the metadata with the newly created ones included
1224     if newmetadata:
1225         apps = metadata.read_metadata()
1226
1227     # Scan the archive repo for apks as well
1228     if len(repodirs) > 1:
1229         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1230         if cc:
1231             cachechanged = True
1232     else:
1233         archapks = []
1234
1235     # Some information from the apks needs to be applied up to the application
1236     # level. When doing this, we use the info from the most recent version's apk.
1237     # We deal with figuring out when the app was added and last updated at the
1238     # same time.
1239     for appid, app in apps.iteritems():
1240         bestver = 0
1241         for apk in apks + archapks:
1242             if apk['id'] == appid:
1243                 if apk['versioncode'] > bestver:
1244                     bestver = apk['versioncode']
1245                     bestapk = apk
1246
1247                 if 'added' in apk:
1248                     if not app.added or apk['added'] < app.added:
1249                         app.added = apk['added']
1250                     if not app.lastupdated or apk['added'] > app.lastupdated:
1251                         app.lastupdated = apk['added']
1252
1253         if not app.added:
1254             logging.debug("Don't know when " + appid + " was added")
1255         if not app.lastupdated:
1256             logging.debug("Don't know when " + appid + " was last updated")
1257
1258         if bestver == 0:
1259             if app.Name is None:
1260                 app.Name = app.AutoName or appid
1261             app.icon = None
1262             logging.debug("Application " + appid + " has no packages")
1263         else:
1264             if app.Name is None:
1265                 app.Name = bestapk['name']
1266             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1267
1268     # Sort the app list by name, then the web site doesn't have to by default.
1269     # (we had to wait until we'd scanned the apks to do this, because mostly the
1270     # name comes from there!)
1271     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid].Name.upper())
1272
1273     # APKs are placed into multiple repos based on the app package, providing
1274     # per-app subscription feeds for nightly builds and things like it
1275     if config['per_app_repos']:
1276         add_apks_to_per_app_repos(repodirs[0], apks)
1277         for appid, app in apps.iteritems():
1278             repodir = os.path.join(appid, 'fdroid', 'repo')
1279             appdict = dict()
1280             appdict[appid] = app
1281             if os.path.isdir(repodir):
1282                 make_index(appdict, [appid], apks, repodir, False, categories)
1283             else:
1284                 logging.info('Skipping index generation for ' + appid)
1285         return
1286
1287     if len(repodirs) > 1:
1288         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1289
1290     # Make the index for the main repo...
1291     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1292
1293     # If there's an archive repo,  make the index for it. We already scanned it
1294     # earlier on.
1295     if len(repodirs) > 1:
1296         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1297
1298     if config['update_stats']:
1299
1300         # Update known apks info...
1301         knownapks.writeifchanged()
1302
1303         # Generate latest apps data for widget
1304         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1305             data = ''
1306             for line in file(os.path.join('stats', 'latestapps.txt')):
1307                 appid = line.rstrip()
1308                 data += appid + "\t"
1309                 app = apps[appid]
1310                 data += app.Name + "\t"
1311                 if app.icon is not None:
1312                     data += app.icon + "\t"
1313                 data += app.License + "\n"
1314             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
1315                 f.write(data)
1316
1317     if cachechanged:
1318         with open(apkcachefile, 'wb') as cf:
1319             pickle.dump(apkcache, cf)
1320
1321     # Update the wiki...
1322     if options.wiki:
1323         update_wiki(apps, sortedids, apks + archapks)
1324
1325     logging.info("Finished.")
1326
1327 if __name__ == "__main__":
1328     main()