fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import socket
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 import urlparse
  31 from datetime import datetime, timedelta
  32 from xml.dom.minidom import Document
  33 from argparse import ArgumentParser
  34 import time
  35 from pyasn1.error import PyAsn1Error
  36 from pyasn1.codec.der import decoder, encoder
  37 from pyasn1_modules import rfc2315
  38 from hashlib import md5
  39 from binascii import hexlify, unhexlify
  40
  41 from PIL import Image
  42 import logging
  43
  44 import common
  45 import metadata
  46 from common import FDroidPopen, SdkToolsPopen
  47 from metadata import MetaDataException
  48
  49 screen_densities = ['640', '480', '320', '240', '160', '120']
  50
  51 all_screen_densities = ['0'] + screen_densities
  52
  53
  54 def dpi_to_px(density):
  55     return (int(density) * 48) / 160
  56
  57
  58 def px_to_dpi(px):
  59     return (int(px) * 160) / 48
  60
  61
  62 def get_icon_dir(repodir, density):
  63     if density == '0':
  64         return os.path.join(repodir, "icons")
  65     return os.path.join(repodir, "icons-%s" % density)
  66
  67
  68 def get_icon_dirs(repodir):
  69     for density in screen_densities:
  70         yield get_icon_dir(repodir, density)
  71
  72
  73 def get_all_icon_dirs(repodir):
  74     for density in all_screen_densities:
  75         yield get_icon_dir(repodir, density)
  76
  77
  78 def update_wiki(apps, sortedids, apks):
  79     """Update the wiki
  80
  81     :param apps: fully populated list of all applications
  82     :param apks: all apks, except...
  83     """
  84     logging.info("Updating wiki")
  85     wikicat = 'Apps'
  86     wikiredircat = 'App Redirects'
  87     import mwclient
  88     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  89                          path=config['wiki_path'])
  90     site.login(config['wiki_user'], config['wiki_password'])
  91     generated_pages = {}
  92     generated_redirects = {}
  93
  94     for appid in sortedids:
  95         app = apps[appid]
  96
  97         wikidata = ''
  98         if app.Disabled:
  99             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 100         if app.AntiFeatures:
 101             for af in app.AntiFeatures:
 102                 wikidata += '{{AntiFeature|' + af + '}}\n'
 103         if app.RequiresRoot:
 104             requiresroot = 'Yes'
 105         else:
 106             requiresroot = 'No'
 107         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 108             appid,
 109             app.Name,
 110             time.strftime('%Y-%m-%d', app.added) if app.added else '',
 111             time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
 112             app.SourceCode,
 113             app.IssueTracker,
 114             app.WebSite,
 115             app.Changelog,
 116             app.Donate,
 117             app.FlattrID,
 118             app.Bitcoin,
 119             app.Litecoin,
 120             app.License,
 121             requiresroot,
 122             app.AuthorName,
 123             app.AuthorEmail)
 124
 125         if app.Provides:
 126             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 127
 128         wikidata += app.Summary
 129         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 130
 131         wikidata += "=Description=\n"
 132         wikidata += metadata.description_wiki(app.Description) + "\n"
 133
 134         wikidata += "=Maintainer Notes=\n"
 135         if app.MaintainerNotes:
 136             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 137         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 138
 139         # Get a list of all packages for this application...
 140         apklist = []
 141         gotcurrentver = False
 142         cantupdate = False
 143         buildfails = False
 144         for apk in apks:
 145             if apk['id'] == appid:
 146                 if str(apk['versioncode']) == app.CurrentVersionCode:
 147                     gotcurrentver = True
 148                 apklist.append(apk)
 149         # Include ones we can't build, as a special case...
 150         for build in app.builds:
 151             if build.disable:
 152                 if build.vercode == app.CurrentVersionCode:
 153                     cantupdate = True
 154                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 155                 apklist.append({'versioncode': int(build.vercode),
 156                                 'version': build.version,
 157                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 158                                 })
 159             else:
 160                 builtit = False
 161                 for apk in apklist:
 162                     if apk['versioncode'] == int(build.vercode):
 163                         builtit = True
 164                         break
 165                 if not builtit:
 166                     buildfails = True
 167                     apklist.append({'versioncode': int(build.vercode),
 168                                     'version': build.version,
 169                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
 170                                     })
 171         if app.CurrentVersionCode == '0':
 172             cantupdate = True
 173         # Sort with most recent first...
 174         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 175
 176         wikidata += "=Versions=\n"
 177         if len(apklist) == 0:
 178             wikidata += "We currently have no versions of this app available."
 179         elif not gotcurrentver:
 180             wikidata += "We don't have the current version of this app."
 181         else:
 182             wikidata += "We have the current version of this app."
 183         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 184         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 185         if len(app.NoSourceSince) > 0:
 186             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 187         if len(app.CurrentVersion) > 0:
 188             wikidata += "The current (recommended) version is " + app.CurrentVersion
 189             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 190         validapks = 0
 191         for apk in apklist:
 192             wikidata += "==" + apk['version'] + "==\n"
 193
 194             if 'buildproblem' in apk:
 195                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 196             else:
 197                 validapks += 1
 198                 wikidata += "This version is built and signed by "
 199                 if 'srcname' in apk:
 200                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 201                 else:
 202                     wikidata += "the original developer.\n\n"
 203             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 204
 205         wikidata += '\n[[Category:' + wikicat + ']]\n'
 206         if len(app.NoSourceSince) > 0:
 207             wikidata += '\n[[Category:Apps missing source code]]\n'
 208         if validapks == 0 and not app.Disabled:
 209             wikidata += '\n[[Category:Apps with no packages]]\n'
 210         if cantupdate and not app.Disabled:
 211             wikidata += "\n[[Category:Apps we cannot update]]\n"
 212         if buildfails and not app.Disabled:
 213             wikidata += "\n[[Category:Apps with failing builds]]\n"
 214         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 215             wikidata += '\n[[Category:Apps to Update]]\n'
 216         if app.Disabled:
 217             wikidata += '\n[[Category:Apps that are disabled]]\n'
 218         if app.UpdateCheckMode == 'None' and not app.Disabled:
 219             wikidata += '\n[[Category:Apps with no update check]]\n'
 220         for appcat in app.Categories:
 221             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 222
 223         # We can't have underscores in the page name, even if they're in
 224         # the package ID, because MediaWiki messes with them...
 225         pagename = appid.replace('_', ' ')
 226
 227         # Drop a trailing newline, because mediawiki is going to drop it anyway
 228         # and it we don't we'll think the page has changed when it hasn't...
 229         if wikidata.endswith('\n'):
 230             wikidata = wikidata[:-1]
 231
 232         generated_pages[pagename] = wikidata
 233
 234         # Make a redirect from the name to the ID too, unless there's
 235         # already an existing page with the name and it isn't a redirect.
 236         noclobber = False
 237         apppagename = app.Name.replace('_', ' ')
 238         apppagename = apppagename.replace('{', '')
 239         apppagename = apppagename.replace('}', ' ')
 240         apppagename = apppagename.replace(':', ' ')
 241         # Drop double spaces caused mostly by replacing ':' above
 242         apppagename = apppagename.replace('  ', ' ')
 243         for expagename in site.allpages(prefix=apppagename,
 244                                         filterredir='nonredirects',
 245                                         generator=False):
 246             if expagename == apppagename:
 247                 noclobber = True
 248         # Another reason not to make the redirect page is if the app name
 249         # is the same as it's ID, because that will overwrite the real page
 250         # with an redirect to itself! (Although it seems like an odd
 251         # scenario this happens a lot, e.g. where there is metadata but no
 252         # builds or binaries to extract a name from.
 253         if apppagename == pagename:
 254             noclobber = True
 255         if not noclobber:
 256             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 257
 258     for tcat, genp in [(wikicat, generated_pages),
 259                        (wikiredircat, generated_redirects)]:
 260         catpages = site.Pages['Category:' + tcat]
 261         existingpages = []
 262         for page in catpages:
 263             existingpages.append(page.name)
 264             if page.name in genp:
 265                 pagetxt = page.edit()
 266                 if pagetxt != genp[page.name]:
 267                     logging.debug("Updating modified page " + page.name)
 268                     page.save(genp[page.name], summary='Auto-updated')
 269                 else:
 270                     logging.debug("Page " + page.name + " is unchanged")
 271             else:
 272                 logging.warn("Deleting page " + page.name)
 273                 page.delete('No longer published')
 274         for pagename, text in genp.items():
 275             logging.debug("Checking " + pagename)
 276             if pagename not in existingpages:
 277                 logging.debug("Creating page " + pagename)
 278                 try:
 279                     newpage = site.Pages[pagename]
 280                     newpage.save(text, summary='Auto-created')
 281                 except:
 282                     logging.error("...FAILED to create page '{0}'".format(pagename))
 283
 284     # Purge server cache to ensure counts are up to date
 285     site.pages['Repository Maintenance'].purge()
 286
 287
 288 def delete_disabled_builds(apps, apkcache, repodirs):
 289     """Delete disabled build outputs.
 290
 291     :param apps: list of all applications, as per metadata.read_metadata
 292     :param apkcache: current apk cache information
 293     :param repodirs: the repo directories to process
 294     """
 295     for appid, app in apps.iteritems():
 296         for build in app.builds:
 297             if not build.disable:
 298                 continue
 299             apkfilename = appid + '_' + str(build.vercode) + '.apk'
 300             iconfilename = "%s.%s.png" % (
 301                 appid,
 302                 build.vercode)
 303             for repodir in repodirs:
 304                 files = [
 305                     os.path.join(repodir, apkfilename),
 306                     os.path.join(repodir, apkfilename + '.asc'),
 307                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 308                 ]
 309                 for density in all_screen_densities:
 310                     repo_dir = get_icon_dir(repodir, density)
 311                     files.append(os.path.join(repo_dir, iconfilename))
 312
 313                 for f in files:
 314                     if os.path.exists(f):
 315                         logging.info("Deleting disabled build output " + f)
 316                         os.remove(f)
 317             if apkfilename in apkcache:
 318                 del apkcache[apkfilename]
 319
 320
 321 def resize_icon(iconpath, density):
 322
 323     if not os.path.isfile(iconpath):
 324         return
 325
 326     try:
 327         im = Image.open(iconpath)
 328         size = dpi_to_px(density)
 329
 330         if any(length > size for length in im.size):
 331             oldsize = im.size
 332             im.thumbnail((size, size), Image.ANTIALIAS)
 333             logging.debug("%s was too large at %s - new size is %s" % (
 334                 iconpath, oldsize, im.size))
 335             im.save(iconpath, "PNG")
 336
 337     except Exception as e:
 338         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 339
 340
 341 def resize_all_icons(repodirs):
 342     """Resize all icons that exceed the max size
 343
 344     :param repodirs: the repo directories to process
 345     """
 346     for repodir in repodirs:
 347         for density in screen_densities:
 348             icon_dir = get_icon_dir(repodir, density)
 349             icon_glob = os.path.join(icon_dir, '*.png')
 350             for iconpath in glob.glob(icon_glob):
 351                 resize_icon(iconpath, density)
 352
 353
 354 # A signature block file with a .DSA, .RSA, or .EC extension
 355 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 356
 357
 358 def getsig(apkpath):
 359     """ Get the signing certificate of an apk. To get the same md5 has that
 360     Android gets, we encode the .RSA certificate in a specific format and pass
 361     it hex-encoded to the md5 digest algorithm.
 362
 363     :param apkpath: path to the apk
 364     :returns: A string containing the md5 of the signature of the apk or None
 365               if an error occurred.
 366     """
 367
 368     cert = None
 369
 370     # verify the jar signature is correct
 371     args = [config['jarsigner'], '-verify', apkpath]
 372     p = FDroidPopen(args)
 373     if p.returncode != 0:
 374         logging.critical(apkpath + " has a bad signature!")
 375         return None
 376
 377     with zipfile.ZipFile(apkpath, 'r') as apk:
 378
 379         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 380
 381         if len(certs) < 1:
 382             logging.error("Found no signing certificates on %s" % apkpath)
 383             return None
 384         if len(certs) > 1:
 385             logging.error("Found multiple signing certificates on %s" % apkpath)
 386             return None
 387
 388         cert = apk.read(certs[0])
 389
 390     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 391     if content.getComponentByName('contentType') != rfc2315.signedData:
 392         logging.error("Unexpected format.")
 393         return None
 394
 395     content = decoder.decode(content.getComponentByName('content'),
 396                              asn1Spec=rfc2315.SignedData())[0]
 397     try:
 398         certificates = content.getComponentByName('certificates')
 399     except PyAsn1Error:
 400         logging.error("Certificates not found.")
 401         return None
 402
 403     cert_encoded = encoder.encode(certificates)[4:]
 404
 405     return md5(cert_encoded.encode('hex')).hexdigest()
 406
 407
 408 def scan_apks(apps, apkcache, repodir, knownapks):
 409     """Scan the apks in the given repo directory.
 410
 411     This also extracts the icons.
 412
 413     :param apps: list of all applications, as per metadata.read_metadata
 414     :param apkcache: current apk cache information
 415     :param repodir: repo directory to scan
 416     :param knownapks: known apks info
 417     :returns: (apks, cachechanged) where apks is a list of apk information,
 418               and cachechanged is True if the apkcache got changed.
 419     """
 420
 421     cachechanged = False
 422
 423     for icon_dir in get_all_icon_dirs(repodir):
 424         if os.path.exists(icon_dir):
 425             if options.clean:
 426                 shutil.rmtree(icon_dir)
 427                 os.makedirs(icon_dir)
 428         else:
 429             os.makedirs(icon_dir)
 430
 431     apks = []
 432     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 433     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 434     vername_pat = re.compile(".*versionName='([^']*)'.*")
 435     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 436     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 437     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 438     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 439     string_pat = re.compile(".*'([^']*)'.*")
 440     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 441
 442         apkfilename = apkfile[len(repodir) + 1:]
 443         if ' ' in apkfilename:
 444             logging.critical("Spaces in filenames are not allowed.")
 445             sys.exit(1)
 446
 447         # Calculate the sha256...
 448         sha = hashlib.sha256()
 449         with open(apkfile, 'rb') as f:
 450             while True:
 451                 t = f.read(16384)
 452                 if len(t) == 0:
 453                     break
 454                 sha.update(t)
 455             shasum = sha.hexdigest()
 456
 457         usecache = False
 458         if apkfilename in apkcache:
 459             apk = apkcache[apkfilename]
 460             if apk['sha256'] == shasum:
 461                 logging.debug("Reading " + apkfilename + " from cache")
 462                 usecache = True
 463             else:
 464                 logging.debug("Ignoring stale cache data for " + apkfilename)
 465
 466         if not usecache:
 467             logging.debug("Processing " + apkfilename)
 468             apk = {}
 469             apk['apkname'] = apkfilename
 470             apk['sha256'] = shasum
 471             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 472             if os.path.exists(os.path.join(repodir, srcfilename)):
 473                 apk['srcname'] = srcfilename
 474             apk['size'] = os.path.getsize(apkfile)
 475             apk['permissions'] = set()
 476             apk['features'] = set()
 477             apk['icons_src'] = {}
 478             apk['icons'] = {}
 479             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 480             if p.returncode != 0:
 481                 if options.delete_unknown:
 482                     if os.path.exists(apkfile):
 483                         logging.error("Failed to get apk information, deleting " + apkfile)
 484                         os.remove(apkfile)
 485                     else:
 486                         logging.error("Could not find {0} to remove it".format(apkfile))
 487                 else:
 488                     logging.error("Failed to get apk information, skipping " + apkfile)
 489                 continue
 490             for line in p.output.splitlines():
 491                 if line.startswith("package:"):
 492                     try:
 493                         apk['id'] = re.match(name_pat, line).group(1)
 494                         apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
 495                         apk['version'] = re.match(vername_pat, line).group(1)
 496                     except Exception as e:
 497                         logging.error("Package matching failed: " + str(e))
 498                         logging.info("Line was: " + line)
 499                         sys.exit(1)
 500                 elif line.startswith("application:"):
 501                     apk['name'] = re.match(label_pat, line).group(1)
 502                     # Keep path to non-dpi icon in case we need it
 503                     match = re.match(icon_pat_nodpi, line)
 504                     if match:
 505                         apk['icons_src']['-1'] = match.group(1)
 506                 elif line.startswith("launchable-activity:"):
 507                     # Only use launchable-activity as fallback to application
 508                     if not apk['name']:
 509                         apk['name'] = re.match(label_pat, line).group(1)
 510                     if '-1' not in apk['icons_src']:
 511                         match = re.match(icon_pat_nodpi, line)
 512                         if match:
 513                             apk['icons_src']['-1'] = match.group(1)
 514                 elif line.startswith("application-icon-"):
 515                     match = re.match(icon_pat, line)
 516                     if match:
 517                         density = match.group(1)
 518                         path = match.group(2)
 519                         apk['icons_src'][density] = path
 520                 elif line.startswith("sdkVersion:"):
 521                     m = re.match(sdkversion_pat, line)
 522                     if m is None:
 523                         logging.error(line.replace('sdkVersion:', '')
 524                                       + ' is not a valid minSdkVersion!')
 525                     else:
 526                         apk['sdkversion'] = m.group(1)
 527                 elif line.startswith("maxSdkVersion:"):
 528                     apk['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 529                 elif line.startswith("native-code:"):
 530                     apk['nativecode'] = []
 531                     for arch in line[13:].split(' '):
 532                         apk['nativecode'].append(arch[1:-1])
 533                 elif line.startswith("uses-permission:"):
 534                     perm = re.match(string_pat, line).group(1)
 535                     if perm.startswith("android.permission."):
 536                         perm = perm[19:]
 537                     apk['permissions'].add(perm)
 538                 elif line.startswith("uses-feature:"):
 539                     perm = re.match(string_pat, line).group(1)
 540                     # Filter out this, it's only added with the latest SDK tools and
 541                     # causes problems for lots of apps.
 542                     if perm != "android.hardware.screen.portrait" \
 543                             and perm != "android.hardware.screen.landscape":
 544                         if perm.startswith("android.feature."):
 545                             perm = perm[16:]
 546                         apk['features'].add(perm)
 547
 548             if 'sdkversion' not in apk:
 549                 logging.warn("No SDK version information found in {0}".format(apkfile))
 550                 apk['sdkversion'] = 0
 551
 552             # Check for debuggable apks...
 553             if common.isApkDebuggable(apkfile, config):
 554                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 555
 556             # Get the signature (or md5 of, to be precise)...
 557             logging.debug('Getting signature of {0}'.format(apkfile))
 558             apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 559             if not apk['sig']:
 560                 logging.critical("Failed to get apk signature")
 561                 sys.exit(1)
 562
 563             apkzip = zipfile.ZipFile(apkfile, 'r')
 564
 565             # if an APK has files newer than the system time, suggest updating
 566             # the system clock.  This is useful for offline systems, used for
 567             # signing, which do not have another source of clock sync info. It
 568             # has to be more than 24 hours newer because ZIP/APK files do not
 569             # store timezone info
 570             manifest = apkzip.getinfo('AndroidManifest.xml')
 571             if manifest.date_time[1] == 0:  # month can't be zero
 572                 logging.debug('AndroidManifest.xml has no date')
 573             else:
 574                 dt_obj = datetime(*manifest.date_time)
 575                 checkdt = dt_obj - timedelta(1)
 576                 if datetime.today() < checkdt:
 577                     logging.warn('System clock is older than manifest in: '
 578                                  + apkfilename
 579                                  + '\nSet clock to that time using:\n'
 580                                  + 'sudo date -s "' + str(dt_obj) + '"')
 581
 582             iconfilename = "%s.%s.png" % (
 583                 apk['id'],
 584                 apk['versioncode'])
 585
 586             # Extract the icon file...
 587             empty_densities = []
 588             for density in screen_densities:
 589                 if density not in apk['icons_src']:
 590                     empty_densities.append(density)
 591                     continue
 592                 iconsrc = apk['icons_src'][density]
 593                 icon_dir = get_icon_dir(repodir, density)
 594                 icondest = os.path.join(icon_dir, iconfilename)
 595
 596                 try:
 597                     with open(icondest, 'wb') as f:
 598                         f.write(apkzip.read(iconsrc))
 599                     apk['icons'][density] = iconfilename
 600
 601                 except:
 602                     logging.warn("Error retrieving icon file")
 603                     del apk['icons'][density]
 604                     del apk['icons_src'][density]
 605                     empty_densities.append(density)
 606
 607             if '-1' in apk['icons_src']:
 608                 iconsrc = apk['icons_src']['-1']
 609                 iconpath = os.path.join(
 610                     get_icon_dir(repodir, '0'), iconfilename)
 611                 with open(iconpath, 'wb') as f:
 612                     f.write(apkzip.read(iconsrc))
 613                 try:
 614                     im = Image.open(iconpath)
 615                     dpi = px_to_dpi(im.size[0])
 616                     for density in screen_densities:
 617                         if density in apk['icons']:
 618                             break
 619                         if density == screen_densities[-1] or dpi >= int(density):
 620                             apk['icons'][density] = iconfilename
 621                             shutil.move(iconpath,
 622                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 623                             empty_densities.remove(density)
 624                             break
 625                 except Exception as e:
 626                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 627
 628             if apk['icons']:
 629                 apk['icon'] = iconfilename
 630
 631             apkzip.close()
 632
 633             # First try resizing down to not lose quality
 634             last_density = None
 635             for density in screen_densities:
 636                 if density not in empty_densities:
 637                     last_density = density
 638                     continue
 639                 if last_density is None:
 640                     continue
 641                 logging.debug("Density %s not available, resizing down from %s"
 642                               % (density, last_density))
 643
 644                 last_iconpath = os.path.join(
 645                     get_icon_dir(repodir, last_density), iconfilename)
 646                 iconpath = os.path.join(
 647                     get_icon_dir(repodir, density), iconfilename)
 648                 try:
 649                     im = Image.open(last_iconpath)
 650                 except:
 651                     logging.warn("Invalid image file at %s" % last_iconpath)
 652                     continue
 653
 654                 size = dpi_to_px(density)
 655
 656                 im.thumbnail((size, size), Image.ANTIALIAS)
 657                 im.save(iconpath, "PNG")
 658                 empty_densities.remove(density)
 659
 660             # Then just copy from the highest resolution available
 661             last_density = None
 662             for density in reversed(screen_densities):
 663                 if density not in empty_densities:
 664                     last_density = density
 665                     continue
 666                 if last_density is None:
 667                     continue
 668                 logging.debug("Density %s not available, copying from lower density %s"
 669                               % (density, last_density))
 670
 671                 shutil.copyfile(
 672                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 673                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 674
 675                 empty_densities.remove(density)
 676
 677             for density in screen_densities:
 678                 icon_dir = get_icon_dir(repodir, density)
 679                 icondest = os.path.join(icon_dir, iconfilename)
 680                 resize_icon(icondest, density)
 681
 682             # Copy from icons-mdpi to icons since mdpi is the baseline density
 683             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 684             if os.path.isfile(baseline):
 685                 apk['icons']['0'] = iconfilename
 686                 shutil.copyfile(baseline,
 687                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 688
 689             # Record in known apks, getting the added date at the same time..
 690             added = knownapks.recordapk(apk['apkname'], apk['id'])
 691             if added:
 692                 apk['added'] = added
 693
 694             apkcache[apkfilename] = apk
 695             cachechanged = True
 696
 697         apks.append(apk)
 698
 699     return apks, cachechanged
 700
 701
 702 repo_pubkey_fingerprint = None
 703
 704
 705 # Generate a certificate fingerprint the same way keytool does it
 706 # (but with slightly different formatting)
 707 def cert_fingerprint(data):
 708     digest = hashlib.sha256(data).digest()
 709     ret = []
 710     ret.append(' '.join("%02X" % ord(b) for b in digest))
 711     return " ".join(ret)
 712
 713
 714 def extract_pubkey():
 715     global repo_pubkey_fingerprint
 716     if 'repo_pubkey' in config:
 717         pubkey = unhexlify(config['repo_pubkey'])
 718     else:
 719         p = FDroidPopen([config['keytool'], '-exportcert',
 720                          '-alias', config['repo_keyalias'],
 721                          '-keystore', config['keystore'],
 722                          '-storepass:file', config['keystorepassfile']]
 723                         + config['smartcardoptions'], output=False)
 724         if p.returncode != 0 or len(p.output) < 20:
 725             msg = "Failed to get repo pubkey!"
 726             if config['keystore'] == 'NONE':
 727                 msg += ' Is your crypto smartcard plugged in?'
 728             logging.critical(msg)
 729             sys.exit(1)
 730         pubkey = p.output
 731     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 732     return hexlify(pubkey)
 733
 734
 735 def make_index(apps, sortedids, apks, repodir, archive, categories):
 736     """Make a repo index.
 737
 738     :param apps: fully populated apps list
 739     :param apks: full populated apks list
 740     :param repodir: the repo directory
 741     :param archive: True if this is the archive repo, False if it's the
 742                     main one.
 743     :param categories: list of categories
 744     """
 745
 746     doc = Document()
 747
 748     def addElement(name, value, doc, parent):
 749         el = doc.createElement(name)
 750         el.appendChild(doc.createTextNode(value))
 751         parent.appendChild(el)
 752
 753     def addElementNonEmpty(name, value, doc, parent):
 754         if not value:
 755             return
 756         addElement(name, value, doc, parent)
 757
 758     def addElementCDATA(name, value, doc, parent):
 759         el = doc.createElement(name)
 760         el.appendChild(doc.createCDATASection(value))
 761         parent.appendChild(el)
 762
 763     root = doc.createElement("fdroid")
 764     doc.appendChild(root)
 765
 766     repoel = doc.createElement("repo")
 767
 768     mirrorcheckfailed = False
 769     for mirror in config.get('mirrors', []):
 770         base = os.path.basename(urlparse.urlparse(mirror).path.rstrip('/'))
 771         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
 772             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 773             mirrorcheckfailed = True
 774     if mirrorcheckfailed:
 775         sys.exit(1)
 776
 777     if archive:
 778         repoel.setAttribute("name", config['archive_name'])
 779         if config['repo_maxage'] != 0:
 780             repoel.setAttribute("maxage", str(config['repo_maxage']))
 781         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 782         repoel.setAttribute("url", config['archive_url'])
 783         addElement('description', config['archive_description'], doc, repoel)
 784         urlbasepath = os.path.basename(urlparse.urlparse(config['archive_url']).path)
 785         for mirror in config.get('mirrors', []):
 786             addElement('mirror', urlparse.urljoin(mirror, urlbasepath), doc, repoel)
 787
 788     else:
 789         repoel.setAttribute("name", config['repo_name'])
 790         if config['repo_maxage'] != 0:
 791             repoel.setAttribute("maxage", str(config['repo_maxage']))
 792         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 793         repoel.setAttribute("url", config['repo_url'])
 794         addElement('description', config['repo_description'], doc, repoel)
 795         urlbasepath = os.path.basename(urlparse.urlparse(config['repo_url']).path)
 796         for mirror in config.get('mirrors', []):
 797             addElement('mirror', urlparse.urljoin(mirror, urlbasepath), doc, repoel)
 798
 799     repoel.setAttribute("version", "15")
 800     repoel.setAttribute("timestamp", str(int(time.time())))
 801
 802     nosigningkey = False
 803     if not options.nosign:
 804         if 'repo_keyalias' not in config:
 805             nosigningkey = True
 806             logging.critical("'repo_keyalias' not found in config.py!")
 807         if 'keystore' not in config:
 808             nosigningkey = True
 809             logging.critical("'keystore' not found in config.py!")
 810         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 811             nosigningkey = True
 812             logging.critical("'keystorepass' not found in config.py!")
 813         if 'keypass' not in config and 'keypassfile' not in config:
 814             nosigningkey = True
 815             logging.critical("'keypass' not found in config.py!")
 816         if not os.path.exists(config['keystore']):
 817             nosigningkey = True
 818             logging.critical("'" + config['keystore'] + "' does not exist!")
 819         if nosigningkey:
 820             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 821             logging.warning("\tfdroid update --create-key")
 822             sys.exit(1)
 823
 824     repoel.setAttribute("pubkey", extract_pubkey())
 825     root.appendChild(repoel)
 826
 827     for appid in sortedids:
 828         app = apps[appid]
 829
 830         if app.Disabled is not None:
 831             continue
 832
 833         # Get a list of the apks for this app...
 834         apklist = []
 835         for apk in apks:
 836             if apk['id'] == appid:
 837                 apklist.append(apk)
 838
 839         if len(apklist) == 0:
 840             continue
 841
 842         apel = doc.createElement("application")
 843         apel.setAttribute("id", app.id)
 844         root.appendChild(apel)
 845
 846         addElement('id', app.id, doc, apel)
 847         if app.added:
 848             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
 849         if app.lastupdated:
 850             addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
 851         addElement('name', app.Name, doc, apel)
 852         addElement('summary', app.Summary, doc, apel)
 853         if app.icon:
 854             addElement('icon', app.icon, doc, apel)
 855
 856         def linkres(appid):
 857             if appid in apps:
 858                 return ("fdroid.app:" + appid, apps[appid].Name)
 859             raise MetaDataException("Cannot resolve app id " + appid)
 860
 861         addElement('desc',
 862                    metadata.description_html(app.Description, linkres),
 863                    doc, apel)
 864         addElement('license', app.License, doc, apel)
 865         if app.Categories:
 866             addElement('categories', ','.join(app.Categories), doc, apel)
 867             # We put the first (primary) category in LAST, which will have
 868             # the desired effect of making clients that only understand one
 869             # category see that one.
 870             addElement('category', app.Categories[0], doc, apel)
 871         addElement('web', app.WebSite, doc, apel)
 872         addElement('source', app.SourceCode, doc, apel)
 873         addElement('tracker', app.IssueTracker, doc, apel)
 874         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 875         addElementNonEmpty('author', app.AuthorName, doc, apel)
 876         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 877         addElementNonEmpty('donate', app.Donate, doc, apel)
 878         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 879         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 880         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 881
 882         # These elements actually refer to the current version (i.e. which
 883         # one is recommended. They are historically mis-named, and need
 884         # changing, but stay like this for now to support existing clients.
 885         addElement('marketversion', app.CurrentVersion, doc, apel)
 886         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 887
 888         if app.AntiFeatures:
 889             af = app.AntiFeatures
 890             if af:
 891                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 892         if app.Provides:
 893             pv = app.Provides.split(',')
 894             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 895         if app.RequiresRoot:
 896             addElement('requirements', 'root', doc, apel)
 897
 898         # Sort the apk list into version order, just so the web site
 899         # doesn't have to do any work by default...
 900         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 901
 902         # Check for duplicates - they will make the client unhappy...
 903         for i in range(len(apklist) - 1):
 904             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 905                 logging.critical("duplicate versions: '%s' - '%s'" % (
 906                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 907                 sys.exit(1)
 908
 909         current_version_code = 0
 910         current_version_file = None
 911         for apk in apklist:
 912             # find the APK for the "Current Version"
 913             if current_version_code < apk['versioncode']:
 914                 current_version_code = apk['versioncode']
 915             if current_version_code < int(app.CurrentVersionCode):
 916                 current_version_file = apk['apkname']
 917
 918             apkel = doc.createElement("package")
 919             apel.appendChild(apkel)
 920             addElement('version', apk['version'], doc, apkel)
 921             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 922             addElement('apkname', apk['apkname'], doc, apkel)
 923             if 'srcname' in apk:
 924                 addElement('srcname', apk['srcname'], doc, apkel)
 925             for hash_type in ['sha256']:
 926                 if hash_type not in apk:
 927                     continue
 928                 hashel = doc.createElement("hash")
 929                 hashel.setAttribute("type", hash_type)
 930                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 931                 apkel.appendChild(hashel)
 932             addElement('sig', apk['sig'], doc, apkel)
 933             addElement('size', str(apk['size']), doc, apkel)
 934             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 935             if 'maxsdkversion' in apk:
 936                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 937             if 'added' in apk:
 938                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 939             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
 940             if 'nativecode' in apk:
 941                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 942             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
 943
 944         if current_version_file is not None \
 945                 and config['make_current_version_link'] \
 946                 and repodir == 'repo':  # only create these
 947             namefield = config['current_version_name_source']
 948             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
 949             apklinkname = sanitized_name + '.apk'
 950             current_version_path = os.path.join(repodir, current_version_file)
 951             if os.path.islink(apklinkname):
 952                 os.remove(apklinkname)
 953             os.symlink(current_version_path, apklinkname)
 954             # also symlink gpg signature, if it exists
 955             for extension in ('.asc', '.sig'):
 956                 sigfile_path = current_version_path + extension
 957                 if os.path.exists(sigfile_path):
 958                     siglinkname = apklinkname + extension
 959                     if os.path.islink(siglinkname):
 960                         os.remove(siglinkname)
 961                     os.symlink(sigfile_path, siglinkname)
 962
 963     if options.pretty:
 964         output = doc.toprettyxml()
 965     else:
 966         output = doc.toxml()
 967
 968     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 969         f.write(output)
 970
 971     if 'repo_keyalias' in config:
 972
 973         if options.nosign:
 974             logging.info("Creating unsigned index in preparation for signing")
 975         else:
 976             logging.info("Creating signed index with this key (SHA256):")
 977             logging.info("%s" % repo_pubkey_fingerprint)
 978
 979         # Create a jar of the index...
 980         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
 981         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 982         if p.returncode != 0:
 983             logging.critical("Failed to create {0}".format(jar_output))
 984             sys.exit(1)
 985
 986         # Sign the index...
 987         signed = os.path.join(repodir, 'index.jar')
 988         if options.nosign:
 989             # Remove old signed index if not signing
 990             if os.path.exists(signed):
 991                 os.remove(signed)
 992         else:
 993             args = [config['jarsigner'], '-keystore', config['keystore'],
 994                     '-storepass:file', config['keystorepassfile'],
 995                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
 996                     signed, config['repo_keyalias']]
 997             if config['keystore'] == 'NONE':
 998                 args += config['smartcardoptions']
 999             else:  # smardcards never use -keypass
1000                 args += ['-keypass:file', config['keypassfile']]
1001             p = FDroidPopen(args)
1002             if p.returncode != 0:
1003                 logging.critical("Failed to sign index")
1004                 sys.exit(1)
1005
1006     # Copy the repo icon into the repo directory...
1007     icon_dir = os.path.join(repodir, 'icons')
1008     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1009     shutil.copyfile(config['repo_icon'], iconfilename)
1010
1011     # Write a category list in the repo to allow quick access...
1012     catdata = ''
1013     for cat in categories:
1014         catdata += cat + '\n'
1015     with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
1016         f.write(catdata)
1017
1018
1019 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1020
1021     for appid, app in apps.iteritems():
1022
1023         if app.ArchivePolicy:
1024             keepversions = int(app.ArchivePolicy[:-9])
1025         else:
1026             keepversions = defaultkeepversions
1027
1028         def filter_apk_list_sorted(apk_list):
1029             res = []
1030             for apk in apk_list:
1031                 if apk['id'] == appid:
1032                     res.append(apk)
1033
1034             # Sort the apk list by version code. First is highest/newest.
1035             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1036
1037         def move_file(from_dir, to_dir, filename, ignore_missing):
1038             from_path = os.path.join(from_dir, filename)
1039             if ignore_missing and not os.path.exists(from_path):
1040                 return
1041             to_path = os.path.join(to_dir, filename)
1042             shutil.move(from_path, to_path)
1043
1044         if len(apks) > keepversions:
1045             apklist = filter_apk_list_sorted(apks)
1046             # Move back the ones we don't want.
1047             for apk in apklist[keepversions:]:
1048                 logging.info("Moving " + apk['apkname'] + " to archive")
1049                 move_file(repodir, archivedir, apk['apkname'], False)
1050                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1051                 for density in all_screen_densities:
1052                     repo_icon_dir = get_icon_dir(repodir, density)
1053                     archive_icon_dir = get_icon_dir(archivedir, density)
1054                     if density not in apk['icons']:
1055                         continue
1056                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1057                 if 'srcname' in apk:
1058                     move_file(repodir, archivedir, apk['srcname'], False)
1059                 archapks.append(apk)
1060                 apks.remove(apk)
1061         elif len(apks) < keepversions and len(archapks) > 0:
1062             required = keepversions - len(apks)
1063             archapklist = filter_apk_list_sorted(archapks)
1064             # Move forward the ones we want again.
1065             for apk in archapklist[:required]:
1066                 logging.info("Moving " + apk['apkname'] + " from archive")
1067                 move_file(archivedir, repodir, apk['apkname'], False)
1068                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1069                 for density in all_screen_densities:
1070                     repo_icon_dir = get_icon_dir(repodir, density)
1071                     archive_icon_dir = get_icon_dir(archivedir, density)
1072                     if density not in apk['icons']:
1073                         continue
1074                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1075                 if 'srcname' in apk:
1076                     move_file(archivedir, repodir, apk['srcname'], False)
1077                 archapks.remove(apk)
1078                 apks.append(apk)
1079
1080
1081 def add_apks_to_per_app_repos(repodir, apks):
1082     apks_per_app = dict()
1083     for apk in apks:
1084         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1085         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1086         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1087         apks_per_app[apk['id']] = apk
1088
1089         if not os.path.exists(apk['per_app_icons']):
1090             logging.info('Adding new repo for only ' + apk['id'])
1091             os.makedirs(apk['per_app_icons'])
1092
1093         apkpath = os.path.join(repodir, apk['apkname'])
1094         shutil.copy(apkpath, apk['per_app_repo'])
1095         apksigpath = apkpath + '.sig'
1096         if os.path.exists(apksigpath):
1097             shutil.copy(apksigpath, apk['per_app_repo'])
1098         apkascpath = apkpath + '.asc'
1099         if os.path.exists(apkascpath):
1100             shutil.copy(apkascpath, apk['per_app_repo'])
1101
1102
1103 config = None
1104 options = None
1105
1106
1107 def main():
1108
1109     global config, options
1110
1111     # Parse command line...
1112     parser = ArgumentParser()
1113     common.setup_global_opts(parser)
1114     parser.add_argument("--create-key", action="store_true", default=False,
1115                         help="Create a repo signing key in a keystore")
1116     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1117                         help="Create skeleton metadata files that are missing")
1118     parser.add_argument("--delete-unknown", action="store_true", default=False,
1119                         help="Delete APKs without metadata from the repo")
1120     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1121                         help="Report on build data status")
1122     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1123                         help="Interactively ask about things that need updating.")
1124     parser.add_argument("-I", "--icons", action="store_true", default=False,
1125                         help="Resize all the icons exceeding the max pixel size and exit")
1126     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1127                         help="Specify editor to use in interactive mode. Default " +
1128                         "is /etc/alternatives/editor")
1129     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1130                         help="Update the wiki")
1131     parser.add_argument("--pretty", action="store_true", default=False,
1132                         help="Produce human-readable index.xml")
1133     parser.add_argument("--clean", action="store_true", default=False,
1134                         help="Clean update - don't uses caches, reprocess all apks")
1135     parser.add_argument("--nosign", action="store_true", default=False,
1136                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1137     options = parser.parse_args()
1138
1139     config = common.read_config(options)
1140
1141     if not ('jarsigner' in config and 'keytool' in config):
1142         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1143         sys.exit(1)
1144
1145     repodirs = ['repo']
1146     if config['archive_older'] != 0:
1147         repodirs.append('archive')
1148         if not os.path.exists('archive'):
1149             os.mkdir('archive')
1150
1151     if options.icons:
1152         resize_all_icons(repodirs)
1153         sys.exit(0)
1154
1155     # check that icons exist now, rather than fail at the end of `fdroid update`
1156     for k in ['repo_icon', 'archive_icon']:
1157         if k in config:
1158             if not os.path.exists(config[k]):
1159                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1160                 sys.exit(1)
1161
1162     # if the user asks to create a keystore, do it now, reusing whatever it can
1163     if options.create_key:
1164         if os.path.exists(config['keystore']):
1165             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1166             logging.critical("\t'" + config['keystore'] + "'")
1167             sys.exit(1)
1168
1169         if 'repo_keyalias' not in config:
1170             config['repo_keyalias'] = socket.getfqdn()
1171             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1172         if 'keydname' not in config:
1173             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1174             common.write_to_config(config, 'keydname', config['keydname'])
1175         if 'keystore' not in config:
1176             config['keystore'] = common.default_config.keystore
1177             common.write_to_config(config, 'keystore', config['keystore'])
1178
1179         password = common.genpassword()
1180         if 'keystorepass' not in config:
1181             config['keystorepass'] = password
1182             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1183         if 'keypass' not in config:
1184             config['keypass'] = password
1185             common.write_to_config(config, 'keypass', config['keypass'])
1186         common.genkeystore(config)
1187
1188     # Get all apps...
1189     apps = metadata.read_metadata()
1190
1191     # Generate a list of categories...
1192     categories = set()
1193     for app in apps.itervalues():
1194         categories.update(app.Categories)
1195
1196     # Read known apks data (will be updated and written back when we've finished)
1197     knownapks = common.KnownApks()
1198
1199     # Gather information about all the apk files in the repo directory, using
1200     # cached data if possible.
1201     apkcachefile = os.path.join('tmp', 'apkcache')
1202     if not options.clean and os.path.exists(apkcachefile):
1203         with open(apkcachefile, 'rb') as cf:
1204             apkcache = pickle.load(cf)
1205     else:
1206         apkcache = {}
1207
1208     delete_disabled_builds(apps, apkcache, repodirs)
1209
1210     # Scan all apks in the main repo
1211     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks)
1212
1213     # Generate warnings for apk's with no metadata (or create skeleton
1214     # metadata files, if requested on the command line)
1215     newmetadata = False
1216     for apk in apks:
1217         if apk['id'] not in apps:
1218             if options.create_metadata:
1219                 if 'name' not in apk:
1220                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1221                     continue
1222                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1223                 f.write("License:Unknown\n")
1224                 f.write("Web Site:\n")
1225                 f.write("Source Code:\n")
1226                 f.write("Issue Tracker:\n")
1227                 f.write("Changelog:\n")
1228                 f.write("Summary:" + apk['name'] + "\n")
1229                 f.write("Description:\n")
1230                 f.write(apk['name'] + "\n")
1231                 f.write(".\n")
1232                 f.close()
1233                 logging.info("Generated skeleton metadata for " + apk['id'])
1234                 newmetadata = True
1235             else:
1236                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1237                 if options.delete_unknown:
1238                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1239                     rmf = os.path.join(repodirs[0], apk['apkname'])
1240                     if not os.path.exists(rmf):
1241                         logging.error("Could not find {0} to remove it".format(rmf))
1242                     else:
1243                         os.remove(rmf)
1244                 else:
1245                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1246
1247     # update the metadata with the newly created ones included
1248     if newmetadata:
1249         apps = metadata.read_metadata()
1250
1251     # Scan the archive repo for apks as well
1252     if len(repodirs) > 1:
1253         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1254         if cc:
1255             cachechanged = True
1256     else:
1257         archapks = []
1258
1259     # Some information from the apks needs to be applied up to the application
1260     # level. When doing this, we use the info from the most recent version's apk.
1261     # We deal with figuring out when the app was added and last updated at the
1262     # same time.
1263     for appid, app in apps.iteritems():
1264         bestver = 0
1265         for apk in apks + archapks:
1266             if apk['id'] == appid:
1267                 if apk['versioncode'] > bestver:
1268                     bestver = apk['versioncode']
1269                     bestapk = apk
1270
1271                 if 'added' in apk:
1272                     if not app.added or apk['added'] < app.added:
1273                         app.added = apk['added']
1274                     if not app.lastupdated or apk['added'] > app.lastupdated:
1275                         app.lastupdated = apk['added']
1276
1277         if not app.added:
1278             logging.debug("Don't know when " + appid + " was added")
1279         if not app.lastupdated:
1280             logging.debug("Don't know when " + appid + " was last updated")
1281
1282         if bestver == 0:
1283             if app.Name is None:
1284                 app.Name = app.AutoName or appid
1285             app.icon = None
1286             logging.debug("Application " + appid + " has no packages")
1287         else:
1288             if app.Name is None:
1289                 app.Name = bestapk['name']
1290             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1291             if app.CurrentVersionCode is None:
1292                 app.CurrentVersionCode = str(bestver)
1293
1294     # Sort the app list by name, then the web site doesn't have to by default.
1295     # (we had to wait until we'd scanned the apks to do this, because mostly the
1296     # name comes from there!)
1297     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid].Name.upper())
1298
1299     # APKs are placed into multiple repos based on the app package, providing
1300     # per-app subscription feeds for nightly builds and things like it
1301     if config['per_app_repos']:
1302         add_apks_to_per_app_repos(repodirs[0], apks)
1303         for appid, app in apps.iteritems():
1304             repodir = os.path.join(appid, 'fdroid', 'repo')
1305             appdict = dict()
1306             appdict[appid] = app
1307             if os.path.isdir(repodir):
1308                 make_index(appdict, [appid], apks, repodir, False, categories)
1309             else:
1310                 logging.info('Skipping index generation for ' + appid)
1311         return
1312
1313     if len(repodirs) > 1:
1314         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1315
1316     # Make the index for the main repo...
1317     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1318
1319     # If there's an archive repo,  make the index for it. We already scanned it
1320     # earlier on.
1321     if len(repodirs) > 1:
1322         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1323
1324     if config['update_stats']:
1325
1326         # Update known apks info...
1327         knownapks.writeifchanged()
1328
1329         # Generate latest apps data for widget
1330         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1331             data = ''
1332             for line in file(os.path.join('stats', 'latestapps.txt')):
1333                 appid = line.rstrip()
1334                 data += appid + "\t"
1335                 app = apps[appid]
1336                 data += app.Name + "\t"
1337                 if app.icon is not None:
1338                     data += app.icon + "\t"
1339                 data += app.License + "\n"
1340             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
1341                 f.write(data)
1342
1343     if cachechanged:
1344         with open(apkcachefile, 'wb') as cf:
1345             pickle.dump(apkcache, cf)
1346
1347     # Update the wiki...
1348     if options.wiki:
1349         update_wiki(apps, sortedids, apks + archapks)
1350
1351     logging.info("Finished.")
1352
1353 if __name__ == "__main__":
1354     main()