fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import socket
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 import urlparse
  31 from datetime import datetime, timedelta
  32 from xml.dom.minidom import Document
  33 from argparse import ArgumentParser
  34 import time
  35 from pyasn1.error import PyAsn1Error
  36 from pyasn1.codec.der import decoder, encoder
  37 from pyasn1_modules import rfc2315
  38 from hashlib import md5
  39 from binascii import hexlify, unhexlify
  40
  41 from PIL import Image
  42 import logging
  43
  44 import common
  45 import metadata
  46 from common import FDroidPopen, SdkToolsPopen
  47 from metadata import MetaDataException
  48
  49 screen_densities = ['640', '480', '320', '240', '160', '120']
  50
  51 all_screen_densities = ['0'] + screen_densities
  52
  53
  54 def dpi_to_px(density):
  55     return (int(density) * 48) / 160
  56
  57
  58 def px_to_dpi(px):
  59     return (int(px) * 160) / 48
  60
  61
  62 def get_icon_dir(repodir, density):
  63     if density == '0':
  64         return os.path.join(repodir, "icons")
  65     return os.path.join(repodir, "icons-%s" % density)
  66
  67
  68 def get_icon_dirs(repodir):
  69     for density in screen_densities:
  70         yield get_icon_dir(repodir, density)
  71
  72
  73 def get_all_icon_dirs(repodir):
  74     for density in all_screen_densities:
  75         yield get_icon_dir(repodir, density)
  76
  77
  78 def update_wiki(apps, sortedids, apks):
  79     """Update the wiki
  80
  81     :param apps: fully populated list of all applications
  82     :param apks: all apks, except...
  83     """
  84     logging.info("Updating wiki")
  85     wikicat = 'Apps'
  86     wikiredircat = 'App Redirects'
  87     import mwclient
  88     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  89                          path=config['wiki_path'])
  90     site.login(config['wiki_user'], config['wiki_password'])
  91     generated_pages = {}
  92     generated_redirects = {}
  93
  94     for appid in sortedids:
  95         app = apps[appid]
  96
  97         wikidata = ''
  98         if app.Disabled:
  99             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 100         if app.AntiFeatures:
 101             for af in app.AntiFeatures:
 102                 wikidata += '{{AntiFeature|' + af + '}}\n'
 103         if app.RequiresRoot:
 104             requiresroot = 'Yes'
 105         else:
 106             requiresroot = 'No'
 107         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 108             appid,
 109             app.Name,
 110             time.strftime('%Y-%m-%d', app.added) if app.added else '',
 111             time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
 112             app.SourceCode,
 113             app.IssueTracker,
 114             app.WebSite,
 115             app.Changelog,
 116             app.Donate,
 117             app.FlattrID,
 118             app.Bitcoin,
 119             app.Litecoin,
 120             app.License,
 121             requiresroot,
 122             app.AuthorName,
 123             app.AuthorEmail)
 124
 125         if app.Provides:
 126             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 127
 128         wikidata += app.Summary
 129         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 130
 131         wikidata += "=Description=\n"
 132         wikidata += metadata.description_wiki(app.Description) + "\n"
 133
 134         wikidata += "=Maintainer Notes=\n"
 135         if app.MaintainerNotes:
 136             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 137         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 138
 139         # Get a list of all packages for this application...
 140         apklist = []
 141         gotcurrentver = False
 142         cantupdate = False
 143         buildfails = False
 144         for apk in apks:
 145             if apk['id'] == appid:
 146                 if str(apk['versioncode']) == app.CurrentVersionCode:
 147                     gotcurrentver = True
 148                 apklist.append(apk)
 149         # Include ones we can't build, as a special case...
 150         for build in app.builds:
 151             if build.disable:
 152                 if build.vercode == app.CurrentVersionCode:
 153                     cantupdate = True
 154                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 155                 apklist.append({'versioncode': int(build.vercode),
 156                                 'version': build.version,
 157                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 158                                 })
 159             else:
 160                 builtit = False
 161                 for apk in apklist:
 162                     if apk['versioncode'] == int(build.vercode):
 163                         builtit = True
 164                         break
 165                 if not builtit:
 166                     buildfails = True
 167                     apklist.append({'versioncode': int(build.vercode),
 168                                     'version': build.version,
 169                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
 170                                     })
 171         if app.CurrentVersionCode == '0':
 172             cantupdate = True
 173         # Sort with most recent first...
 174         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 175
 176         wikidata += "=Versions=\n"
 177         if len(apklist) == 0:
 178             wikidata += "We currently have no versions of this app available."
 179         elif not gotcurrentver:
 180             wikidata += "We don't have the current version of this app."
 181         else:
 182             wikidata += "We have the current version of this app."
 183         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 184         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 185         if len(app.NoSourceSince) > 0:
 186             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 187         if len(app.CurrentVersion) > 0:
 188             wikidata += "The current (recommended) version is " + app.CurrentVersion
 189             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 190         validapks = 0
 191         for apk in apklist:
 192             wikidata += "==" + apk['version'] + "==\n"
 193
 194             if 'buildproblem' in apk:
 195                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 196             else:
 197                 validapks += 1
 198                 wikidata += "This version is built and signed by "
 199                 if 'srcname' in apk:
 200                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 201                 else:
 202                     wikidata += "the original developer.\n\n"
 203             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 204
 205         wikidata += '\n[[Category:' + wikicat + ']]\n'
 206         if len(app.NoSourceSince) > 0:
 207             wikidata += '\n[[Category:Apps missing source code]]\n'
 208         if validapks == 0 and not app.Disabled:
 209             wikidata += '\n[[Category:Apps with no packages]]\n'
 210         if cantupdate and not app.Disabled:
 211             wikidata += "\n[[Category:Apps we can't update]]\n"
 212         if buildfails and not app.Disabled:
 213             wikidata += "\n[[Category:Apps with failing builds]]\n"
 214         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 215             wikidata += '\n[[Category:Apps to Update]]\n'
 216         if app.Disabled:
 217             wikidata += '\n[[Category:Apps that are disabled]]\n'
 218         if app.UpdateCheckMode == 'None' and not app.Disabled:
 219             wikidata += '\n[[Category:Apps with no update check]]\n'
 220         for appcat in app.Categories:
 221             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 222
 223         # We can't have underscores in the page name, even if they're in
 224         # the package ID, because MediaWiki messes with them...
 225         pagename = appid.replace('_', ' ')
 226
 227         # Drop a trailing newline, because mediawiki is going to drop it anyway
 228         # and it we don't we'll think the page has changed when it hasn't...
 229         if wikidata.endswith('\n'):
 230             wikidata = wikidata[:-1]
 231
 232         generated_pages[pagename] = wikidata
 233
 234         # Make a redirect from the name to the ID too, unless there's
 235         # already an existing page with the name and it isn't a redirect.
 236         noclobber = False
 237         apppagename = app.Name.replace('_', ' ')
 238         apppagename = apppagename.replace('{', '')
 239         apppagename = apppagename.replace('}', ' ')
 240         apppagename = apppagename.replace(':', ' ')
 241         # Drop double spaces caused mostly by replacing ':' above
 242         apppagename = apppagename.replace('  ', ' ')
 243         for expagename in site.allpages(prefix=apppagename,
 244                                         filterredir='nonredirects',
 245                                         generator=False):
 246             if expagename == apppagename:
 247                 noclobber = True
 248         # Another reason not to make the redirect page is if the app name
 249         # is the same as it's ID, because that will overwrite the real page
 250         # with an redirect to itself! (Although it seems like an odd
 251         # scenario this happens a lot, e.g. where there is metadata but no
 252         # builds or binaries to extract a name from.
 253         if apppagename == pagename:
 254             noclobber = True
 255         if not noclobber:
 256             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 257
 258     for tcat, genp in [(wikicat, generated_pages),
 259                        (wikiredircat, generated_redirects)]:
 260         catpages = site.Pages['Category:' + tcat]
 261         existingpages = []
 262         for page in catpages:
 263             existingpages.append(page.name)
 264             if page.name in genp:
 265                 pagetxt = page.edit()
 266                 if pagetxt != genp[page.name]:
 267                     logging.debug("Updating modified page " + page.name)
 268                     page.save(genp[page.name], summary='Auto-updated')
 269                 else:
 270                     logging.debug("Page " + page.name + " is unchanged")
 271             else:
 272                 logging.warn("Deleting page " + page.name)
 273                 page.delete('No longer published')
 274         for pagename, text in genp.items():
 275             logging.debug("Checking " + pagename)
 276             if pagename not in existingpages:
 277                 logging.debug("Creating page " + pagename)
 278                 try:
 279                     newpage = site.Pages[pagename]
 280                     newpage.save(text, summary='Auto-created')
 281                 except:
 282                     logging.error("...FAILED to create page '{0}'".format(pagename))
 283
 284     # Purge server cache to ensure counts are up to date
 285     site.pages['Repository Maintenance'].purge()
 286
 287
 288 def delete_disabled_builds(apps, apkcache, repodirs):
 289     """Delete disabled build outputs.
 290
 291     :param apps: list of all applications, as per metadata.read_metadata
 292     :param apkcache: current apk cache information
 293     :param repodirs: the repo directories to process
 294     """
 295     for appid, app in apps.iteritems():
 296         for build in app.builds:
 297             if not build.disable:
 298                 continue
 299             apkfilename = appid + '_' + str(build.vercode) + '.apk'
 300             iconfilename = "%s.%s.png" % (
 301                 appid,
 302                 build.vercode)
 303             for repodir in repodirs:
 304                 files = [
 305                     os.path.join(repodir, apkfilename),
 306                     os.path.join(repodir, apkfilename + '.asc'),
 307                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 308                 ]
 309                 for density in all_screen_densities:
 310                     repo_dir = get_icon_dir(repodir, density)
 311                     files.append(os.path.join(repo_dir, iconfilename))
 312
 313                 for f in files:
 314                     if os.path.exists(f):
 315                         logging.info("Deleting disabled build output " + f)
 316                         os.remove(f)
 317             if apkfilename in apkcache:
 318                 del apkcache[apkfilename]
 319
 320
 321 def resize_icon(iconpath, density):
 322
 323     if not os.path.isfile(iconpath):
 324         return
 325
 326     try:
 327         im = Image.open(iconpath)
 328         size = dpi_to_px(density)
 329
 330         if any(length > size for length in im.size):
 331             oldsize = im.size
 332             im.thumbnail((size, size), Image.ANTIALIAS)
 333             logging.debug("%s was too large at %s - new size is %s" % (
 334                 iconpath, oldsize, im.size))
 335             im.save(iconpath, "PNG")
 336
 337     except Exception as e:
 338         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 339
 340
 341 def resize_all_icons(repodirs):
 342     """Resize all icons that exceed the max size
 343
 344     :param repodirs: the repo directories to process
 345     """
 346     for repodir in repodirs:
 347         for density in screen_densities:
 348             icon_dir = get_icon_dir(repodir, density)
 349             icon_glob = os.path.join(icon_dir, '*.png')
 350             for iconpath in glob.glob(icon_glob):
 351                 resize_icon(iconpath, density)
 352
 353
 354 # A signature block file with a .DSA, .RSA, or .EC extension
 355 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 356
 357
 358 def getsig(apkpath):
 359     """ Get the signing certificate of an apk. To get the same md5 has that
 360     Android gets, we encode the .RSA certificate in a specific format and pass
 361     it hex-encoded to the md5 digest algorithm.
 362
 363     :param apkpath: path to the apk
 364     :returns: A string containing the md5 of the signature of the apk or None
 365               if an error occurred.
 366     """
 367
 368     cert = None
 369
 370     # verify the jar signature is correct
 371     args = [config['jarsigner'], '-verify', apkpath]
 372     p = FDroidPopen(args)
 373     if p.returncode != 0:
 374         logging.critical(apkpath + " has a bad signature!")
 375         return None
 376
 377     with zipfile.ZipFile(apkpath, 'r') as apk:
 378
 379         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 380
 381         if len(certs) < 1:
 382             logging.error("Found no signing certificates on %s" % apkpath)
 383             return None
 384         if len(certs) > 1:
 385             logging.error("Found multiple signing certificates on %s" % apkpath)
 386             return None
 387
 388         cert = apk.read(certs[0])
 389
 390     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 391     if content.getComponentByName('contentType') != rfc2315.signedData:
 392         logging.error("Unexpected format.")
 393         return None
 394
 395     content = decoder.decode(content.getComponentByName('content'),
 396                              asn1Spec=rfc2315.SignedData())[0]
 397     try:
 398         certificates = content.getComponentByName('certificates')
 399     except PyAsn1Error:
 400         logging.error("Certificates not found.")
 401         return None
 402
 403     cert_encoded = encoder.encode(certificates)[4:]
 404
 405     return md5(cert_encoded.encode('hex')).hexdigest()
 406
 407
 408 def scan_apks(apps, apkcache, repodir, knownapks):
 409     """Scan the apks in the given repo directory.
 410
 411     This also extracts the icons.
 412
 413     :param apps: list of all applications, as per metadata.read_metadata
 414     :param apkcache: current apk cache information
 415     :param repodir: repo directory to scan
 416     :param knownapks: known apks info
 417     :returns: (apks, cachechanged) where apks is a list of apk information,
 418               and cachechanged is True if the apkcache got changed.
 419     """
 420
 421     cachechanged = False
 422
 423     for icon_dir in get_all_icon_dirs(repodir):
 424         if os.path.exists(icon_dir):
 425             if options.clean:
 426                 shutil.rmtree(icon_dir)
 427                 os.makedirs(icon_dir)
 428         else:
 429             os.makedirs(icon_dir)
 430
 431     apks = []
 432     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 433     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 434     vername_pat = re.compile(".*versionName='([^']*)'.*")
 435     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 436     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 437     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 438     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 439     string_pat = re.compile(".*'([^']*)'.*")
 440     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 441
 442         apkfilename = apkfile[len(repodir) + 1:]
 443         if ' ' in apkfilename:
 444             logging.critical("Spaces in filenames are not allowed.")
 445             sys.exit(1)
 446
 447         # Calculate the sha256...
 448         sha = hashlib.sha256()
 449         with open(apkfile, 'rb') as f:
 450             while True:
 451                 t = f.read(16384)
 452                 if len(t) == 0:
 453                     break
 454                 sha.update(t)
 455             shasum = sha.hexdigest()
 456
 457         usecache = False
 458         if apkfilename in apkcache:
 459             apk = apkcache[apkfilename]
 460             if apk['sha256'] == shasum:
 461                 logging.debug("Reading " + apkfilename + " from cache")
 462                 usecache = True
 463             else:
 464                 logging.debug("Ignoring stale cache data for " + apkfilename)
 465
 466         if not usecache:
 467             logging.debug("Processing " + apkfilename)
 468             apk = {}
 469             apk['apkname'] = apkfilename
 470             apk['sha256'] = shasum
 471             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 472             if os.path.exists(os.path.join(repodir, srcfilename)):
 473                 apk['srcname'] = srcfilename
 474             apk['size'] = os.path.getsize(apkfile)
 475             apk['permissions'] = set()
 476             apk['features'] = set()
 477             apk['icons_src'] = {}
 478             apk['icons'] = {}
 479             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 480             if p.returncode != 0:
 481                 if options.delete_unknown:
 482                     if os.path.exists(apkfile):
 483                         logging.error("Failed to get apk information, deleting " + apkfile)
 484                         os.remove(apkfile)
 485                     else:
 486                         logging.error("Could not find {0} to remove it".format(apkfile))
 487                 else:
 488                     logging.error("Failed to get apk information, skipping " + apkfile)
 489                 continue
 490             for line in p.output.splitlines():
 491                 if line.startswith("package:"):
 492                     try:
 493                         apk['id'] = re.match(name_pat, line).group(1)
 494                         apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
 495                         apk['version'] = re.match(vername_pat, line).group(1)
 496                     except Exception as e:
 497                         logging.error("Package matching failed: " + str(e))
 498                         logging.info("Line was: " + line)
 499                         sys.exit(1)
 500                 elif line.startswith("application:"):
 501                     apk['name'] = re.match(label_pat, line).group(1)
 502                     # Keep path to non-dpi icon in case we need it
 503                     match = re.match(icon_pat_nodpi, line)
 504                     if match:
 505                         apk['icons_src']['-1'] = match.group(1)
 506                 elif line.startswith("launchable-activity:"):
 507                     # Only use launchable-activity as fallback to application
 508                     if not apk['name']:
 509                         apk['name'] = re.match(label_pat, line).group(1)
 510                     if '-1' not in apk['icons_src']:
 511                         match = re.match(icon_pat_nodpi, line)
 512                         if match:
 513                             apk['icons_src']['-1'] = match.group(1)
 514                 elif line.startswith("application-icon-"):
 515                     match = re.match(icon_pat, line)
 516                     if match:
 517                         density = match.group(1)
 518                         path = match.group(2)
 519                         apk['icons_src'][density] = path
 520                 elif line.startswith("sdkVersion:"):
 521                     m = re.match(sdkversion_pat, line)
 522                     if m is None:
 523                         logging.error(line.replace('sdkVersion:', '')
 524                                       + ' is not a valid minSdkVersion!')
 525                     else:
 526                         apk['sdkversion'] = m.group(1)
 527                 elif line.startswith("maxSdkVersion:"):
 528                     apk['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 529                 elif line.startswith("native-code:"):
 530                     apk['nativecode'] = []
 531                     for arch in line[13:].split(' '):
 532                         apk['nativecode'].append(arch[1:-1])
 533                 elif line.startswith("uses-permission:"):
 534                     perm = re.match(string_pat, line).group(1)
 535                     if perm.startswith("android.permission."):
 536                         perm = perm[19:]
 537                     apk['permissions'].add(perm)
 538                 elif line.startswith("uses-feature:"):
 539                     perm = re.match(string_pat, line).group(1)
 540                     # Filter out this, it's only added with the latest SDK tools and
 541                     # causes problems for lots of apps.
 542                     if perm != "android.hardware.screen.portrait" \
 543                             and perm != "android.hardware.screen.landscape":
 544                         if perm.startswith("android.feature."):
 545                             perm = perm[16:]
 546                         apk['features'].add(perm)
 547
 548             if 'sdkversion' not in apk:
 549                 logging.warn("No SDK version information found in {0}".format(apkfile))
 550                 apk['sdkversion'] = 0
 551
 552             # Check for debuggable apks...
 553             if common.isApkDebuggable(apkfile, config):
 554                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 555
 556             # Get the signature (or md5 of, to be precise)...
 557             logging.debug('Getting signature of {0}'.format(apkfile))
 558             apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 559             if not apk['sig']:
 560                 logging.critical("Failed to get apk signature")
 561                 sys.exit(1)
 562
 563             apkzip = zipfile.ZipFile(apkfile, 'r')
 564
 565             # if an APK has files newer than the system time, suggest updating
 566             # the system clock.  This is useful for offline systems, used for
 567             # signing, which do not have another source of clock sync info. It
 568             # has to be more than 24 hours newer because ZIP/APK files do not
 569             # store timezone info
 570             manifest = apkzip.getinfo('AndroidManifest.xml')
 571             dt_obj = datetime(*manifest.date_time)
 572             checkdt = dt_obj - timedelta(1)
 573             if datetime.today() < checkdt:
 574                 logging.warn('System clock is older than manifest in: '
 575                              + apkfilename + '\nSet clock to that time using:\n'
 576                              + 'sudo date -s "' + str(dt_obj) + '"')
 577
 578             iconfilename = "%s.%s.png" % (
 579                 apk['id'],
 580                 apk['versioncode'])
 581
 582             # Extract the icon file...
 583             empty_densities = []
 584             for density in screen_densities:
 585                 if density not in apk['icons_src']:
 586                     empty_densities.append(density)
 587                     continue
 588                 iconsrc = apk['icons_src'][density]
 589                 icon_dir = get_icon_dir(repodir, density)
 590                 icondest = os.path.join(icon_dir, iconfilename)
 591
 592                 try:
 593                     with open(icondest, 'wb') as f:
 594                         f.write(apkzip.read(iconsrc))
 595                     apk['icons'][density] = iconfilename
 596
 597                 except:
 598                     logging.warn("Error retrieving icon file")
 599                     del apk['icons'][density]
 600                     del apk['icons_src'][density]
 601                     empty_densities.append(density)
 602
 603             if '-1' in apk['icons_src']:
 604                 iconsrc = apk['icons_src']['-1']
 605                 iconpath = os.path.join(
 606                     get_icon_dir(repodir, '0'), iconfilename)
 607                 with open(iconpath, 'wb') as f:
 608                     f.write(apkzip.read(iconsrc))
 609                 try:
 610                     im = Image.open(iconpath)
 611                     dpi = px_to_dpi(im.size[0])
 612                     for density in screen_densities:
 613                         if density in apk['icons']:
 614                             break
 615                         if density == screen_densities[-1] or dpi >= int(density):
 616                             apk['icons'][density] = iconfilename
 617                             shutil.move(iconpath,
 618                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 619                             empty_densities.remove(density)
 620                             break
 621                 except Exception as e:
 622                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 623
 624             if apk['icons']:
 625                 apk['icon'] = iconfilename
 626
 627             apkzip.close()
 628
 629             # First try resizing down to not lose quality
 630             last_density = None
 631             for density in screen_densities:
 632                 if density not in empty_densities:
 633                     last_density = density
 634                     continue
 635                 if last_density is None:
 636                     continue
 637                 logging.debug("Density %s not available, resizing down from %s"
 638                               % (density, last_density))
 639
 640                 last_iconpath = os.path.join(
 641                     get_icon_dir(repodir, last_density), iconfilename)
 642                 iconpath = os.path.join(
 643                     get_icon_dir(repodir, density), iconfilename)
 644                 try:
 645                     im = Image.open(last_iconpath)
 646                 except:
 647                     logging.warn("Invalid image file at %s" % last_iconpath)
 648                     continue
 649
 650                 size = dpi_to_px(density)
 651
 652                 im.thumbnail((size, size), Image.ANTIALIAS)
 653                 im.save(iconpath, "PNG")
 654                 empty_densities.remove(density)
 655
 656             # Then just copy from the highest resolution available
 657             last_density = None
 658             for density in reversed(screen_densities):
 659                 if density not in empty_densities:
 660                     last_density = density
 661                     continue
 662                 if last_density is None:
 663                     continue
 664                 logging.debug("Density %s not available, copying from lower density %s"
 665                               % (density, last_density))
 666
 667                 shutil.copyfile(
 668                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 669                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 670
 671                 empty_densities.remove(density)
 672
 673             for density in screen_densities:
 674                 icon_dir = get_icon_dir(repodir, density)
 675                 icondest = os.path.join(icon_dir, iconfilename)
 676                 resize_icon(icondest, density)
 677
 678             # Copy from icons-mdpi to icons since mdpi is the baseline density
 679             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 680             if os.path.isfile(baseline):
 681                 apk['icons']['0'] = iconfilename
 682                 shutil.copyfile(baseline,
 683                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 684
 685             # Record in known apks, getting the added date at the same time..
 686             added = knownapks.recordapk(apk['apkname'], apk['id'])
 687             if added:
 688                 apk['added'] = added
 689
 690             apkcache[apkfilename] = apk
 691             cachechanged = True
 692
 693         apks.append(apk)
 694
 695     return apks, cachechanged
 696
 697
 698 repo_pubkey_fingerprint = None
 699
 700
 701 # Generate a certificate fingerprint the same way keytool does it
 702 # (but with slightly different formatting)
 703 def cert_fingerprint(data):
 704     digest = hashlib.sha256(data).digest()
 705     ret = []
 706     ret.append(' '.join("%02X" % ord(b) for b in digest))
 707     return " ".join(ret)
 708
 709
 710 def extract_pubkey():
 711     global repo_pubkey_fingerprint
 712     if 'repo_pubkey' in config:
 713         pubkey = unhexlify(config['repo_pubkey'])
 714     else:
 715         p = FDroidPopen([config['keytool'], '-exportcert',
 716                          '-alias', config['repo_keyalias'],
 717                          '-keystore', config['keystore'],
 718                          '-storepass:file', config['keystorepassfile']]
 719                         + config['smartcardoptions'], output=False)
 720         if p.returncode != 0 or len(p.output) < 20:
 721             msg = "Failed to get repo pubkey!"
 722             if config['keystore'] == 'NONE':
 723                 msg += ' Is your crypto smartcard plugged in?'
 724             logging.critical(msg)
 725             sys.exit(1)
 726         pubkey = p.output
 727     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 728     return hexlify(pubkey)
 729
 730
 731 def make_index(apps, sortedids, apks, repodir, archive, categories):
 732     """Make a repo index.
 733
 734     :param apps: fully populated apps list
 735     :param apks: full populated apks list
 736     :param repodir: the repo directory
 737     :param archive: True if this is the archive repo, False if it's the
 738                     main one.
 739     :param categories: list of categories
 740     """
 741
 742     doc = Document()
 743
 744     def addElement(name, value, doc, parent):
 745         el = doc.createElement(name)
 746         el.appendChild(doc.createTextNode(value))
 747         parent.appendChild(el)
 748
 749     def addElementNonEmpty(name, value, doc, parent):
 750         if not value:
 751             return
 752         addElement(name, value, doc, parent)
 753
 754     def addElementCDATA(name, value, doc, parent):
 755         el = doc.createElement(name)
 756         el.appendChild(doc.createCDATASection(value))
 757         parent.appendChild(el)
 758
 759     root = doc.createElement("fdroid")
 760     doc.appendChild(root)
 761
 762     repoel = doc.createElement("repo")
 763
 764     mirrorcheckfailed = False
 765     for mirror in config.get('mirrors', []):
 766         base = os.path.basename(urlparse.urlparse(mirror).path.rstrip('/'))
 767         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
 768             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 769             mirrorcheckfailed = True
 770     if mirrorcheckfailed:
 771         sys.exit(1)
 772
 773     if archive:
 774         repoel.setAttribute("name", config['archive_name'])
 775         if config['repo_maxage'] != 0:
 776             repoel.setAttribute("maxage", str(config['repo_maxage']))
 777         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 778         repoel.setAttribute("url", config['archive_url'])
 779         addElement('description', config['archive_description'], doc, repoel)
 780         urlbasepath = os.path.basename(urlparse.urlparse(config['archive_url']).path)
 781         for mirror in config.get('mirrors', []):
 782             addElement('mirror', urlparse.urljoin(mirror, urlbasepath), doc, repoel)
 783
 784     else:
 785         repoel.setAttribute("name", config['repo_name'])
 786         if config['repo_maxage'] != 0:
 787             repoel.setAttribute("maxage", str(config['repo_maxage']))
 788         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 789         repoel.setAttribute("url", config['repo_url'])
 790         addElement('description', config['repo_description'], doc, repoel)
 791         urlbasepath = os.path.basename(urlparse.urlparse(config['repo_url']).path)
 792         for mirror in config.get('mirrors', []):
 793             addElement('mirror', urlparse.urljoin(mirror, urlbasepath), doc, repoel)
 794
 795     repoel.setAttribute("version", "15")
 796     repoel.setAttribute("timestamp", str(int(time.time())))
 797
 798     nosigningkey = False
 799     if not options.nosign:
 800         if 'repo_keyalias' not in config:
 801             nosigningkey = True
 802             logging.critical("'repo_keyalias' not found in config.py!")
 803         if 'keystore' not in config:
 804             nosigningkey = True
 805             logging.critical("'keystore' not found in config.py!")
 806         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 807             nosigningkey = True
 808             logging.critical("'keystorepass' not found in config.py!")
 809         if 'keypass' not in config and 'keypassfile' not in config:
 810             nosigningkey = True
 811             logging.critical("'keypass' not found in config.py!")
 812         if not os.path.exists(config['keystore']):
 813             nosigningkey = True
 814             logging.critical("'" + config['keystore'] + "' does not exist!")
 815         if nosigningkey:
 816             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 817             logging.warning("\tfdroid update --create-key")
 818             sys.exit(1)
 819
 820     repoel.setAttribute("pubkey", extract_pubkey())
 821     root.appendChild(repoel)
 822
 823     for appid in sortedids:
 824         app = apps[appid]
 825
 826         if app.Disabled is not None:
 827             continue
 828
 829         # Get a list of the apks for this app...
 830         apklist = []
 831         for apk in apks:
 832             if apk['id'] == appid:
 833                 apklist.append(apk)
 834
 835         if len(apklist) == 0:
 836             continue
 837
 838         apel = doc.createElement("application")
 839         apel.setAttribute("id", app.id)
 840         root.appendChild(apel)
 841
 842         addElement('id', app.id, doc, apel)
 843         if app.added:
 844             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
 845         if app.lastupdated:
 846             addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
 847         addElement('name', app.Name, doc, apel)
 848         addElement('summary', app.Summary, doc, apel)
 849         if app.icon:
 850             addElement('icon', app.icon, doc, apel)
 851
 852         def linkres(appid):
 853             if appid in apps:
 854                 return ("fdroid.app:" + appid, apps[appid].Name)
 855             raise MetaDataException("Cannot resolve app id " + appid)
 856
 857         addElement('desc',
 858                    metadata.description_html(app.Description, linkres),
 859                    doc, apel)
 860         addElement('license', app.License, doc, apel)
 861         if app.Categories:
 862             addElement('categories', ','.join(app.Categories), doc, apel)
 863             # We put the first (primary) category in LAST, which will have
 864             # the desired effect of making clients that only understand one
 865             # category see that one.
 866             addElement('category', app.Categories[0], doc, apel)
 867         addElement('web', app.WebSite, doc, apel)
 868         addElement('source', app.SourceCode, doc, apel)
 869         addElement('tracker', app.IssueTracker, doc, apel)
 870         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 871         addElementNonEmpty('author', app.AuthorName, doc, apel)
 872         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 873         addElementNonEmpty('donate', app.Donate, doc, apel)
 874         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 875         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 876         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 877
 878         # These elements actually refer to the current version (i.e. which
 879         # one is recommended. They are historically mis-named, and need
 880         # changing, but stay like this for now to support existing clients.
 881         addElement('marketversion', app.CurrentVersion, doc, apel)
 882         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 883
 884         if app.AntiFeatures:
 885             af = app.AntiFeatures
 886             if af:
 887                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 888         if app.Provides:
 889             pv = app.Provides.split(',')
 890             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 891         if app.RequiresRoot:
 892             addElement('requirements', 'root', doc, apel)
 893
 894         # Sort the apk list into version order, just so the web site
 895         # doesn't have to do any work by default...
 896         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 897
 898         # Check for duplicates - they will make the client unhappy...
 899         for i in range(len(apklist) - 1):
 900             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 901                 logging.critical("duplicate versions: '%s' - '%s'" % (
 902                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 903                 sys.exit(1)
 904
 905         current_version_code = 0
 906         current_version_file = None
 907         for apk in apklist:
 908             # find the APK for the "Current Version"
 909             if current_version_code < apk['versioncode']:
 910                 current_version_code = apk['versioncode']
 911             if current_version_code < int(app.CurrentVersionCode):
 912                 current_version_file = apk['apkname']
 913
 914             apkel = doc.createElement("package")
 915             apel.appendChild(apkel)
 916             addElement('version', apk['version'], doc, apkel)
 917             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 918             addElement('apkname', apk['apkname'], doc, apkel)
 919             if 'srcname' in apk:
 920                 addElement('srcname', apk['srcname'], doc, apkel)
 921             for hash_type in ['sha256']:
 922                 if hash_type not in apk:
 923                     continue
 924                 hashel = doc.createElement("hash")
 925                 hashel.setAttribute("type", hash_type)
 926                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 927                 apkel.appendChild(hashel)
 928             addElement('sig', apk['sig'], doc, apkel)
 929             addElement('size', str(apk['size']), doc, apkel)
 930             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 931             if 'maxsdkversion' in apk:
 932                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 933             if 'added' in apk:
 934                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 935             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
 936             if 'nativecode' in apk:
 937                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 938             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
 939
 940         if current_version_file is not None \
 941                 and config['make_current_version_link'] \
 942                 and repodir == 'repo':  # only create these
 943             namefield = config['current_version_name_source']
 944             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
 945             apklinkname = sanitized_name + '.apk'
 946             current_version_path = os.path.join(repodir, current_version_file)
 947             if os.path.islink(apklinkname):
 948                 os.remove(apklinkname)
 949             os.symlink(current_version_path, apklinkname)
 950             # also symlink gpg signature, if it exists
 951             for extension in ('.asc', '.sig'):
 952                 sigfile_path = current_version_path + extension
 953                 if os.path.exists(sigfile_path):
 954                     siglinkname = apklinkname + extension
 955                     if os.path.islink(siglinkname):
 956                         os.remove(siglinkname)
 957                     os.symlink(sigfile_path, siglinkname)
 958
 959     if options.pretty:
 960         output = doc.toprettyxml()
 961     else:
 962         output = doc.toxml()
 963
 964     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 965         f.write(output)
 966
 967     if 'repo_keyalias' in config:
 968
 969         if options.nosign:
 970             logging.info("Creating unsigned index in preparation for signing")
 971         else:
 972             logging.info("Creating signed index with this key (SHA256):")
 973             logging.info("%s" % repo_pubkey_fingerprint)
 974
 975         # Create a jar of the index...
 976         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
 977         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 978         if p.returncode != 0:
 979             logging.critical("Failed to create {0}".format(jar_output))
 980             sys.exit(1)
 981
 982         # Sign the index...
 983         signed = os.path.join(repodir, 'index.jar')
 984         if options.nosign:
 985             # Remove old signed index if not signing
 986             if os.path.exists(signed):
 987                 os.remove(signed)
 988         else:
 989             args = [config['jarsigner'], '-keystore', config['keystore'],
 990                     '-storepass:file', config['keystorepassfile'],
 991                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
 992                     signed, config['repo_keyalias']]
 993             if config['keystore'] == 'NONE':
 994                 args += config['smartcardoptions']
 995             else:  # smardcards never use -keypass
 996                 args += ['-keypass:file', config['keypassfile']]
 997             p = FDroidPopen(args)
 998             if p.returncode != 0:
 999                 logging.critical("Failed to sign index")
1000                 sys.exit(1)
1001
1002     # Copy the repo icon into the repo directory...
1003     icon_dir = os.path.join(repodir, 'icons')
1004     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1005     shutil.copyfile(config['repo_icon'], iconfilename)
1006
1007     # Write a category list in the repo to allow quick access...
1008     catdata = ''
1009     for cat in categories:
1010         catdata += cat + '\n'
1011     with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
1012         f.write(catdata)
1013
1014
1015 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1016
1017     for appid, app in apps.iteritems():
1018
1019         if app.ArchivePolicy:
1020             keepversions = int(app.ArchivePolicy[:-9])
1021         else:
1022             keepversions = defaultkeepversions
1023
1024         def filter_apk_list_sorted(apk_list):
1025             res = []
1026             for apk in apk_list:
1027                 if apk['id'] == appid:
1028                     res.append(apk)
1029
1030             # Sort the apk list by version code. First is highest/newest.
1031             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1032
1033         def move_file(from_dir, to_dir, filename, ignore_missing):
1034             from_path = os.path.join(from_dir, filename)
1035             if ignore_missing and not os.path.exists(from_path):
1036                 return
1037             to_path = os.path.join(to_dir, filename)
1038             shutil.move(from_path, to_path)
1039
1040         if len(apks) > keepversions:
1041             apklist = filter_apk_list_sorted(apks)
1042             # Move back the ones we don't want.
1043             for apk in apklist[keepversions:]:
1044                 logging.info("Moving " + apk['apkname'] + " to archive")
1045                 move_file(repodir, archivedir, apk['apkname'], False)
1046                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1047                 for density in all_screen_densities:
1048                     repo_icon_dir = get_icon_dir(repodir, density)
1049                     archive_icon_dir = get_icon_dir(archivedir, density)
1050                     if density not in apk['icons']:
1051                         continue
1052                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1053                 if 'srcname' in apk:
1054                     move_file(repodir, archivedir, apk['srcname'], False)
1055                 archapks.append(apk)
1056                 apks.remove(apk)
1057         elif len(apks) < keepversions and len(archapks) > 0:
1058             required = keepversions - len(apks)
1059             archapklist = filter_apk_list_sorted(archapks)
1060             # Move forward the ones we want again.
1061             for apk in archapklist[:required]:
1062                 logging.info("Moving " + apk['apkname'] + " from archive")
1063                 move_file(archivedir, repodir, apk['apkname'], False)
1064                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1065                 for density in all_screen_densities:
1066                     repo_icon_dir = get_icon_dir(repodir, density)
1067                     archive_icon_dir = get_icon_dir(archivedir, density)
1068                     if density not in apk['icons']:
1069                         continue
1070                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1071                 if 'srcname' in apk:
1072                     move_file(archivedir, repodir, apk['srcname'], False)
1073                 archapks.remove(apk)
1074                 apks.append(apk)
1075
1076
1077 def add_apks_to_per_app_repos(repodir, apks):
1078     apks_per_app = dict()
1079     for apk in apks:
1080         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1081         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1082         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1083         apks_per_app[apk['id']] = apk
1084
1085         if not os.path.exists(apk['per_app_icons']):
1086             logging.info('Adding new repo for only ' + apk['id'])
1087             os.makedirs(apk['per_app_icons'])
1088
1089         apkpath = os.path.join(repodir, apk['apkname'])
1090         shutil.copy(apkpath, apk['per_app_repo'])
1091         apksigpath = apkpath + '.sig'
1092         if os.path.exists(apksigpath):
1093             shutil.copy(apksigpath, apk['per_app_repo'])
1094         apkascpath = apkpath + '.asc'
1095         if os.path.exists(apkascpath):
1096             shutil.copy(apkascpath, apk['per_app_repo'])
1097
1098
1099 config = None
1100 options = None
1101
1102
1103 def main():
1104
1105     global config, options
1106
1107     # Parse command line...
1108     parser = ArgumentParser()
1109     common.setup_global_opts(parser)
1110     parser.add_argument("--create-key", action="store_true", default=False,
1111                         help="Create a repo signing key in a keystore")
1112     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1113                         help="Create skeleton metadata files that are missing")
1114     parser.add_argument("--delete-unknown", action="store_true", default=False,
1115                         help="Delete APKs without metadata from the repo")
1116     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1117                         help="Report on build data status")
1118     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1119                         help="Interactively ask about things that need updating.")
1120     parser.add_argument("-I", "--icons", action="store_true", default=False,
1121                         help="Resize all the icons exceeding the max pixel size and exit")
1122     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1123                         help="Specify editor to use in interactive mode. Default " +
1124                         "is /etc/alternatives/editor")
1125     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1126                         help="Update the wiki")
1127     parser.add_argument("--pretty", action="store_true", default=False,
1128                         help="Produce human-readable index.xml")
1129     parser.add_argument("--clean", action="store_true", default=False,
1130                         help="Clean update - don't uses caches, reprocess all apks")
1131     parser.add_argument("--nosign", action="store_true", default=False,
1132                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1133     options = parser.parse_args()
1134
1135     config = common.read_config(options)
1136
1137     if not ('jarsigner' in config and 'keytool' in config):
1138         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1139         sys.exit(1)
1140
1141     repodirs = ['repo']
1142     if config['archive_older'] != 0:
1143         repodirs.append('archive')
1144         if not os.path.exists('archive'):
1145             os.mkdir('archive')
1146
1147     if options.icons:
1148         resize_all_icons(repodirs)
1149         sys.exit(0)
1150
1151     # check that icons exist now, rather than fail at the end of `fdroid update`
1152     for k in ['repo_icon', 'archive_icon']:
1153         if k in config:
1154             if not os.path.exists(config[k]):
1155                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1156                 sys.exit(1)
1157
1158     # if the user asks to create a keystore, do it now, reusing whatever it can
1159     if options.create_key:
1160         if os.path.exists(config['keystore']):
1161             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1162             logging.critical("\t'" + config['keystore'] + "'")
1163             sys.exit(1)
1164
1165         if 'repo_keyalias' not in config:
1166             config['repo_keyalias'] = socket.getfqdn()
1167             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1168         if 'keydname' not in config:
1169             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1170             common.write_to_config(config, 'keydname', config['keydname'])
1171         if 'keystore' not in config:
1172             config['keystore'] = common.default_config.keystore
1173             common.write_to_config(config, 'keystore', config['keystore'])
1174
1175         password = common.genpassword()
1176         if 'keystorepass' not in config:
1177             config['keystorepass'] = password
1178             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1179         if 'keypass' not in config:
1180             config['keypass'] = password
1181             common.write_to_config(config, 'keypass', config['keypass'])
1182         common.genkeystore(config)
1183
1184     # Get all apps...
1185     apps = metadata.read_metadata()
1186
1187     # Generate a list of categories...
1188     categories = set()
1189     for app in apps.itervalues():
1190         categories.update(app.Categories)
1191
1192     # Read known apks data (will be updated and written back when we've finished)
1193     knownapks = common.KnownApks()
1194
1195     # Gather information about all the apk files in the repo directory, using
1196     # cached data if possible.
1197     apkcachefile = os.path.join('tmp', 'apkcache')
1198     if not options.clean and os.path.exists(apkcachefile):
1199         with open(apkcachefile, 'rb') as cf:
1200             apkcache = pickle.load(cf)
1201     else:
1202         apkcache = {}
1203
1204     delete_disabled_builds(apps, apkcache, repodirs)
1205
1206     # Scan all apks in the main repo
1207     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks)
1208
1209     # Generate warnings for apk's with no metadata (or create skeleton
1210     # metadata files, if requested on the command line)
1211     newmetadata = False
1212     for apk in apks:
1213         if apk['id'] not in apps:
1214             if options.create_metadata:
1215                 if 'name' not in apk:
1216                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1217                     continue
1218                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1219                 f.write("License:Unknown\n")
1220                 f.write("Web Site:\n")
1221                 f.write("Source Code:\n")
1222                 f.write("Issue Tracker:\n")
1223                 f.write("Changelog:\n")
1224                 f.write("Summary:" + apk['name'] + "\n")
1225                 f.write("Description:\n")
1226                 f.write(apk['name'] + "\n")
1227                 f.write(".\n")
1228                 f.close()
1229                 logging.info("Generated skeleton metadata for " + apk['id'])
1230                 newmetadata = True
1231             else:
1232                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1233                 if options.delete_unknown:
1234                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1235                     rmf = os.path.join(repodirs[0], apk['apkname'])
1236                     if not os.path.exists(rmf):
1237                         logging.error("Could not find {0} to remove it".format(rmf))
1238                     else:
1239                         os.remove(rmf)
1240                 else:
1241                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1242
1243     # update the metadata with the newly created ones included
1244     if newmetadata:
1245         apps = metadata.read_metadata()
1246
1247     # Scan the archive repo for apks as well
1248     if len(repodirs) > 1:
1249         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1250         if cc:
1251             cachechanged = True
1252     else:
1253         archapks = []
1254
1255     # Some information from the apks needs to be applied up to the application
1256     # level. When doing this, we use the info from the most recent version's apk.
1257     # We deal with figuring out when the app was added and last updated at the
1258     # same time.
1259     for appid, app in apps.iteritems():
1260         bestver = 0
1261         for apk in apks + archapks:
1262             if apk['id'] == appid:
1263                 if apk['versioncode'] > bestver:
1264                     bestver = apk['versioncode']
1265                     bestapk = apk
1266
1267                 if 'added' in apk:
1268                     if not app.added or apk['added'] < app.added:
1269                         app.added = apk['added']
1270                     if not app.lastupdated or apk['added'] > app.lastupdated:
1271                         app.lastupdated = apk['added']
1272
1273         if not app.added:
1274             logging.debug("Don't know when " + appid + " was added")
1275         if not app.lastupdated:
1276             logging.debug("Don't know when " + appid + " was last updated")
1277
1278         if bestver == 0:
1279             if app.Name is None:
1280                 app.Name = app.AutoName or appid
1281             app.icon = None
1282             logging.debug("Application " + appid + " has no packages")
1283         else:
1284             if app.Name is None:
1285                 app.Name = bestapk['name']
1286             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1287             if app.CurrentVersionCode is None:
1288                 app.CurrentVersionCode = str(bestver)
1289
1290     # Sort the app list by name, then the web site doesn't have to by default.
1291     # (we had to wait until we'd scanned the apks to do this, because mostly the
1292     # name comes from there!)
1293     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid].Name.upper())
1294
1295     # APKs are placed into multiple repos based on the app package, providing
1296     # per-app subscription feeds for nightly builds and things like it
1297     if config['per_app_repos']:
1298         add_apks_to_per_app_repos(repodirs[0], apks)
1299         for appid, app in apps.iteritems():
1300             repodir = os.path.join(appid, 'fdroid', 'repo')
1301             appdict = dict()
1302             appdict[appid] = app
1303             if os.path.isdir(repodir):
1304                 make_index(appdict, [appid], apks, repodir, False, categories)
1305             else:
1306                 logging.info('Skipping index generation for ' + appid)
1307         return
1308
1309     if len(repodirs) > 1:
1310         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1311
1312     # Make the index for the main repo...
1313     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1314
1315     # If there's an archive repo,  make the index for it. We already scanned it
1316     # earlier on.
1317     if len(repodirs) > 1:
1318         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1319
1320     if config['update_stats']:
1321
1322         # Update known apks info...
1323         knownapks.writeifchanged()
1324
1325         # Generate latest apps data for widget
1326         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1327             data = ''
1328             for line in file(os.path.join('stats', 'latestapps.txt')):
1329                 appid = line.rstrip()
1330                 data += appid + "\t"
1331                 app = apps[appid]
1332                 data += app.Name + "\t"
1333                 if app.icon is not None:
1334                     data += app.icon + "\t"
1335                 data += app.License + "\n"
1336             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
1337                 f.write(data)
1338
1339     if cachechanged:
1340         with open(apkcachefile, 'wb') as cf:
1341             pickle.dump(apkcache, cf)
1342
1343     # Update the wiki...
1344     if options.wiki:
1345         update_wiki(apps, sortedids, apks + archapks)
1346
1347     logging.info("Finished.")
1348
1349 if __name__ == "__main__":
1350     main()