fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import socket
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 import urlparse
  31 from datetime import datetime, timedelta
  32 from xml.dom.minidom import Document
  33 from argparse import ArgumentParser
  34 import time
  35 from pyasn1.error import PyAsn1Error
  36 from pyasn1.codec.der import decoder, encoder
  37 from pyasn1_modules import rfc2315
  38 from hashlib import md5
  39 from binascii import hexlify, unhexlify
  40
  41 from PIL import Image
  42 import logging
  43
  44 import common
  45 import metadata
  46 from common import FDroidPopen, SdkToolsPopen
  47 from metadata import MetaDataException
  48
  49 screen_densities = ['640', '480', '320', '240', '160', '120']
  50
  51 all_screen_densities = ['0'] + screen_densities
  52
  53
  54 def dpi_to_px(density):
  55     return (int(density) * 48) / 160
  56
  57
  58 def px_to_dpi(px):
  59     return (int(px) * 160) / 48
  60
  61
  62 def get_icon_dir(repodir, density):
  63     if density == '0':
  64         return os.path.join(repodir, "icons")
  65     return os.path.join(repodir, "icons-%s" % density)
  66
  67
  68 def get_icon_dirs(repodir):
  69     for density in screen_densities:
  70         yield get_icon_dir(repodir, density)
  71
  72
  73 def get_all_icon_dirs(repodir):
  74     for density in all_screen_densities:
  75         yield get_icon_dir(repodir, density)
  76
  77
  78 def update_wiki(apps, sortedids, apks):
  79     """Update the wiki
  80
  81     :param apps: fully populated list of all applications
  82     :param apks: all apks, except...
  83     """
  84     logging.info("Updating wiki")
  85     wikicat = 'Apps'
  86     wikiredircat = 'App Redirects'
  87     import mwclient
  88     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  89                          path=config['wiki_path'])
  90     site.login(config['wiki_user'], config['wiki_password'])
  91     generated_pages = {}
  92     generated_redirects = {}
  93
  94     for appid in sortedids:
  95         app = apps[appid]
  96
  97         wikidata = ''
  98         if app.Disabled:
  99             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 100         if app.AntiFeatures:
 101             for af in app.AntiFeatures:
 102                 wikidata += '{{AntiFeature|' + af + '}}\n'
 103         if app.RequiresRoot:
 104             requiresroot = 'Yes'
 105         else:
 106             requiresroot = 'No'
 107         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 108             appid,
 109             app.Name,
 110             time.strftime('%Y-%m-%d', app.added) if app.added else '',
 111             time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
 112             app.SourceCode,
 113             app.IssueTracker,
 114             app.WebSite,
 115             app.Changelog,
 116             app.Donate,
 117             app.FlattrID,
 118             app.Bitcoin,
 119             app.Litecoin,
 120             app.License,
 121             requiresroot,
 122             app.AuthorName,
 123             app.AuthorEmail)
 124
 125         if app.Provides:
 126             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 127
 128         wikidata += app.Summary
 129         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 130
 131         wikidata += "=Description=\n"
 132         wikidata += metadata.description_wiki(app.Description) + "\n"
 133
 134         wikidata += "=Maintainer Notes=\n"
 135         if app.MaintainerNotes:
 136             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 137         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 138
 139         # Get a list of all packages for this application...
 140         apklist = []
 141         gotcurrentver = False
 142         cantupdate = False
 143         buildfails = False
 144         for apk in apks:
 145             if apk['id'] == appid:
 146                 if str(apk['versioncode']) == app.CurrentVersionCode:
 147                     gotcurrentver = True
 148                 apklist.append(apk)
 149         # Include ones we can't build, as a special case...
 150         for build in app.builds:
 151             if build.disable:
 152                 if build.vercode == app.CurrentVersionCode:
 153                     cantupdate = True
 154                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 155                 apklist.append({'versioncode': int(build.vercode),
 156                                 'version': build.version,
 157                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 158                                 })
 159             else:
 160                 builtit = False
 161                 for apk in apklist:
 162                     if apk['versioncode'] == int(build.vercode):
 163                         builtit = True
 164                         break
 165                 if not builtit:
 166                     buildfails = True
 167                     apklist.append({'versioncode': int(build.vercode),
 168                                     'version': build.version,
 169                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
 170                                     })
 171         if app.CurrentVersionCode == '0':
 172             cantupdate = True
 173         # Sort with most recent first...
 174         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 175
 176         wikidata += "=Versions=\n"
 177         if len(apklist) == 0:
 178             wikidata += "We currently have no versions of this app available."
 179         elif not gotcurrentver:
 180             wikidata += "We don't have the current version of this app."
 181         else:
 182             wikidata += "We have the current version of this app."
 183         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 184         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 185         if len(app.NoSourceSince) > 0:
 186             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 187         if len(app.CurrentVersion) > 0:
 188             wikidata += "The current (recommended) version is " + app.CurrentVersion
 189             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 190         validapks = 0
 191         for apk in apklist:
 192             wikidata += "==" + apk['version'] + "==\n"
 193
 194             if 'buildproblem' in apk:
 195                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 196             else:
 197                 validapks += 1
 198                 wikidata += "This version is built and signed by "
 199                 if 'srcname' in apk:
 200                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 201                 else:
 202                     wikidata += "the original developer.\n\n"
 203             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 204
 205         wikidata += '\n[[Category:' + wikicat + ']]\n'
 206         if len(app.NoSourceSince) > 0:
 207             wikidata += '\n[[Category:Apps missing source code]]\n'
 208         if validapks == 0 and not app.Disabled:
 209             wikidata += '\n[[Category:Apps with no packages]]\n'
 210         if cantupdate and not app.Disabled:
 211             wikidata += "\n[[Category:Apps we cannot update]]\n"
 212         if buildfails and not app.Disabled:
 213             wikidata += "\n[[Category:Apps with failing builds]]\n"
 214         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 215             wikidata += '\n[[Category:Apps to Update]]\n'
 216         if app.Disabled:
 217             wikidata += '\n[[Category:Apps that are disabled]]\n'
 218         if app.UpdateCheckMode == 'None' and not app.Disabled:
 219             wikidata += '\n[[Category:Apps with no update check]]\n'
 220         for appcat in app.Categories:
 221             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 222
 223         # We can't have underscores in the page name, even if they're in
 224         # the package ID, because MediaWiki messes with them...
 225         pagename = appid.replace('_', ' ')
 226
 227         # Drop a trailing newline, because mediawiki is going to drop it anyway
 228         # and it we don't we'll think the page has changed when it hasn't...
 229         if wikidata.endswith('\n'):
 230             wikidata = wikidata[:-1]
 231
 232         generated_pages[pagename] = wikidata
 233
 234         # Make a redirect from the name to the ID too, unless there's
 235         # already an existing page with the name and it isn't a redirect.
 236         noclobber = False
 237         apppagename = app.Name.replace('_', ' ')
 238         apppagename = apppagename.replace('{', '')
 239         apppagename = apppagename.replace('}', ' ')
 240         apppagename = apppagename.replace(':', ' ')
 241         # Drop double spaces caused mostly by replacing ':' above
 242         apppagename = apppagename.replace('  ', ' ')
 243         for expagename in site.allpages(prefix=apppagename,
 244                                         filterredir='nonredirects',
 245                                         generator=False):
 246             if expagename == apppagename:
 247                 noclobber = True
 248         # Another reason not to make the redirect page is if the app name
 249         # is the same as it's ID, because that will overwrite the real page
 250         # with an redirect to itself! (Although it seems like an odd
 251         # scenario this happens a lot, e.g. where there is metadata but no
 252         # builds or binaries to extract a name from.
 253         if apppagename == pagename:
 254             noclobber = True
 255         if not noclobber:
 256             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 257
 258     for tcat, genp in [(wikicat, generated_pages),
 259                        (wikiredircat, generated_redirects)]:
 260         catpages = site.Pages['Category:' + tcat]
 261         existingpages = []
 262         for page in catpages:
 263             existingpages.append(page.name)
 264             if page.name in genp:
 265                 pagetxt = page.edit()
 266                 if pagetxt != genp[page.name]:
 267                     logging.debug("Updating modified page " + page.name)
 268                     page.save(genp[page.name], summary='Auto-updated')
 269                 else:
 270                     logging.debug("Page " + page.name + " is unchanged")
 271             else:
 272                 logging.warn("Deleting page " + page.name)
 273                 page.delete('No longer published')
 274         for pagename, text in genp.items():
 275             logging.debug("Checking " + pagename)
 276             if pagename not in existingpages:
 277                 logging.debug("Creating page " + pagename)
 278                 try:
 279                     newpage = site.Pages[pagename]
 280                     newpage.save(text, summary='Auto-created')
 281                 except:
 282                     logging.error("...FAILED to create page '{0}'".format(pagename))
 283
 284     # Purge server cache to ensure counts are up to date
 285     site.pages['Repository Maintenance'].purge()
 286
 287
 288 def delete_disabled_builds(apps, apkcache, repodirs):
 289     """Delete disabled build outputs.
 290
 291     :param apps: list of all applications, as per metadata.read_metadata
 292     :param apkcache: current apk cache information
 293     :param repodirs: the repo directories to process
 294     """
 295     for appid, app in apps.iteritems():
 296         for build in app.builds:
 297             if not build.disable:
 298                 continue
 299             apkfilename = appid + '_' + str(build.vercode) + '.apk'
 300             iconfilename = "%s.%s.png" % (
 301                 appid,
 302                 build.vercode)
 303             for repodir in repodirs:
 304                 files = [
 305                     os.path.join(repodir, apkfilename),
 306                     os.path.join(repodir, apkfilename + '.asc'),
 307                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 308                 ]
 309                 for density in all_screen_densities:
 310                     repo_dir = get_icon_dir(repodir, density)
 311                     files.append(os.path.join(repo_dir, iconfilename))
 312
 313                 for f in files:
 314                     if os.path.exists(f):
 315                         logging.info("Deleting disabled build output " + f)
 316                         os.remove(f)
 317             if apkfilename in apkcache:
 318                 del apkcache[apkfilename]
 319
 320
 321 def resize_icon(iconpath, density):
 322
 323     if not os.path.isfile(iconpath):
 324         return
 325
 326     try:
 327         im = Image.open(iconpath)
 328         size = dpi_to_px(density)
 329
 330         if any(length > size for length in im.size):
 331             oldsize = im.size
 332             im.thumbnail((size, size), Image.ANTIALIAS)
 333             logging.debug("%s was too large at %s - new size is %s" % (
 334                 iconpath, oldsize, im.size))
 335             im.save(iconpath, "PNG")
 336
 337     except Exception as e:
 338         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 339
 340
 341 def resize_all_icons(repodirs):
 342     """Resize all icons that exceed the max size
 343
 344     :param repodirs: the repo directories to process
 345     """
 346     for repodir in repodirs:
 347         for density in screen_densities:
 348             icon_dir = get_icon_dir(repodir, density)
 349             icon_glob = os.path.join(icon_dir, '*.png')
 350             for iconpath in glob.glob(icon_glob):
 351                 resize_icon(iconpath, density)
 352
 353
 354 # A signature block file with a .DSA, .RSA, or .EC extension
 355 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 356
 357
 358 def getsig(apkpath):
 359     """ Get the signing certificate of an apk. To get the same md5 has that
 360     Android gets, we encode the .RSA certificate in a specific format and pass
 361     it hex-encoded to the md5 digest algorithm.
 362
 363     :param apkpath: path to the apk
 364     :returns: A string containing the md5 of the signature of the apk or None
 365               if an error occurred.
 366     """
 367
 368     cert = None
 369
 370     # verify the jar signature is correct
 371     args = [config['jarsigner'], '-verify', apkpath]
 372     p = FDroidPopen(args)
 373     if p.returncode != 0:
 374         logging.critical(apkpath + " has a bad signature!")
 375         return None
 376
 377     with zipfile.ZipFile(apkpath, 'r') as apk:
 378
 379         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 380
 381         if len(certs) < 1:
 382             logging.error("Found no signing certificates on %s" % apkpath)
 383             return None
 384         if len(certs) > 1:
 385             logging.error("Found multiple signing certificates on %s" % apkpath)
 386             return None
 387
 388         cert = apk.read(certs[0])
 389
 390     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 391     if content.getComponentByName('contentType') != rfc2315.signedData:
 392         logging.error("Unexpected format.")
 393         return None
 394
 395     content = decoder.decode(content.getComponentByName('content'),
 396                              asn1Spec=rfc2315.SignedData())[0]
 397     try:
 398         certificates = content.getComponentByName('certificates')
 399     except PyAsn1Error:
 400         logging.error("Certificates not found.")
 401         return None
 402
 403     cert_encoded = encoder.encode(certificates)[4:]
 404
 405     return md5(cert_encoded.encode('hex')).hexdigest()
 406
 407
 408 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
 409     """Scan the apks in the given repo directory.
 410
 411     This also extracts the icons.
 412
 413     :param apps: list of all applications, as per metadata.read_metadata
 414     :param apkcache: current apk cache information
 415     :param repodir: repo directory to scan
 416     :param knownapks: known apks info
 417     :param use_date_from_apk: use date from APK (instead of current date)
 418                               for newly added APKs
 419     :returns: (apks, cachechanged) where apks is a list of apk information,
 420               and cachechanged is True if the apkcache got changed.
 421     """
 422
 423     cachechanged = False
 424
 425     for icon_dir in get_all_icon_dirs(repodir):
 426         if os.path.exists(icon_dir):
 427             if options.clean:
 428                 shutil.rmtree(icon_dir)
 429                 os.makedirs(icon_dir)
 430         else:
 431             os.makedirs(icon_dir)
 432
 433     apks = []
 434     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 435     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 436     vername_pat = re.compile(".*versionName='([^']*)'.*")
 437     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 438     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 439     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 440     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 441     string_pat = re.compile(".*'([^']*)'.*")
 442     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 443
 444         apkfilename = apkfile[len(repodir) + 1:]
 445         if ' ' in apkfilename:
 446             logging.critical("Spaces in filenames are not allowed.")
 447             sys.exit(1)
 448
 449         # Calculate the sha256...
 450         sha = hashlib.sha256()
 451         with open(apkfile, 'rb') as f:
 452             while True:
 453                 t = f.read(16384)
 454                 if len(t) == 0:
 455                     break
 456                 sha.update(t)
 457             shasum = sha.hexdigest()
 458
 459         usecache = False
 460         if apkfilename in apkcache:
 461             apk = apkcache[apkfilename]
 462             if apk['sha256'] == shasum:
 463                 logging.debug("Reading " + apkfilename + " from cache")
 464                 usecache = True
 465             else:
 466                 logging.debug("Ignoring stale cache data for " + apkfilename)
 467
 468         if not usecache:
 469             logging.debug("Processing " + apkfilename)
 470             apk = {}
 471             apk['apkname'] = apkfilename
 472             apk['sha256'] = shasum
 473             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 474             if os.path.exists(os.path.join(repodir, srcfilename)):
 475                 apk['srcname'] = srcfilename
 476             apk['size'] = os.path.getsize(apkfile)
 477             apk['permissions'] = set()
 478             apk['features'] = set()
 479             apk['icons_src'] = {}
 480             apk['icons'] = {}
 481             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 482             if p.returncode != 0:
 483                 if options.delete_unknown:
 484                     if os.path.exists(apkfile):
 485                         logging.error("Failed to get apk information, deleting " + apkfile)
 486                         os.remove(apkfile)
 487                     else:
 488                         logging.error("Could not find {0} to remove it".format(apkfile))
 489                 else:
 490                     logging.error("Failed to get apk information, skipping " + apkfile)
 491                 continue
 492             for line in p.output.splitlines():
 493                 if line.startswith("package:"):
 494                     try:
 495                         apk['id'] = re.match(name_pat, line).group(1)
 496                         apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
 497                         apk['version'] = re.match(vername_pat, line).group(1)
 498                     except Exception as e:
 499                         logging.error("Package matching failed: " + str(e))
 500                         logging.info("Line was: " + line)
 501                         sys.exit(1)
 502                 elif line.startswith("application:"):
 503                     apk['name'] = re.match(label_pat, line).group(1)
 504                     # Keep path to non-dpi icon in case we need it
 505                     match = re.match(icon_pat_nodpi, line)
 506                     if match:
 507                         apk['icons_src']['-1'] = match.group(1)
 508                 elif line.startswith("launchable-activity:"):
 509                     # Only use launchable-activity as fallback to application
 510                     if not apk['name']:
 511                         apk['name'] = re.match(label_pat, line).group(1)
 512                     if '-1' not in apk['icons_src']:
 513                         match = re.match(icon_pat_nodpi, line)
 514                         if match:
 515                             apk['icons_src']['-1'] = match.group(1)
 516                 elif line.startswith("application-icon-"):
 517                     match = re.match(icon_pat, line)
 518                     if match:
 519                         density = match.group(1)
 520                         path = match.group(2)
 521                         apk['icons_src'][density] = path
 522                 elif line.startswith("sdkVersion:"):
 523                     m = re.match(sdkversion_pat, line)
 524                     if m is None:
 525                         logging.error(line.replace('sdkVersion:', '')
 526                                       + ' is not a valid minSdkVersion!')
 527                     else:
 528                         apk['sdkversion'] = m.group(1)
 529                 elif line.startswith("maxSdkVersion:"):
 530                     apk['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 531                 elif line.startswith("native-code:"):
 532                     apk['nativecode'] = []
 533                     for arch in line[13:].split(' '):
 534                         apk['nativecode'].append(arch[1:-1])
 535                 elif line.startswith("uses-permission:"):
 536                     perm = re.match(string_pat, line).group(1)
 537                     if perm.startswith("android.permission."):
 538                         perm = perm[19:]
 539                     apk['permissions'].add(perm)
 540                 elif line.startswith("uses-feature:"):
 541                     perm = re.match(string_pat, line).group(1)
 542                     # Filter out this, it's only added with the latest SDK tools and
 543                     # causes problems for lots of apps.
 544                     if perm != "android.hardware.screen.portrait" \
 545                             and perm != "android.hardware.screen.landscape":
 546                         if perm.startswith("android.feature."):
 547                             perm = perm[16:]
 548                         apk['features'].add(perm)
 549
 550             if 'sdkversion' not in apk:
 551                 logging.warn("No SDK version information found in {0}".format(apkfile))
 552                 apk['sdkversion'] = 0
 553
 554             # Check for debuggable apks...
 555             if common.isApkDebuggable(apkfile, config):
 556                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 557
 558             # Get the signature (or md5 of, to be precise)...
 559             logging.debug('Getting signature of {0}'.format(apkfile))
 560             apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 561             if not apk['sig']:
 562                 logging.critical("Failed to get apk signature")
 563                 sys.exit(1)
 564
 565             apkzip = zipfile.ZipFile(apkfile, 'r')
 566
 567             # if an APK has files newer than the system time, suggest updating
 568             # the system clock.  This is useful for offline systems, used for
 569             # signing, which do not have another source of clock sync info. It
 570             # has to be more than 24 hours newer because ZIP/APK files do not
 571             # store timezone info
 572             manifest = apkzip.getinfo('AndroidManifest.xml')
 573             if manifest.date_time[1] == 0:  # month can't be zero
 574                 logging.debug('AndroidManifest.xml has no date')
 575             else:
 576                 dt_obj = datetime(*manifest.date_time)
 577                 checkdt = dt_obj - timedelta(1)
 578                 if datetime.today() < checkdt:
 579                     logging.warn('System clock is older than manifest in: '
 580                                  + apkfilename
 581                                  + '\nSet clock to that time using:\n'
 582                                  + 'sudo date -s "' + str(dt_obj) + '"')
 583
 584             iconfilename = "%s.%s.png" % (
 585                 apk['id'],
 586                 apk['versioncode'])
 587
 588             # Extract the icon file...
 589             empty_densities = []
 590             for density in screen_densities:
 591                 if density not in apk['icons_src']:
 592                     empty_densities.append(density)
 593                     continue
 594                 iconsrc = apk['icons_src'][density]
 595                 icon_dir = get_icon_dir(repodir, density)
 596                 icondest = os.path.join(icon_dir, iconfilename)
 597
 598                 try:
 599                     with open(icondest, 'wb') as f:
 600                         f.write(apkzip.read(iconsrc))
 601                     apk['icons'][density] = iconfilename
 602
 603                 except:
 604                     logging.warn("Error retrieving icon file")
 605                     del apk['icons'][density]
 606                     del apk['icons_src'][density]
 607                     empty_densities.append(density)
 608
 609             if '-1' in apk['icons_src']:
 610                 iconsrc = apk['icons_src']['-1']
 611                 iconpath = os.path.join(
 612                     get_icon_dir(repodir, '0'), iconfilename)
 613                 with open(iconpath, 'wb') as f:
 614                     f.write(apkzip.read(iconsrc))
 615                 try:
 616                     im = Image.open(iconpath)
 617                     dpi = px_to_dpi(im.size[0])
 618                     for density in screen_densities:
 619                         if density in apk['icons']:
 620                             break
 621                         if density == screen_densities[-1] or dpi >= int(density):
 622                             apk['icons'][density] = iconfilename
 623                             shutil.move(iconpath,
 624                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 625                             empty_densities.remove(density)
 626                             break
 627                 except Exception as e:
 628                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 629
 630             if apk['icons']:
 631                 apk['icon'] = iconfilename
 632
 633             apkzip.close()
 634
 635             # First try resizing down to not lose quality
 636             last_density = None
 637             for density in screen_densities:
 638                 if density not in empty_densities:
 639                     last_density = density
 640                     continue
 641                 if last_density is None:
 642                     continue
 643                 logging.debug("Density %s not available, resizing down from %s"
 644                               % (density, last_density))
 645
 646                 last_iconpath = os.path.join(
 647                     get_icon_dir(repodir, last_density), iconfilename)
 648                 iconpath = os.path.join(
 649                     get_icon_dir(repodir, density), iconfilename)
 650                 try:
 651                     im = Image.open(last_iconpath)
 652                 except:
 653                     logging.warn("Invalid image file at %s" % last_iconpath)
 654                     continue
 655
 656                 size = dpi_to_px(density)
 657
 658                 im.thumbnail((size, size), Image.ANTIALIAS)
 659                 im.save(iconpath, "PNG")
 660                 empty_densities.remove(density)
 661
 662             # Then just copy from the highest resolution available
 663             last_density = None
 664             for density in reversed(screen_densities):
 665                 if density not in empty_densities:
 666                     last_density = density
 667                     continue
 668                 if last_density is None:
 669                     continue
 670                 logging.debug("Density %s not available, copying from lower density %s"
 671                               % (density, last_density))
 672
 673                 shutil.copyfile(
 674                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 675                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 676
 677                 empty_densities.remove(density)
 678
 679             for density in screen_densities:
 680                 icon_dir = get_icon_dir(repodir, density)
 681                 icondest = os.path.join(icon_dir, iconfilename)
 682                 resize_icon(icondest, density)
 683
 684             # Copy from icons-mdpi to icons since mdpi is the baseline density
 685             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 686             if os.path.isfile(baseline):
 687                 apk['icons']['0'] = iconfilename
 688                 shutil.copyfile(baseline,
 689                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 690
 691             # Record in known apks, getting the added date at the same time..
 692             added = knownapks.recordapk(apk['apkname'], apk['id'])
 693             if added:
 694                 if use_date_from_apk and manifest.date_time[1] != 0:
 695                     added = datetime(*manifest.date_time).timetuple()
 696                     logging.debug("Using date from APK")
 697
 698                 apk['added'] = added
 699
 700             apkcache[apkfilename] = apk
 701             cachechanged = True
 702
 703         apks.append(apk)
 704
 705     return apks, cachechanged
 706
 707
 708 repo_pubkey_fingerprint = None
 709
 710
 711 # Generate a certificate fingerprint the same way keytool does it
 712 # (but with slightly different formatting)
 713 def cert_fingerprint(data):
 714     digest = hashlib.sha256(data).digest()
 715     ret = []
 716     ret.append(' '.join("%02X" % ord(b) for b in digest))
 717     return " ".join(ret)
 718
 719
 720 def extract_pubkey():
 721     global repo_pubkey_fingerprint
 722     if 'repo_pubkey' in config:
 723         pubkey = unhexlify(config['repo_pubkey'])
 724     else:
 725         p = FDroidPopen([config['keytool'], '-exportcert',
 726                          '-alias', config['repo_keyalias'],
 727                          '-keystore', config['keystore'],
 728                          '-storepass:file', config['keystorepassfile']]
 729                         + config['smartcardoptions'], output=False)
 730         if p.returncode != 0 or len(p.output) < 20:
 731             msg = "Failed to get repo pubkey!"
 732             if config['keystore'] == 'NONE':
 733                 msg += ' Is your crypto smartcard plugged in?'
 734             logging.critical(msg)
 735             sys.exit(1)
 736         pubkey = p.output
 737     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 738     return hexlify(pubkey)
 739
 740
 741 def make_index(apps, sortedids, apks, repodir, archive, categories):
 742     """Make a repo index.
 743
 744     :param apps: fully populated apps list
 745     :param apks: full populated apks list
 746     :param repodir: the repo directory
 747     :param archive: True if this is the archive repo, False if it's the
 748                     main one.
 749     :param categories: list of categories
 750     """
 751
 752     doc = Document()
 753
 754     def addElement(name, value, doc, parent):
 755         el = doc.createElement(name)
 756         el.appendChild(doc.createTextNode(value))
 757         parent.appendChild(el)
 758
 759     def addElementNonEmpty(name, value, doc, parent):
 760         if not value:
 761             return
 762         addElement(name, value, doc, parent)
 763
 764     def addElementCDATA(name, value, doc, parent):
 765         el = doc.createElement(name)
 766         el.appendChild(doc.createCDATASection(value))
 767         parent.appendChild(el)
 768
 769     root = doc.createElement("fdroid")
 770     doc.appendChild(root)
 771
 772     repoel = doc.createElement("repo")
 773
 774     mirrorcheckfailed = False
 775     for mirror in config.get('mirrors', []):
 776         base = os.path.basename(urlparse.urlparse(mirror).path.rstrip('/'))
 777         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
 778             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 779             mirrorcheckfailed = True
 780     if mirrorcheckfailed:
 781         sys.exit(1)
 782
 783     if archive:
 784         repoel.setAttribute("name", config['archive_name'])
 785         if config['repo_maxage'] != 0:
 786             repoel.setAttribute("maxage", str(config['repo_maxage']))
 787         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 788         repoel.setAttribute("url", config['archive_url'])
 789         addElement('description', config['archive_description'], doc, repoel)
 790         urlbasepath = os.path.basename(urlparse.urlparse(config['archive_url']).path)
 791         for mirror in config.get('mirrors', []):
 792             addElement('mirror', urlparse.urljoin(mirror, urlbasepath), doc, repoel)
 793
 794     else:
 795         repoel.setAttribute("name", config['repo_name'])
 796         if config['repo_maxage'] != 0:
 797             repoel.setAttribute("maxage", str(config['repo_maxage']))
 798         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 799         repoel.setAttribute("url", config['repo_url'])
 800         addElement('description', config['repo_description'], doc, repoel)
 801         urlbasepath = os.path.basename(urlparse.urlparse(config['repo_url']).path)
 802         for mirror in config.get('mirrors', []):
 803             addElement('mirror', urlparse.urljoin(mirror, urlbasepath), doc, repoel)
 804
 805     repoel.setAttribute("version", "15")
 806     repoel.setAttribute("timestamp", str(int(time.time())))
 807
 808     nosigningkey = False
 809     if not options.nosign:
 810         if 'repo_keyalias' not in config:
 811             nosigningkey = True
 812             logging.critical("'repo_keyalias' not found in config.py!")
 813         if 'keystore' not in config:
 814             nosigningkey = True
 815             logging.critical("'keystore' not found in config.py!")
 816         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 817             nosigningkey = True
 818             logging.critical("'keystorepass' not found in config.py!")
 819         if 'keypass' not in config and 'keypassfile' not in config:
 820             nosigningkey = True
 821             logging.critical("'keypass' not found in config.py!")
 822         if not os.path.exists(config['keystore']):
 823             nosigningkey = True
 824             logging.critical("'" + config['keystore'] + "' does not exist!")
 825         if nosigningkey:
 826             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 827             logging.warning("\tfdroid update --create-key")
 828             sys.exit(1)
 829
 830     repoel.setAttribute("pubkey", extract_pubkey())
 831     root.appendChild(repoel)
 832
 833     for appid in sortedids:
 834         app = apps[appid]
 835
 836         if app.Disabled is not None:
 837             continue
 838
 839         # Get a list of the apks for this app...
 840         apklist = []
 841         for apk in apks:
 842             if apk['id'] == appid:
 843                 apklist.append(apk)
 844
 845         if len(apklist) == 0:
 846             continue
 847
 848         apel = doc.createElement("application")
 849         apel.setAttribute("id", app.id)
 850         root.appendChild(apel)
 851
 852         addElement('id', app.id, doc, apel)
 853         if app.added:
 854             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
 855         if app.lastupdated:
 856             addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
 857         addElement('name', app.Name, doc, apel)
 858         addElement('summary', app.Summary, doc, apel)
 859         if app.icon:
 860             addElement('icon', app.icon, doc, apel)
 861
 862         def linkres(appid):
 863             if appid in apps:
 864                 return ("fdroid.app:" + appid, apps[appid].Name)
 865             raise MetaDataException("Cannot resolve app id " + appid)
 866
 867         addElement('desc',
 868                    metadata.description_html(app.Description, linkres),
 869                    doc, apel)
 870         addElement('license', app.License, doc, apel)
 871         if app.Categories:
 872             addElement('categories', ','.join(app.Categories), doc, apel)
 873             # We put the first (primary) category in LAST, which will have
 874             # the desired effect of making clients that only understand one
 875             # category see that one.
 876             addElement('category', app.Categories[0], doc, apel)
 877         addElement('web', app.WebSite, doc, apel)
 878         addElement('source', app.SourceCode, doc, apel)
 879         addElement('tracker', app.IssueTracker, doc, apel)
 880         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 881         addElementNonEmpty('author', app.AuthorName, doc, apel)
 882         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 883         addElementNonEmpty('donate', app.Donate, doc, apel)
 884         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 885         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 886         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 887
 888         # These elements actually refer to the current version (i.e. which
 889         # one is recommended. They are historically mis-named, and need
 890         # changing, but stay like this for now to support existing clients.
 891         addElement('marketversion', app.CurrentVersion, doc, apel)
 892         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 893
 894         if app.AntiFeatures:
 895             af = app.AntiFeatures
 896             if af:
 897                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 898         if app.Provides:
 899             pv = app.Provides.split(',')
 900             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 901         if app.RequiresRoot:
 902             addElement('requirements', 'root', doc, apel)
 903
 904         # Sort the apk list into version order, just so the web site
 905         # doesn't have to do any work by default...
 906         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 907
 908         # Check for duplicates - they will make the client unhappy...
 909         for i in range(len(apklist) - 1):
 910             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 911                 logging.critical("duplicate versions: '%s' - '%s'" % (
 912                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 913                 sys.exit(1)
 914
 915         current_version_code = 0
 916         current_version_file = None
 917         for apk in apklist:
 918             # find the APK for the "Current Version"
 919             if current_version_code < apk['versioncode']:
 920                 current_version_code = apk['versioncode']
 921             if current_version_code < int(app.CurrentVersionCode):
 922                 current_version_file = apk['apkname']
 923
 924             apkel = doc.createElement("package")
 925             apel.appendChild(apkel)
 926             addElement('version', apk['version'], doc, apkel)
 927             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 928             addElement('apkname', apk['apkname'], doc, apkel)
 929             if 'srcname' in apk:
 930                 addElement('srcname', apk['srcname'], doc, apkel)
 931             for hash_type in ['sha256']:
 932                 if hash_type not in apk:
 933                     continue
 934                 hashel = doc.createElement("hash")
 935                 hashel.setAttribute("type", hash_type)
 936                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 937                 apkel.appendChild(hashel)
 938             addElement('sig', apk['sig'], doc, apkel)
 939             addElement('size', str(apk['size']), doc, apkel)
 940             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 941             if 'maxsdkversion' in apk:
 942                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 943             if 'added' in apk:
 944                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 945             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
 946             if 'nativecode' in apk:
 947                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 948             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
 949
 950         if current_version_file is not None \
 951                 and config['make_current_version_link'] \
 952                 and repodir == 'repo':  # only create these
 953             namefield = config['current_version_name_source']
 954             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
 955             apklinkname = sanitized_name + '.apk'
 956             current_version_path = os.path.join(repodir, current_version_file)
 957             if os.path.islink(apklinkname):
 958                 os.remove(apklinkname)
 959             os.symlink(current_version_path, apklinkname)
 960             # also symlink gpg signature, if it exists
 961             for extension in ('.asc', '.sig'):
 962                 sigfile_path = current_version_path + extension
 963                 if os.path.exists(sigfile_path):
 964                     siglinkname = apklinkname + extension
 965                     if os.path.islink(siglinkname):
 966                         os.remove(siglinkname)
 967                     os.symlink(sigfile_path, siglinkname)
 968
 969     if options.pretty:
 970         output = doc.toprettyxml()
 971     else:
 972         output = doc.toxml()
 973
 974     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 975         f.write(output)
 976
 977     if 'repo_keyalias' in config:
 978
 979         if options.nosign:
 980             logging.info("Creating unsigned index in preparation for signing")
 981         else:
 982             logging.info("Creating signed index with this key (SHA256):")
 983             logging.info("%s" % repo_pubkey_fingerprint)
 984
 985         # Create a jar of the index...
 986         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
 987         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 988         if p.returncode != 0:
 989             logging.critical("Failed to create {0}".format(jar_output))
 990             sys.exit(1)
 991
 992         # Sign the index...
 993         signed = os.path.join(repodir, 'index.jar')
 994         if options.nosign:
 995             # Remove old signed index if not signing
 996             if os.path.exists(signed):
 997                 os.remove(signed)
 998         else:
 999             args = [config['jarsigner'], '-keystore', config['keystore'],
1000                     '-storepass:file', config['keystorepassfile'],
1001                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1002                     signed, config['repo_keyalias']]
1003             if config['keystore'] == 'NONE':
1004                 args += config['smartcardoptions']
1005             else:  # smardcards never use -keypass
1006                 args += ['-keypass:file', config['keypassfile']]
1007             p = FDroidPopen(args)
1008             if p.returncode != 0:
1009                 logging.critical("Failed to sign index")
1010                 sys.exit(1)
1011
1012     # Copy the repo icon into the repo directory...
1013     icon_dir = os.path.join(repodir, 'icons')
1014     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1015     shutil.copyfile(config['repo_icon'], iconfilename)
1016
1017     # Write a category list in the repo to allow quick access...
1018     catdata = ''
1019     for cat in categories:
1020         catdata += cat + '\n'
1021     with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
1022         f.write(catdata)
1023
1024
1025 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1026
1027     for appid, app in apps.iteritems():
1028
1029         if app.ArchivePolicy:
1030             keepversions = int(app.ArchivePolicy[:-9])
1031         else:
1032             keepversions = defaultkeepversions
1033
1034         def filter_apk_list_sorted(apk_list):
1035             res = []
1036             for apk in apk_list:
1037                 if apk['id'] == appid:
1038                     res.append(apk)
1039
1040             # Sort the apk list by version code. First is highest/newest.
1041             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1042
1043         def move_file(from_dir, to_dir, filename, ignore_missing):
1044             from_path = os.path.join(from_dir, filename)
1045             if ignore_missing and not os.path.exists(from_path):
1046                 return
1047             to_path = os.path.join(to_dir, filename)
1048             shutil.move(from_path, to_path)
1049
1050         if len(apks) > keepversions:
1051             apklist = filter_apk_list_sorted(apks)
1052             # Move back the ones we don't want.
1053             for apk in apklist[keepversions:]:
1054                 logging.info("Moving " + apk['apkname'] + " to archive")
1055                 move_file(repodir, archivedir, apk['apkname'], False)
1056                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1057                 for density in all_screen_densities:
1058                     repo_icon_dir = get_icon_dir(repodir, density)
1059                     archive_icon_dir = get_icon_dir(archivedir, density)
1060                     if density not in apk['icons']:
1061                         continue
1062                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1063                 if 'srcname' in apk:
1064                     move_file(repodir, archivedir, apk['srcname'], False)
1065                 archapks.append(apk)
1066                 apks.remove(apk)
1067         elif len(apks) < keepversions and len(archapks) > 0:
1068             required = keepversions - len(apks)
1069             archapklist = filter_apk_list_sorted(archapks)
1070             # Move forward the ones we want again.
1071             for apk in archapklist[:required]:
1072                 logging.info("Moving " + apk['apkname'] + " from archive")
1073                 move_file(archivedir, repodir, apk['apkname'], False)
1074                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1075                 for density in all_screen_densities:
1076                     repo_icon_dir = get_icon_dir(repodir, density)
1077                     archive_icon_dir = get_icon_dir(archivedir, density)
1078                     if density not in apk['icons']:
1079                         continue
1080                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1081                 if 'srcname' in apk:
1082                     move_file(archivedir, repodir, apk['srcname'], False)
1083                 archapks.remove(apk)
1084                 apks.append(apk)
1085
1086
1087 def add_apks_to_per_app_repos(repodir, apks):
1088     apks_per_app = dict()
1089     for apk in apks:
1090         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1091         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1092         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1093         apks_per_app[apk['id']] = apk
1094
1095         if not os.path.exists(apk['per_app_icons']):
1096             logging.info('Adding new repo for only ' + apk['id'])
1097             os.makedirs(apk['per_app_icons'])
1098
1099         apkpath = os.path.join(repodir, apk['apkname'])
1100         shutil.copy(apkpath, apk['per_app_repo'])
1101         apksigpath = apkpath + '.sig'
1102         if os.path.exists(apksigpath):
1103             shutil.copy(apksigpath, apk['per_app_repo'])
1104         apkascpath = apkpath + '.asc'
1105         if os.path.exists(apkascpath):
1106             shutil.copy(apkascpath, apk['per_app_repo'])
1107
1108
1109 config = None
1110 options = None
1111
1112
1113 def main():
1114
1115     global config, options
1116
1117     # Parse command line...
1118     parser = ArgumentParser()
1119     common.setup_global_opts(parser)
1120     parser.add_argument("--create-key", action="store_true", default=False,
1121                         help="Create a repo signing key in a keystore")
1122     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1123                         help="Create skeleton metadata files that are missing")
1124     parser.add_argument("--delete-unknown", action="store_true", default=False,
1125                         help="Delete APKs without metadata from the repo")
1126     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1127                         help="Report on build data status")
1128     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1129                         help="Interactively ask about things that need updating.")
1130     parser.add_argument("-I", "--icons", action="store_true", default=False,
1131                         help="Resize all the icons exceeding the max pixel size and exit")
1132     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1133                         help="Specify editor to use in interactive mode. Default " +
1134                         "is /etc/alternatives/editor")
1135     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1136                         help="Update the wiki")
1137     parser.add_argument("--pretty", action="store_true", default=False,
1138                         help="Produce human-readable index.xml")
1139     parser.add_argument("--clean", action="store_true", default=False,
1140                         help="Clean update - don't uses caches, reprocess all apks")
1141     parser.add_argument("--nosign", action="store_true", default=False,
1142                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1143     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1144                         help="Use date from apk instead of current time for newly added apks")
1145     options = parser.parse_args()
1146
1147     config = common.read_config(options)
1148
1149     if not ('jarsigner' in config and 'keytool' in config):
1150         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1151         sys.exit(1)
1152
1153     repodirs = ['repo']
1154     if config['archive_older'] != 0:
1155         repodirs.append('archive')
1156         if not os.path.exists('archive'):
1157             os.mkdir('archive')
1158
1159     if options.icons:
1160         resize_all_icons(repodirs)
1161         sys.exit(0)
1162
1163     # check that icons exist now, rather than fail at the end of `fdroid update`
1164     for k in ['repo_icon', 'archive_icon']:
1165         if k in config:
1166             if not os.path.exists(config[k]):
1167                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1168                 sys.exit(1)
1169
1170     # if the user asks to create a keystore, do it now, reusing whatever it can
1171     if options.create_key:
1172         if os.path.exists(config['keystore']):
1173             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1174             logging.critical("\t'" + config['keystore'] + "'")
1175             sys.exit(1)
1176
1177         if 'repo_keyalias' not in config:
1178             config['repo_keyalias'] = socket.getfqdn()
1179             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1180         if 'keydname' not in config:
1181             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1182             common.write_to_config(config, 'keydname', config['keydname'])
1183         if 'keystore' not in config:
1184             config['keystore'] = common.default_config.keystore
1185             common.write_to_config(config, 'keystore', config['keystore'])
1186
1187         password = common.genpassword()
1188         if 'keystorepass' not in config:
1189             config['keystorepass'] = password
1190             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1191         if 'keypass' not in config:
1192             config['keypass'] = password
1193             common.write_to_config(config, 'keypass', config['keypass'])
1194         common.genkeystore(config)
1195
1196     # Get all apps...
1197     apps = metadata.read_metadata()
1198
1199     # Generate a list of categories...
1200     categories = set()
1201     for app in apps.itervalues():
1202         categories.update(app.Categories)
1203
1204     # Read known apks data (will be updated and written back when we've finished)
1205     knownapks = common.KnownApks()
1206
1207     # Gather information about all the apk files in the repo directory, using
1208     # cached data if possible.
1209     apkcachefile = os.path.join('tmp', 'apkcache')
1210     if not options.clean and os.path.exists(apkcachefile):
1211         with open(apkcachefile, 'rb') as cf:
1212             apkcache = pickle.load(cf)
1213     else:
1214         apkcache = {}
1215
1216     delete_disabled_builds(apps, apkcache, repodirs)
1217
1218     # Scan all apks in the main repo
1219     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1220
1221     # Generate warnings for apk's with no metadata (or create skeleton
1222     # metadata files, if requested on the command line)
1223     newmetadata = False
1224     for apk in apks:
1225         if apk['id'] not in apps:
1226             if options.create_metadata:
1227                 if 'name' not in apk:
1228                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1229                     continue
1230                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1231                 f.write("License:Unknown\n")
1232                 f.write("Web Site:\n")
1233                 f.write("Source Code:\n")
1234                 f.write("Issue Tracker:\n")
1235                 f.write("Changelog:\n")
1236                 f.write("Summary:" + apk['name'] + "\n")
1237                 f.write("Description:\n")
1238                 f.write(apk['name'] + "\n")
1239                 f.write(".\n")
1240                 f.close()
1241                 logging.info("Generated skeleton metadata for " + apk['id'])
1242                 newmetadata = True
1243             else:
1244                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1245                 if options.delete_unknown:
1246                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1247                     rmf = os.path.join(repodirs[0], apk['apkname'])
1248                     if not os.path.exists(rmf):
1249                         logging.error("Could not find {0} to remove it".format(rmf))
1250                     else:
1251                         os.remove(rmf)
1252                 else:
1253                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1254
1255     # update the metadata with the newly created ones included
1256     if newmetadata:
1257         apps = metadata.read_metadata()
1258
1259     # Scan the archive repo for apks as well
1260     if len(repodirs) > 1:
1261         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1262         if cc:
1263             cachechanged = True
1264     else:
1265         archapks = []
1266
1267     # Some information from the apks needs to be applied up to the application
1268     # level. When doing this, we use the info from the most recent version's apk.
1269     # We deal with figuring out when the app was added and last updated at the
1270     # same time.
1271     for appid, app in apps.iteritems():
1272         bestver = 0
1273         for apk in apks + archapks:
1274             if apk['id'] == appid:
1275                 if apk['versioncode'] > bestver:
1276                     bestver = apk['versioncode']
1277                     bestapk = apk
1278
1279                 if 'added' in apk:
1280                     if not app.added or apk['added'] < app.added:
1281                         app.added = apk['added']
1282                     if not app.lastupdated or apk['added'] > app.lastupdated:
1283                         app.lastupdated = apk['added']
1284
1285         if not app.added:
1286             logging.debug("Don't know when " + appid + " was added")
1287         if not app.lastupdated:
1288             logging.debug("Don't know when " + appid + " was last updated")
1289
1290         if bestver == 0:
1291             if app.Name is None:
1292                 app.Name = app.AutoName or appid
1293             app.icon = None
1294             logging.debug("Application " + appid + " has no packages")
1295         else:
1296             if app.Name is None:
1297                 app.Name = bestapk['name']
1298             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1299             if app.CurrentVersionCode is None:
1300                 app.CurrentVersionCode = str(bestver)
1301
1302     # Sort the app list by name, then the web site doesn't have to by default.
1303     # (we had to wait until we'd scanned the apks to do this, because mostly the
1304     # name comes from there!)
1305     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid].Name.upper())
1306
1307     # APKs are placed into multiple repos based on the app package, providing
1308     # per-app subscription feeds for nightly builds and things like it
1309     if config['per_app_repos']:
1310         add_apks_to_per_app_repos(repodirs[0], apks)
1311         for appid, app in apps.iteritems():
1312             repodir = os.path.join(appid, 'fdroid', 'repo')
1313             appdict = dict()
1314             appdict[appid] = app
1315             if os.path.isdir(repodir):
1316                 make_index(appdict, [appid], apks, repodir, False, categories)
1317             else:
1318                 logging.info('Skipping index generation for ' + appid)
1319         return
1320
1321     if len(repodirs) > 1:
1322         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1323
1324     # Make the index for the main repo...
1325     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1326
1327     # If there's an archive repo,  make the index for it. We already scanned it
1328     # earlier on.
1329     if len(repodirs) > 1:
1330         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1331
1332     if config['update_stats']:
1333
1334         # Update known apks info...
1335         knownapks.writeifchanged()
1336
1337         # Generate latest apps data for widget
1338         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1339             data = ''
1340             for line in file(os.path.join('stats', 'latestapps.txt')):
1341                 appid = line.rstrip()
1342                 data += appid + "\t"
1343                 app = apps[appid]
1344                 data += app.Name + "\t"
1345                 if app.icon is not None:
1346                     data += app.icon + "\t"
1347                 data += app.License + "\n"
1348             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
1349                 f.write(data)
1350
1351     if cachechanged:
1352         with open(apkcachefile, 'wb') as cf:
1353             pickle.dump(apkcache, cf)
1354
1355     # Update the wiki...
1356     if options.wiki:
1357         update_wiki(apps, sortedids, apks + archapks)
1358
1359     logging.info("Finished.")
1360
1361 if __name__ == "__main__":
1362     main()