fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   5 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU Affero General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Affero General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Affero General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 import sys
  21 import os
  22 import shutil
  23 import glob
  24 import re
  25 import socket
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 import urlparse
  30 from datetime import datetime, timedelta
  31 from xml.dom.minidom import Document
  32 from argparse import ArgumentParser
  33 import time
  34 from pyasn1.error import PyAsn1Error
  35 from pyasn1.codec.der import decoder, encoder
  36 from pyasn1_modules import rfc2315
  37 from hashlib import md5
  38 from binascii import hexlify, unhexlify
  39
  40 from PIL import Image
  41 import logging
  42
  43 import common
  44 import metadata
  45 from common import FDroidPopen, SdkToolsPopen
  46 from metadata import MetaDataException
  47
  48 screen_densities = ['640', '480', '320', '240', '160', '120']
  49
  50 all_screen_densities = ['0'] + screen_densities
  51
  52
  53 def dpi_to_px(density):
  54     return (int(density) * 48) / 160
  55
  56
  57 def px_to_dpi(px):
  58     return (int(px) * 160) / 48
  59
  60
  61 def get_icon_dir(repodir, density):
  62     if density == '0':
  63         return os.path.join(repodir, "icons")
  64     return os.path.join(repodir, "icons-%s" % density)
  65
  66
  67 def get_icon_dirs(repodir):
  68     for density in screen_densities:
  69         yield get_icon_dir(repodir, density)
  70
  71
  72 def get_all_icon_dirs(repodir):
  73     for density in all_screen_densities:
  74         yield get_icon_dir(repodir, density)
  75
  76
  77 def update_wiki(apps, sortedids, apks):
  78     """Update the wiki
  79
  80     :param apps: fully populated list of all applications
  81     :param apks: all apks, except...
  82     """
  83     logging.info("Updating wiki")
  84     wikicat = 'Apps'
  85     wikiredircat = 'App Redirects'
  86     import mwclient
  87     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  88                          path=config['wiki_path'])
  89     site.login(config['wiki_user'], config['wiki_password'])
  90     generated_pages = {}
  91     generated_redirects = {}
  92
  93     for appid in sortedids:
  94         app = apps[appid]
  95
  96         wikidata = ''
  97         if app.Disabled:
  98             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
  99         if app.AntiFeatures:
 100             for af in app.AntiFeatures:
 101                 wikidata += '{{AntiFeature|' + af + '}}\n'
 102         if app.RequiresRoot:
 103             requiresroot = 'Yes'
 104         else:
 105             requiresroot = 'No'
 106         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 107             appid,
 108             app.Name,
 109             time.strftime('%Y-%m-%d', app.added) if app.added else '',
 110             time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
 111             app.SourceCode,
 112             app.IssueTracker,
 113             app.WebSite,
 114             app.Changelog,
 115             app.Donate,
 116             app.FlattrID,
 117             app.Bitcoin,
 118             app.Litecoin,
 119             app.License,
 120             requiresroot,
 121             app.AuthorName,
 122             app.AuthorEmail)
 123
 124         if app.Provides:
 125             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 126
 127         wikidata += app.Summary
 128         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 129
 130         wikidata += "=Description=\n"
 131         wikidata += metadata.description_wiki(app.Description) + "\n"
 132
 133         wikidata += "=Maintainer Notes=\n"
 134         if app.MaintainerNotes:
 135             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 136         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 137
 138         # Get a list of all packages for this application...
 139         apklist = []
 140         gotcurrentver = False
 141         cantupdate = False
 142         buildfails = False
 143         for apk in apks:
 144             if apk['id'] == appid:
 145                 if str(apk['versioncode']) == app.CurrentVersionCode:
 146                     gotcurrentver = True
 147                 apklist.append(apk)
 148         # Include ones we can't build, as a special case...
 149         for build in app.builds:
 150             if build.disable:
 151                 if build.vercode == app.CurrentVersionCode:
 152                     cantupdate = True
 153                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 154                 apklist.append({'versioncode': int(build.vercode),
 155                                 'version': build.version,
 156                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 157                                 })
 158             else:
 159                 builtit = False
 160                 for apk in apklist:
 161                     if apk['versioncode'] == int(build.vercode):
 162                         builtit = True
 163                         break
 164                 if not builtit:
 165                     buildfails = True
 166                     apklist.append({'versioncode': int(build.vercode),
 167                                     'version': build.version,
 168                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
 169                                     })
 170         if app.CurrentVersionCode == '0':
 171             cantupdate = True
 172         # Sort with most recent first...
 173         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 174
 175         wikidata += "=Versions=\n"
 176         if len(apklist) == 0:
 177             wikidata += "We currently have no versions of this app available."
 178         elif not gotcurrentver:
 179             wikidata += "We don't have the current version of this app."
 180         else:
 181             wikidata += "We have the current version of this app."
 182         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 183         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 184         if len(app.NoSourceSince) > 0:
 185             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 186         if len(app.CurrentVersion) > 0:
 187             wikidata += "The current (recommended) version is " + app.CurrentVersion
 188             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 189         validapks = 0
 190         for apk in apklist:
 191             wikidata += "==" + apk['version'] + "==\n"
 192
 193             if 'buildproblem' in apk:
 194                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 195             else:
 196                 validapks += 1
 197                 wikidata += "This version is built and signed by "
 198                 if 'srcname' in apk:
 199                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 200                 else:
 201                     wikidata += "the original developer.\n\n"
 202             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 203
 204         wikidata += '\n[[Category:' + wikicat + ']]\n'
 205         if len(app.NoSourceSince) > 0:
 206             wikidata += '\n[[Category:Apps missing source code]]\n'
 207         if validapks == 0 and not app.Disabled:
 208             wikidata += '\n[[Category:Apps with no packages]]\n'
 209         if cantupdate and not app.Disabled:
 210             wikidata += "\n[[Category:Apps we cannot update]]\n"
 211         if buildfails and not app.Disabled:
 212             wikidata += "\n[[Category:Apps with failing builds]]\n"
 213         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 214             wikidata += '\n[[Category:Apps to Update]]\n'
 215         if app.Disabled:
 216             wikidata += '\n[[Category:Apps that are disabled]]\n'
 217         if app.UpdateCheckMode == 'None' and not app.Disabled:
 218             wikidata += '\n[[Category:Apps with no update check]]\n'
 219         for appcat in app.Categories:
 220             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 221
 222         # We can't have underscores in the page name, even if they're in
 223         # the package ID, because MediaWiki messes with them...
 224         pagename = appid.replace('_', ' ')
 225
 226         # Drop a trailing newline, because mediawiki is going to drop it anyway
 227         # and it we don't we'll think the page has changed when it hasn't...
 228         if wikidata.endswith('\n'):
 229             wikidata = wikidata[:-1]
 230
 231         generated_pages[pagename] = wikidata
 232
 233         # Make a redirect from the name to the ID too, unless there's
 234         # already an existing page with the name and it isn't a redirect.
 235         noclobber = False
 236         apppagename = app.Name.replace('_', ' ')
 237         apppagename = apppagename.replace('{', '')
 238         apppagename = apppagename.replace('}', ' ')
 239         apppagename = apppagename.replace(':', ' ')
 240         # Drop double spaces caused mostly by replacing ':' above
 241         apppagename = apppagename.replace('  ', ' ')
 242         for expagename in site.allpages(prefix=apppagename,
 243                                         filterredir='nonredirects',
 244                                         generator=False):
 245             if expagename == apppagename:
 246                 noclobber = True
 247         # Another reason not to make the redirect page is if the app name
 248         # is the same as it's ID, because that will overwrite the real page
 249         # with an redirect to itself! (Although it seems like an odd
 250         # scenario this happens a lot, e.g. where there is metadata but no
 251         # builds or binaries to extract a name from.
 252         if apppagename == pagename:
 253             noclobber = True
 254         if not noclobber:
 255             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 256
 257     for tcat, genp in [(wikicat, generated_pages),
 258                        (wikiredircat, generated_redirects)]:
 259         catpages = site.Pages['Category:' + tcat]
 260         existingpages = []
 261         for page in catpages:
 262             existingpages.append(page.name)
 263             if page.name in genp:
 264                 pagetxt = page.edit()
 265                 if pagetxt != genp[page.name]:
 266                     logging.debug("Updating modified page " + page.name)
 267                     page.save(genp[page.name], summary='Auto-updated')
 268                 else:
 269                     logging.debug("Page " + page.name + " is unchanged")
 270             else:
 271                 logging.warn("Deleting page " + page.name)
 272                 page.delete('No longer published')
 273         for pagename, text in genp.items():
 274             logging.debug("Checking " + pagename)
 275             if pagename not in existingpages:
 276                 logging.debug("Creating page " + pagename)
 277                 try:
 278                     newpage = site.Pages[pagename]
 279                     newpage.save(text, summary='Auto-created')
 280                 except:
 281                     logging.error("...FAILED to create page '{0}'".format(pagename))
 282
 283     # Purge server cache to ensure counts are up to date
 284     site.pages['Repository Maintenance'].purge()
 285
 286
 287 def delete_disabled_builds(apps, apkcache, repodirs):
 288     """Delete disabled build outputs.
 289
 290     :param apps: list of all applications, as per metadata.read_metadata
 291     :param apkcache: current apk cache information
 292     :param repodirs: the repo directories to process
 293     """
 294     for appid, app in apps.iteritems():
 295         for build in app.builds:
 296             if not build.disable:
 297                 continue
 298             apkfilename = appid + '_' + str(build.vercode) + '.apk'
 299             iconfilename = "%s.%s.png" % (
 300                 appid,
 301                 build.vercode)
 302             for repodir in repodirs:
 303                 files = [
 304                     os.path.join(repodir, apkfilename),
 305                     os.path.join(repodir, apkfilename + '.asc'),
 306                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 307                 ]
 308                 for density in all_screen_densities:
 309                     repo_dir = get_icon_dir(repodir, density)
 310                     files.append(os.path.join(repo_dir, iconfilename))
 311
 312                 for f in files:
 313                     if os.path.exists(f):
 314                         logging.info("Deleting disabled build output " + f)
 315                         os.remove(f)
 316             if apkfilename in apkcache:
 317                 del apkcache[apkfilename]
 318
 319
 320 def resize_icon(iconpath, density):
 321
 322     if not os.path.isfile(iconpath):
 323         return
 324
 325     try:
 326         im = Image.open(iconpath)
 327         size = dpi_to_px(density)
 328
 329         if any(length > size for length in im.size):
 330             oldsize = im.size
 331             im.thumbnail((size, size), Image.ANTIALIAS)
 332             logging.debug("%s was too large at %s - new size is %s" % (
 333                 iconpath, oldsize, im.size))
 334             im.save(iconpath, "PNG")
 335
 336     except Exception as e:
 337         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 338
 339
 340 def resize_all_icons(repodirs):
 341     """Resize all icons that exceed the max size
 342
 343     :param repodirs: the repo directories to process
 344     """
 345     for repodir in repodirs:
 346         for density in screen_densities:
 347             icon_dir = get_icon_dir(repodir, density)
 348             icon_glob = os.path.join(icon_dir, '*.png')
 349             for iconpath in glob.glob(icon_glob):
 350                 resize_icon(iconpath, density)
 351
 352
 353 # A signature block file with a .DSA, .RSA, or .EC extension
 354 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 355
 356
 357 def getsig(apkpath):
 358     """ Get the signing certificate of an apk. To get the same md5 has that
 359     Android gets, we encode the .RSA certificate in a specific format and pass
 360     it hex-encoded to the md5 digest algorithm.
 361
 362     :param apkpath: path to the apk
 363     :returns: A string containing the md5 of the signature of the apk or None
 364               if an error occurred.
 365     """
 366
 367     cert = None
 368
 369     # verify the jar signature is correct
 370     args = [config['jarsigner'], '-verify', apkpath]
 371     p = FDroidPopen(args)
 372     if p.returncode != 0:
 373         logging.critical(apkpath + " has a bad signature!")
 374         return None
 375
 376     with zipfile.ZipFile(apkpath, 'r') as apk:
 377
 378         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 379
 380         if len(certs) < 1:
 381             logging.error("Found no signing certificates on %s" % apkpath)
 382             return None
 383         if len(certs) > 1:
 384             logging.error("Found multiple signing certificates on %s" % apkpath)
 385             return None
 386
 387         cert = apk.read(certs[0])
 388
 389     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 390     if content.getComponentByName('contentType') != rfc2315.signedData:
 391         logging.error("Unexpected format.")
 392         return None
 393
 394     content = decoder.decode(content.getComponentByName('content'),
 395                              asn1Spec=rfc2315.SignedData())[0]
 396     try:
 397         certificates = content.getComponentByName('certificates')
 398     except PyAsn1Error:
 399         logging.error("Certificates not found.")
 400         return None
 401
 402     cert_encoded = encoder.encode(certificates)[4:]
 403
 404     return md5(cert_encoded.encode('hex')).hexdigest()
 405
 406
 407 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
 408     """Scan the apks in the given repo directory.
 409
 410     This also extracts the icons.
 411
 412     :param apps: list of all applications, as per metadata.read_metadata
 413     :param apkcache: current apk cache information
 414     :param repodir: repo directory to scan
 415     :param knownapks: known apks info
 416     :param use_date_from_apk: use date from APK (instead of current date)
 417                               for newly added APKs
 418     :returns: (apks, cachechanged) where apks is a list of apk information,
 419               and cachechanged is True if the apkcache got changed.
 420     """
 421
 422     cachechanged = False
 423
 424     for icon_dir in get_all_icon_dirs(repodir):
 425         if os.path.exists(icon_dir):
 426             if options.clean:
 427                 shutil.rmtree(icon_dir)
 428                 os.makedirs(icon_dir)
 429         else:
 430             os.makedirs(icon_dir)
 431
 432     apks = []
 433     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 434     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 435     vername_pat = re.compile(".*versionName='([^']*)'.*")
 436     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 437     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 438     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 439     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 440     string_pat = re.compile(".*'([^']*)'.*")
 441     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 442
 443         apkfilename = apkfile[len(repodir) + 1:]
 444         if ' ' in apkfilename:
 445             logging.critical("Spaces in filenames are not allowed.")
 446             sys.exit(1)
 447
 448         # Calculate the sha256...
 449         sha = hashlib.sha256()
 450         with open(apkfile, 'rb') as f:
 451             while True:
 452                 t = f.read(16384)
 453                 if len(t) == 0:
 454                     break
 455                 sha.update(t)
 456             shasum = sha.hexdigest()
 457
 458         usecache = False
 459         if apkfilename in apkcache:
 460             apk = apkcache[apkfilename]
 461             if apk['sha256'] == shasum:
 462                 logging.debug("Reading " + apkfilename + " from cache")
 463                 usecache = True
 464             else:
 465                 logging.debug("Ignoring stale cache data for " + apkfilename)
 466
 467         if not usecache:
 468             logging.debug("Processing " + apkfilename)
 469             apk = {}
 470             apk['apkname'] = apkfilename
 471             apk['sha256'] = shasum
 472             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 473             if os.path.exists(os.path.join(repodir, srcfilename)):
 474                 apk['srcname'] = srcfilename
 475             apk['size'] = os.path.getsize(apkfile)
 476             apk['permissions'] = set()
 477             apk['features'] = set()
 478             apk['icons_src'] = {}
 479             apk['icons'] = {}
 480             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 481             if p.returncode != 0:
 482                 if options.delete_unknown:
 483                     if os.path.exists(apkfile):
 484                         logging.error("Failed to get apk information, deleting " + apkfile)
 485                         os.remove(apkfile)
 486                     else:
 487                         logging.error("Could not find {0} to remove it".format(apkfile))
 488                 else:
 489                     logging.error("Failed to get apk information, skipping " + apkfile)
 490                 continue
 491             for line in p.output.splitlines():
 492                 if line.startswith("package:"):
 493                     try:
 494                         apk['id'] = re.match(name_pat, line).group(1)
 495                         apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
 496                         apk['version'] = re.match(vername_pat, line).group(1)
 497                     except Exception as e:
 498                         logging.error("Package matching failed: " + str(e))
 499                         logging.info("Line was: " + line)
 500                         sys.exit(1)
 501                 elif line.startswith("application:"):
 502                     apk['name'] = re.match(label_pat, line).group(1)
 503                     # Keep path to non-dpi icon in case we need it
 504                     match = re.match(icon_pat_nodpi, line)
 505                     if match:
 506                         apk['icons_src']['-1'] = match.group(1)
 507                 elif line.startswith("launchable-activity:"):
 508                     # Only use launchable-activity as fallback to application
 509                     if not apk['name']:
 510                         apk['name'] = re.match(label_pat, line).group(1)
 511                     if '-1' not in apk['icons_src']:
 512                         match = re.match(icon_pat_nodpi, line)
 513                         if match:
 514                             apk['icons_src']['-1'] = match.group(1)
 515                 elif line.startswith("application-icon-"):
 516                     match = re.match(icon_pat, line)
 517                     if match:
 518                         density = match.group(1)
 519                         path = match.group(2)
 520                         apk['icons_src'][density] = path
 521                 elif line.startswith("sdkVersion:"):
 522                     m = re.match(sdkversion_pat, line)
 523                     if m is None:
 524                         logging.error(line.replace('sdkVersion:', '')
 525                                       + ' is not a valid minSdkVersion!')
 526                     else:
 527                         apk['sdkversion'] = m.group(1)
 528                 elif line.startswith("maxSdkVersion:"):
 529                     apk['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 530                 elif line.startswith("native-code:"):
 531                     apk['nativecode'] = []
 532                     for arch in line[13:].split(' '):
 533                         apk['nativecode'].append(arch[1:-1])
 534                 elif line.startswith("uses-permission:"):
 535                     perm = re.match(string_pat, line).group(1)
 536                     if perm.startswith("android.permission."):
 537                         perm = perm[19:]
 538                     apk['permissions'].add(perm)
 539                 elif line.startswith("uses-feature:"):
 540                     perm = re.match(string_pat, line).group(1)
 541                     # Filter out this, it's only added with the latest SDK tools and
 542                     # causes problems for lots of apps.
 543                     if perm != "android.hardware.screen.portrait" \
 544                             and perm != "android.hardware.screen.landscape":
 545                         if perm.startswith("android.feature."):
 546                             perm = perm[16:]
 547                         apk['features'].add(perm)
 548
 549             if 'sdkversion' not in apk:
 550                 logging.warn("No SDK version information found in {0}".format(apkfile))
 551                 apk['sdkversion'] = 0
 552
 553             # Check for debuggable apks...
 554             if common.isApkDebuggable(apkfile, config):
 555                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 556
 557             # Get the signature (or md5 of, to be precise)...
 558             logging.debug('Getting signature of {0}'.format(apkfile))
 559             apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 560             if not apk['sig']:
 561                 logging.critical("Failed to get apk signature")
 562                 sys.exit(1)
 563
 564             apkzip = zipfile.ZipFile(apkfile, 'r')
 565
 566             # if an APK has files newer than the system time, suggest updating
 567             # the system clock.  This is useful for offline systems, used for
 568             # signing, which do not have another source of clock sync info. It
 569             # has to be more than 24 hours newer because ZIP/APK files do not
 570             # store timezone info
 571             manifest = apkzip.getinfo('AndroidManifest.xml')
 572             if manifest.date_time[1] == 0:  # month can't be zero
 573                 logging.debug('AndroidManifest.xml has no date')
 574             else:
 575                 dt_obj = datetime(*manifest.date_time)
 576                 checkdt = dt_obj - timedelta(1)
 577                 if datetime.today() < checkdt:
 578                     logging.warn('System clock is older than manifest in: '
 579                                  + apkfilename
 580                                  + '\nSet clock to that time using:\n'
 581                                  + 'sudo date -s "' + str(dt_obj) + '"')
 582
 583             iconfilename = "%s.%s.png" % (
 584                 apk['id'],
 585                 apk['versioncode'])
 586
 587             # Extract the icon file...
 588             empty_densities = []
 589             for density in screen_densities:
 590                 if density not in apk['icons_src']:
 591                     empty_densities.append(density)
 592                     continue
 593                 iconsrc = apk['icons_src'][density]
 594                 icon_dir = get_icon_dir(repodir, density)
 595                 icondest = os.path.join(icon_dir, iconfilename)
 596
 597                 try:
 598                     with open(icondest, 'wb') as f:
 599                         f.write(apkzip.read(iconsrc))
 600                     apk['icons'][density] = iconfilename
 601
 602                 except:
 603                     logging.warn("Error retrieving icon file")
 604                     del apk['icons'][density]
 605                     del apk['icons_src'][density]
 606                     empty_densities.append(density)
 607
 608             if '-1' in apk['icons_src']:
 609                 iconsrc = apk['icons_src']['-1']
 610                 iconpath = os.path.join(
 611                     get_icon_dir(repodir, '0'), iconfilename)
 612                 with open(iconpath, 'wb') as f:
 613                     f.write(apkzip.read(iconsrc))
 614                 try:
 615                     im = Image.open(iconpath)
 616                     dpi = px_to_dpi(im.size[0])
 617                     for density in screen_densities:
 618                         if density in apk['icons']:
 619                             break
 620                         if density == screen_densities[-1] or dpi >= int(density):
 621                             apk['icons'][density] = iconfilename
 622                             shutil.move(iconpath,
 623                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 624                             empty_densities.remove(density)
 625                             break
 626                 except Exception as e:
 627                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 628
 629             if apk['icons']:
 630                 apk['icon'] = iconfilename
 631
 632             apkzip.close()
 633
 634             # First try resizing down to not lose quality
 635             last_density = None
 636             for density in screen_densities:
 637                 if density not in empty_densities:
 638                     last_density = density
 639                     continue
 640                 if last_density is None:
 641                     continue
 642                 logging.debug("Density %s not available, resizing down from %s"
 643                               % (density, last_density))
 644
 645                 last_iconpath = os.path.join(
 646                     get_icon_dir(repodir, last_density), iconfilename)
 647                 iconpath = os.path.join(
 648                     get_icon_dir(repodir, density), iconfilename)
 649                 try:
 650                     im = Image.open(last_iconpath)
 651                 except:
 652                     logging.warn("Invalid image file at %s" % last_iconpath)
 653                     continue
 654
 655                 size = dpi_to_px(density)
 656
 657                 im.thumbnail((size, size), Image.ANTIALIAS)
 658                 im.save(iconpath, "PNG")
 659                 empty_densities.remove(density)
 660
 661             # Then just copy from the highest resolution available
 662             last_density = None
 663             for density in reversed(screen_densities):
 664                 if density not in empty_densities:
 665                     last_density = density
 666                     continue
 667                 if last_density is None:
 668                     continue
 669                 logging.debug("Density %s not available, copying from lower density %s"
 670                               % (density, last_density))
 671
 672                 shutil.copyfile(
 673                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 674                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 675
 676                 empty_densities.remove(density)
 677
 678             for density in screen_densities:
 679                 icon_dir = get_icon_dir(repodir, density)
 680                 icondest = os.path.join(icon_dir, iconfilename)
 681                 resize_icon(icondest, density)
 682
 683             # Copy from icons-mdpi to icons since mdpi is the baseline density
 684             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 685             if os.path.isfile(baseline):
 686                 apk['icons']['0'] = iconfilename
 687                 shutil.copyfile(baseline,
 688                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 689
 690             # Record in known apks, getting the added date at the same time..
 691             added = knownapks.recordapk(apk['apkname'], apk['id'])
 692             if added:
 693                 if use_date_from_apk and manifest.date_time[1] != 0:
 694                     added = datetime(*manifest.date_time).timetuple()
 695                     logging.debug("Using date from APK")
 696
 697                 apk['added'] = added
 698
 699             apkcache[apkfilename] = apk
 700             cachechanged = True
 701
 702         apks.append(apk)
 703
 704     return apks, cachechanged
 705
 706
 707 repo_pubkey_fingerprint = None
 708
 709
 710 # Generate a certificate fingerprint the same way keytool does it
 711 # (but with slightly different formatting)
 712 def cert_fingerprint(data):
 713     digest = hashlib.sha256(data).digest()
 714     ret = []
 715     ret.append(' '.join("%02X" % ord(b) for b in digest))
 716     return " ".join(ret)
 717
 718
 719 def extract_pubkey():
 720     global repo_pubkey_fingerprint
 721     if 'repo_pubkey' in config:
 722         pubkey = unhexlify(config['repo_pubkey'])
 723     else:
 724         p = FDroidPopen([config['keytool'], '-exportcert',
 725                          '-alias', config['repo_keyalias'],
 726                          '-keystore', config['keystore'],
 727                          '-storepass:file', config['keystorepassfile']]
 728                         + config['smartcardoptions'],
 729                         output=False, stderr_to_stdout=False)
 730         if p.returncode != 0 or len(p.output) < 20:
 731             msg = "Failed to get repo pubkey!"
 732             if config['keystore'] == 'NONE':
 733                 msg += ' Is your crypto smartcard plugged in?'
 734             logging.critical(msg)
 735             sys.exit(1)
 736         pubkey = p.output
 737     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 738     return hexlify(pubkey)
 739
 740
 741 def make_index(apps, sortedids, apks, repodir, archive, categories):
 742     """Make a repo index.
 743
 744     :param apps: fully populated apps list
 745     :param apks: full populated apks list
 746     :param repodir: the repo directory
 747     :param archive: True if this is the archive repo, False if it's the
 748                     main one.
 749     :param categories: list of categories
 750     """
 751
 752     doc = Document()
 753
 754     def addElement(name, value, doc, parent):
 755         el = doc.createElement(name)
 756         el.appendChild(doc.createTextNode(value))
 757         parent.appendChild(el)
 758
 759     def addElementNonEmpty(name, value, doc, parent):
 760         if not value:
 761             return
 762         addElement(name, value, doc, parent)
 763
 764     def addElementCDATA(name, value, doc, parent):
 765         el = doc.createElement(name)
 766         el.appendChild(doc.createCDATASection(value))
 767         parent.appendChild(el)
 768
 769     root = doc.createElement("fdroid")
 770     doc.appendChild(root)
 771
 772     repoel = doc.createElement("repo")
 773
 774     mirrorcheckfailed = False
 775     for mirror in config.get('mirrors', []):
 776         base = os.path.basename(urlparse.urlparse(mirror).path.rstrip('/'))
 777         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
 778             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 779             mirrorcheckfailed = True
 780     if mirrorcheckfailed:
 781         sys.exit(1)
 782
 783     if archive:
 784         repoel.setAttribute("name", config['archive_name'])
 785         if config['repo_maxage'] != 0:
 786             repoel.setAttribute("maxage", str(config['repo_maxage']))
 787         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 788         repoel.setAttribute("url", config['archive_url'])
 789         addElement('description', config['archive_description'], doc, repoel)
 790         urlbasepath = os.path.basename(urlparse.urlparse(config['archive_url']).path)
 791         for mirror in config.get('mirrors', []):
 792             addElement('mirror', urlparse.urljoin(mirror, urlbasepath), doc, repoel)
 793
 794     else:
 795         repoel.setAttribute("name", config['repo_name'])
 796         if config['repo_maxage'] != 0:
 797             repoel.setAttribute("maxage", str(config['repo_maxage']))
 798         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 799         repoel.setAttribute("url", config['repo_url'])
 800         addElement('description', config['repo_description'], doc, repoel)
 801         urlbasepath = os.path.basename(urlparse.urlparse(config['repo_url']).path)
 802         for mirror in config.get('mirrors', []):
 803             addElement('mirror', urlparse.urljoin(mirror, urlbasepath), doc, repoel)
 804
 805     repoel.setAttribute("version", "15")
 806     repoel.setAttribute("timestamp", str(int(time.time())))
 807
 808     nosigningkey = False
 809     if not options.nosign:
 810         if 'repo_keyalias' not in config:
 811             nosigningkey = True
 812             logging.critical("'repo_keyalias' not found in config.py!")
 813         if 'keystore' not in config:
 814             nosigningkey = True
 815             logging.critical("'keystore' not found in config.py!")
 816         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 817             nosigningkey = True
 818             logging.critical("'keystorepass' not found in config.py!")
 819         if 'keypass' not in config and 'keypassfile' not in config:
 820             nosigningkey = True
 821             logging.critical("'keypass' not found in config.py!")
 822         if not os.path.exists(config['keystore']):
 823             nosigningkey = True
 824             logging.critical("'" + config['keystore'] + "' does not exist!")
 825         if nosigningkey:
 826             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 827             logging.warning("\tfdroid update --create-key")
 828             sys.exit(1)
 829
 830     repoel.setAttribute("pubkey", extract_pubkey())
 831     root.appendChild(repoel)
 832
 833     for appid in sortedids:
 834         app = apps[appid]
 835
 836         if app.Disabled is not None:
 837             continue
 838
 839         # Get a list of the apks for this app...
 840         apklist = []
 841         for apk in apks:
 842             if apk['id'] == appid:
 843                 apklist.append(apk)
 844
 845         if len(apklist) == 0:
 846             continue
 847
 848         apel = doc.createElement("application")
 849         apel.setAttribute("id", app.id)
 850         root.appendChild(apel)
 851
 852         addElement('id', app.id, doc, apel)
 853         if app.added:
 854             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
 855         if app.lastupdated:
 856             addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
 857         addElement('name', app.Name, doc, apel)
 858         addElement('summary', app.Summary, doc, apel)
 859         if app.icon:
 860             addElement('icon', app.icon, doc, apel)
 861
 862         def linkres(appid):
 863             if appid in apps:
 864                 return ("fdroid.app:" + appid, apps[appid].Name)
 865             raise MetaDataException("Cannot resolve app id " + appid)
 866
 867         addElement('desc',
 868                    metadata.description_html(app.Description, linkres),
 869                    doc, apel)
 870         addElement('license', app.License, doc, apel)
 871         if app.Categories:
 872             addElement('categories', ','.join(app.Categories), doc, apel)
 873             # We put the first (primary) category in LAST, which will have
 874             # the desired effect of making clients that only understand one
 875             # category see that one.
 876             addElement('category', app.Categories[0], doc, apel)
 877         addElement('web', app.WebSite, doc, apel)
 878         addElement('source', app.SourceCode, doc, apel)
 879         addElement('tracker', app.IssueTracker, doc, apel)
 880         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 881         addElementNonEmpty('author', app.AuthorName, doc, apel)
 882         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 883         addElementNonEmpty('donate', app.Donate, doc, apel)
 884         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 885         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 886         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 887
 888         # These elements actually refer to the current version (i.e. which
 889         # one is recommended. They are historically mis-named, and need
 890         # changing, but stay like this for now to support existing clients.
 891         addElement('marketversion', app.CurrentVersion, doc, apel)
 892         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 893
 894         if app.AntiFeatures:
 895             af = app.AntiFeatures
 896             if af:
 897                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 898         if app.Provides:
 899             pv = app.Provides.split(',')
 900             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 901         if app.RequiresRoot:
 902             addElement('requirements', 'root', doc, apel)
 903
 904         # Sort the apk list into version order, just so the web site
 905         # doesn't have to do any work by default...
 906         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 907
 908         # Check for duplicates - they will make the client unhappy...
 909         for i in range(len(apklist) - 1):
 910             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 911                 logging.critical("duplicate versions: '%s' - '%s'" % (
 912                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 913                 sys.exit(1)
 914
 915         current_version_code = 0
 916         current_version_file = None
 917         for apk in apklist:
 918             # find the APK for the "Current Version"
 919             if current_version_code < apk['versioncode']:
 920                 current_version_code = apk['versioncode']
 921             if current_version_code < int(app.CurrentVersionCode):
 922                 current_version_file = apk['apkname']
 923
 924             apkel = doc.createElement("package")
 925             apel.appendChild(apkel)
 926             addElement('version', apk['version'], doc, apkel)
 927             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 928             addElement('apkname', apk['apkname'], doc, apkel)
 929             if 'srcname' in apk:
 930                 addElement('srcname', apk['srcname'], doc, apkel)
 931             for hash_type in ['sha256']:
 932                 if hash_type not in apk:
 933                     continue
 934                 hashel = doc.createElement("hash")
 935                 hashel.setAttribute("type", hash_type)
 936                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 937                 apkel.appendChild(hashel)
 938             addElement('sig', apk['sig'], doc, apkel)
 939             addElement('size', str(apk['size']), doc, apkel)
 940             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 941             if 'maxsdkversion' in apk:
 942                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 943             if 'added' in apk:
 944                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 945             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
 946             if 'nativecode' in apk:
 947                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 948             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
 949
 950         if current_version_file is not None \
 951                 and config['make_current_version_link'] \
 952                 and repodir == 'repo':  # only create these
 953             namefield = config['current_version_name_source']
 954             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
 955             apklinkname = sanitized_name + '.apk'
 956             current_version_path = os.path.join(repodir, current_version_file)
 957             if os.path.islink(apklinkname):
 958                 os.remove(apklinkname)
 959             os.symlink(current_version_path, apklinkname)
 960             # also symlink gpg signature, if it exists
 961             for extension in ('.asc', '.sig'):
 962                 sigfile_path = current_version_path + extension
 963                 if os.path.exists(sigfile_path):
 964                     siglinkname = apklinkname + extension
 965                     if os.path.islink(siglinkname):
 966                         os.remove(siglinkname)
 967                     os.symlink(sigfile_path, siglinkname)
 968
 969     if options.pretty:
 970         output = doc.toprettyxml()
 971     else:
 972         output = doc.toxml()
 973
 974     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 975         f.write(output)
 976
 977     if 'repo_keyalias' in config:
 978
 979         if options.nosign:
 980             logging.info("Creating unsigned index in preparation for signing")
 981         else:
 982             logging.info("Creating signed index with this key (SHA256):")
 983             logging.info("%s" % repo_pubkey_fingerprint)
 984
 985         # Create a jar of the index...
 986         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
 987         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 988         if p.returncode != 0:
 989             logging.critical("Failed to create {0}".format(jar_output))
 990             sys.exit(1)
 991
 992         # Sign the index...
 993         signed = os.path.join(repodir, 'index.jar')
 994         if options.nosign:
 995             # Remove old signed index if not signing
 996             if os.path.exists(signed):
 997                 os.remove(signed)
 998         else:
 999             args = [config['jarsigner'], '-keystore', config['keystore'],
1000                     '-storepass:file', config['keystorepassfile'],
1001                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1002                     signed, config['repo_keyalias']]
1003             if config['keystore'] == 'NONE':
1004                 args += config['smartcardoptions']
1005             else:  # smardcards never use -keypass
1006                 args += ['-keypass:file', config['keypassfile']]
1007             p = FDroidPopen(args)
1008             if p.returncode != 0:
1009                 logging.critical("Failed to sign index")
1010                 sys.exit(1)
1011
1012     # Copy the repo icon into the repo directory...
1013     icon_dir = os.path.join(repodir, 'icons')
1014     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1015     shutil.copyfile(config['repo_icon'], iconfilename)
1016
1017     # Write a category list in the repo to allow quick access...
1018     catdata = ''
1019     for cat in categories:
1020         catdata += cat + '\n'
1021     with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
1022         f.write(catdata)
1023
1024
1025 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1026
1027     for appid, app in apps.iteritems():
1028
1029         if app.ArchivePolicy:
1030             keepversions = int(app.ArchivePolicy[:-9])
1031         else:
1032             keepversions = defaultkeepversions
1033
1034         def filter_apk_list_sorted(apk_list):
1035             res = []
1036             for apk in apk_list:
1037                 if apk['id'] == appid:
1038                     res.append(apk)
1039
1040             # Sort the apk list by version code. First is highest/newest.
1041             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1042
1043         def move_file(from_dir, to_dir, filename, ignore_missing):
1044             from_path = os.path.join(from_dir, filename)
1045             if ignore_missing and not os.path.exists(from_path):
1046                 return
1047             to_path = os.path.join(to_dir, filename)
1048             shutil.move(from_path, to_path)
1049
1050         if len(apks) > keepversions:
1051             apklist = filter_apk_list_sorted(apks)
1052             # Move back the ones we don't want.
1053             for apk in apklist[keepversions:]:
1054                 logging.info("Moving " + apk['apkname'] + " to archive")
1055                 move_file(repodir, archivedir, apk['apkname'], False)
1056                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1057                 for density in all_screen_densities:
1058                     repo_icon_dir = get_icon_dir(repodir, density)
1059                     archive_icon_dir = get_icon_dir(archivedir, density)
1060                     if density not in apk['icons']:
1061                         continue
1062                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1063                 if 'srcname' in apk:
1064                     move_file(repodir, archivedir, apk['srcname'], False)
1065                 archapks.append(apk)
1066                 apks.remove(apk)
1067         elif len(apks) < keepversions and len(archapks) > 0:
1068             required = keepversions - len(apks)
1069             archapklist = filter_apk_list_sorted(archapks)
1070             # Move forward the ones we want again.
1071             for apk in archapklist[:required]:
1072                 logging.info("Moving " + apk['apkname'] + " from archive")
1073                 move_file(archivedir, repodir, apk['apkname'], False)
1074                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1075                 for density in all_screen_densities:
1076                     repo_icon_dir = get_icon_dir(repodir, density)
1077                     archive_icon_dir = get_icon_dir(archivedir, density)
1078                     if density not in apk['icons']:
1079                         continue
1080                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1081                 if 'srcname' in apk:
1082                     move_file(archivedir, repodir, apk['srcname'], False)
1083                 archapks.remove(apk)
1084                 apks.append(apk)
1085
1086
1087 def add_apks_to_per_app_repos(repodir, apks):
1088     apks_per_app = dict()
1089     for apk in apks:
1090         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1091         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1092         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1093         apks_per_app[apk['id']] = apk
1094
1095         if not os.path.exists(apk['per_app_icons']):
1096             logging.info('Adding new repo for only ' + apk['id'])
1097             os.makedirs(apk['per_app_icons'])
1098
1099         apkpath = os.path.join(repodir, apk['apkname'])
1100         shutil.copy(apkpath, apk['per_app_repo'])
1101         apksigpath = apkpath + '.sig'
1102         if os.path.exists(apksigpath):
1103             shutil.copy(apksigpath, apk['per_app_repo'])
1104         apkascpath = apkpath + '.asc'
1105         if os.path.exists(apkascpath):
1106             shutil.copy(apkascpath, apk['per_app_repo'])
1107
1108
1109 config = None
1110 options = None
1111
1112
1113 def main():
1114
1115     global config, options
1116
1117     # Parse command line...
1118     parser = ArgumentParser()
1119     common.setup_global_opts(parser)
1120     parser.add_argument("--create-key", action="store_true", default=False,
1121                         help="Create a repo signing key in a keystore")
1122     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1123                         help="Create skeleton metadata files that are missing")
1124     parser.add_argument("--delete-unknown", action="store_true", default=False,
1125                         help="Delete APKs without metadata from the repo")
1126     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1127                         help="Report on build data status")
1128     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1129                         help="Interactively ask about things that need updating.")
1130     parser.add_argument("-I", "--icons", action="store_true", default=False,
1131                         help="Resize all the icons exceeding the max pixel size and exit")
1132     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1133                         help="Specify editor to use in interactive mode. Default " +
1134                         "is /etc/alternatives/editor")
1135     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1136                         help="Update the wiki")
1137     parser.add_argument("--pretty", action="store_true", default=False,
1138                         help="Produce human-readable index.xml")
1139     parser.add_argument("--clean", action="store_true", default=False,
1140                         help="Clean update - don't uses caches, reprocess all apks")
1141     parser.add_argument("--nosign", action="store_true", default=False,
1142                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1143     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1144                         help="Use date from apk instead of current time for newly added apks")
1145     options = parser.parse_args()
1146
1147     config = common.read_config(options)
1148
1149     if not ('jarsigner' in config and 'keytool' in config):
1150         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1151         sys.exit(1)
1152
1153     repodirs = ['repo']
1154     if config['archive_older'] != 0:
1155         repodirs.append('archive')
1156         if not os.path.exists('archive'):
1157             os.mkdir('archive')
1158
1159     if options.icons:
1160         resize_all_icons(repodirs)
1161         sys.exit(0)
1162
1163     # check that icons exist now, rather than fail at the end of `fdroid update`
1164     for k in ['repo_icon', 'archive_icon']:
1165         if k in config:
1166             if not os.path.exists(config[k]):
1167                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1168                 sys.exit(1)
1169
1170     # if the user asks to create a keystore, do it now, reusing whatever it can
1171     if options.create_key:
1172         if os.path.exists(config['keystore']):
1173             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1174             logging.critical("\t'" + config['keystore'] + "'")
1175             sys.exit(1)
1176
1177         if 'repo_keyalias' not in config:
1178             config['repo_keyalias'] = socket.getfqdn()
1179             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1180         if 'keydname' not in config:
1181             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1182             common.write_to_config(config, 'keydname', config['keydname'])
1183         if 'keystore' not in config:
1184             config['keystore'] = common.default_config.keystore
1185             common.write_to_config(config, 'keystore', config['keystore'])
1186
1187         password = common.genpassword()
1188         if 'keystorepass' not in config:
1189             config['keystorepass'] = password
1190             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1191         if 'keypass' not in config:
1192             config['keypass'] = password
1193             common.write_to_config(config, 'keypass', config['keypass'])
1194         common.genkeystore(config)
1195
1196     # Get all apps...
1197     apps = metadata.read_metadata()
1198
1199     # Generate a list of categories...
1200     categories = set()
1201     for app in apps.itervalues():
1202         categories.update(app.Categories)
1203
1204     # Read known apks data (will be updated and written back when we've finished)
1205     knownapks = common.KnownApks()
1206
1207     # Gather information about all the apk files in the repo directory, using
1208     # cached data if possible.
1209     apkcachefile = os.path.join('tmp', 'apkcache')
1210     if not options.clean and os.path.exists(apkcachefile):
1211         with open(apkcachefile, 'rb') as cf:
1212             apkcache = pickle.load(cf)
1213     else:
1214         apkcache = {}
1215
1216     delete_disabled_builds(apps, apkcache, repodirs)
1217
1218     # Scan all apks in the main repo
1219     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1220
1221     # Generate warnings for apk's with no metadata (or create skeleton
1222     # metadata files, if requested on the command line)
1223     newmetadata = False
1224     for apk in apks:
1225         if apk['id'] not in apps:
1226             if options.create_metadata:
1227                 if 'name' not in apk:
1228                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1229                     continue
1230                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1231                 f.write("License:Unknown\n")
1232                 f.write("Web Site:\n")
1233                 f.write("Source Code:\n")
1234                 f.write("Issue Tracker:\n")
1235                 f.write("Changelog:\n")
1236                 f.write("Summary:" + apk['name'] + "\n")
1237                 f.write("Description:\n")
1238                 f.write(apk['name'] + "\n")
1239                 f.write(".\n")
1240                 f.close()
1241                 logging.info("Generated skeleton metadata for " + apk['id'])
1242                 newmetadata = True
1243             else:
1244                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1245                 if options.delete_unknown:
1246                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1247                     rmf = os.path.join(repodirs[0], apk['apkname'])
1248                     if not os.path.exists(rmf):
1249                         logging.error("Could not find {0} to remove it".format(rmf))
1250                     else:
1251                         os.remove(rmf)
1252                 else:
1253                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1254
1255     # update the metadata with the newly created ones included
1256     if newmetadata:
1257         apps = metadata.read_metadata()
1258
1259     # Scan the archive repo for apks as well
1260     if len(repodirs) > 1:
1261         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1262         if cc:
1263             cachechanged = True
1264     else:
1265         archapks = []
1266
1267     # Some information from the apks needs to be applied up to the application
1268     # level. When doing this, we use the info from the most recent version's apk.
1269     # We deal with figuring out when the app was added and last updated at the
1270     # same time.
1271     for appid, app in apps.iteritems():
1272         bestver = 0
1273         for apk in apks + archapks:
1274             if apk['id'] == appid:
1275                 if apk['versioncode'] > bestver:
1276                     bestver = apk['versioncode']
1277                     bestapk = apk
1278
1279                 if 'added' in apk:
1280                     if not app.added or apk['added'] < app.added:
1281                         app.added = apk['added']
1282                     if not app.lastupdated or apk['added'] > app.lastupdated:
1283                         app.lastupdated = apk['added']
1284
1285         if not app.added:
1286             logging.debug("Don't know when " + appid + " was added")
1287         if not app.lastupdated:
1288             logging.debug("Don't know when " + appid + " was last updated")
1289
1290         if bestver == 0:
1291             if app.Name is None:
1292                 app.Name = app.AutoName or appid
1293             app.icon = None
1294             logging.debug("Application " + appid + " has no packages")
1295         else:
1296             if app.Name is None:
1297                 app.Name = bestapk['name']
1298             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1299             if app.CurrentVersionCode is None:
1300                 app.CurrentVersionCode = str(bestver)
1301
1302     # Sort the app list by name, then the web site doesn't have to by default.
1303     # (we had to wait until we'd scanned the apks to do this, because mostly the
1304     # name comes from there!)
1305     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid].Name.upper())
1306
1307     # APKs are placed into multiple repos based on the app package, providing
1308     # per-app subscription feeds for nightly builds and things like it
1309     if config['per_app_repos']:
1310         add_apks_to_per_app_repos(repodirs[0], apks)
1311         for appid, app in apps.iteritems():
1312             repodir = os.path.join(appid, 'fdroid', 'repo')
1313             appdict = dict()
1314             appdict[appid] = app
1315             if os.path.isdir(repodir):
1316                 make_index(appdict, [appid], apks, repodir, False, categories)
1317             else:
1318                 logging.info('Skipping index generation for ' + appid)
1319         return
1320
1321     if len(repodirs) > 1:
1322         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1323
1324     # Make the index for the main repo...
1325     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1326
1327     # If there's an archive repo,  make the index for it. We already scanned it
1328     # earlier on.
1329     if len(repodirs) > 1:
1330         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1331
1332     if config['update_stats']:
1333
1334         # Update known apks info...
1335         knownapks.writeifchanged()
1336
1337         # Generate latest apps data for widget
1338         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1339             data = ''
1340             for line in file(os.path.join('stats', 'latestapps.txt')):
1341                 appid = line.rstrip()
1342                 data += appid + "\t"
1343                 app = apps[appid]
1344                 data += app.Name + "\t"
1345                 if app.icon is not None:
1346                     data += app.icon + "\t"
1347                 data += app.License + "\n"
1348             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
1349                 f.write(data)
1350
1351     if cachechanged:
1352         with open(apkcachefile, 'wb') as cf:
1353             pickle.dump(apkcache, cf)
1354
1355     # Update the wiki...
1356     if options.wiki:
1357         update_wiki(apps, sortedids, apks + archapks)
1358
1359     logging.info("Finished.")
1360
1361 if __name__ == "__main__":
1362     main()