fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import socket
  27 import zipfile
  28 import hashlib
  29 import pickle
  30 import urlparse
  31 from datetime import datetime, timedelta
  32 from xml.dom.minidom import Document
  33 from argparse import ArgumentParser
  34 import time
  35 from pyasn1.error import PyAsn1Error
  36 from pyasn1.codec.der import decoder, encoder
  37 from pyasn1_modules import rfc2315
  38 from hashlib import md5
  39 from binascii import hexlify, unhexlify
  40
  41 from PIL import Image
  42 import logging
  43
  44 import common
  45 import metadata
  46 from common import FDroidPopen, SdkToolsPopen
  47 from metadata import MetaDataException
  48
  49 screen_densities = ['640', '480', '320', '240', '160', '120']
  50
  51 all_screen_densities = ['0'] + screen_densities
  52
  53
  54 def dpi_to_px(density):
  55     return (int(density) * 48) / 160
  56
  57
  58 def px_to_dpi(px):
  59     return (int(px) * 160) / 48
  60
  61
  62 def get_icon_dir(repodir, density):
  63     if density == '0':
  64         return os.path.join(repodir, "icons")
  65     return os.path.join(repodir, "icons-%s" % density)
  66
  67
  68 def get_icon_dirs(repodir):
  69     for density in screen_densities:
  70         yield get_icon_dir(repodir, density)
  71
  72
  73 def get_all_icon_dirs(repodir):
  74     for density in all_screen_densities:
  75         yield get_icon_dir(repodir, density)
  76
  77
  78 def update_wiki(apps, sortedids, apks):
  79     """Update the wiki
  80
  81     :param apps: fully populated list of all applications
  82     :param apks: all apks, except...
  83     """
  84     logging.info("Updating wiki")
  85     wikicat = 'Apps'
  86     wikiredircat = 'App Redirects'
  87     import mwclient
  88     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  89                          path=config['wiki_path'])
  90     site.login(config['wiki_user'], config['wiki_password'])
  91     generated_pages = {}
  92     generated_redirects = {}
  93
  94     for appid in sortedids:
  95         app = apps[appid]
  96
  97         wikidata = ''
  98         if app.Disabled:
  99             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 100         if app.AntiFeatures:
 101             for af in app.AntiFeatures:
 102                 wikidata += '{{AntiFeature|' + af + '}}\n'
 103         if app.RequiresRoot:
 104             requiresroot = 'Yes'
 105         else:
 106             requiresroot = 'No'
 107         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 108             appid,
 109             app.Name,
 110             time.strftime('%Y-%m-%d', app.added) if app.added else '',
 111             time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
 112             app.SourceCode,
 113             app.IssueTracker,
 114             app.WebSite,
 115             app.Changelog,
 116             app.Donate,
 117             app.FlattrID,
 118             app.Bitcoin,
 119             app.Litecoin,
 120             app.License,
 121             requiresroot,
 122             app.AuthorName,
 123             app.AuthorEmail)
 124
 125         if app.Provides:
 126             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 127
 128         wikidata += app.Summary
 129         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 130
 131         wikidata += "=Description=\n"
 132         wikidata += metadata.description_wiki(app.Description) + "\n"
 133
 134         wikidata += "=Maintainer Notes=\n"
 135         if app.MaintainerNotes:
 136             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 137         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 138
 139         # Get a list of all packages for this application...
 140         apklist = []
 141         gotcurrentver = False
 142         cantupdate = False
 143         buildfails = False
 144         for apk in apks:
 145             if apk['id'] == appid:
 146                 if str(apk['versioncode']) == app.CurrentVersionCode:
 147                     gotcurrentver = True
 148                 apklist.append(apk)
 149         # Include ones we can't build, as a special case...
 150         for build in app.builds:
 151             if build.disable:
 152                 if build.vercode == app.CurrentVersionCode:
 153                     cantupdate = True
 154                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 155                 apklist.append({'versioncode': int(build.vercode),
 156                                 'version': build.version,
 157                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 158                                 })
 159             else:
 160                 builtit = False
 161                 for apk in apklist:
 162                     if apk['versioncode'] == int(build.vercode):
 163                         builtit = True
 164                         break
 165                 if not builtit:
 166                     buildfails = True
 167                     apklist.append({'versioncode': int(build.vercode),
 168                                     'version': build.version,
 169                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
 170                                     })
 171         if app.CurrentVersionCode == '0':
 172             cantupdate = True
 173         # Sort with most recent first...
 174         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 175
 176         wikidata += "=Versions=\n"
 177         if len(apklist) == 0:
 178             wikidata += "We currently have no versions of this app available."
 179         elif not gotcurrentver:
 180             wikidata += "We don't have the current version of this app."
 181         else:
 182             wikidata += "We have the current version of this app."
 183         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 184         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 185         if len(app.NoSourceSince) > 0:
 186             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 187         if len(app.CurrentVersion) > 0:
 188             wikidata += "The current (recommended) version is " + app.CurrentVersion
 189             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 190         validapks = 0
 191         for apk in apklist:
 192             wikidata += "==" + apk['version'] + "==\n"
 193
 194             if 'buildproblem' in apk:
 195                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 196             else:
 197                 validapks += 1
 198                 wikidata += "This version is built and signed by "
 199                 if 'srcname' in apk:
 200                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 201                 else:
 202                     wikidata += "the original developer.\n\n"
 203             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 204
 205         wikidata += '\n[[Category:' + wikicat + ']]\n'
 206         if len(app.NoSourceSince) > 0:
 207             wikidata += '\n[[Category:Apps missing source code]]\n'
 208         if validapks == 0 and not app.Disabled:
 209             wikidata += '\n[[Category:Apps with no packages]]\n'
 210         if cantupdate and not app.Disabled:
 211             wikidata += "\n[[Category:Apps we cannot update]]\n"
 212         if buildfails and not app.Disabled:
 213             wikidata += "\n[[Category:Apps with failing builds]]\n"
 214         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 215             wikidata += '\n[[Category:Apps to Update]]\n'
 216         if app.Disabled:
 217             wikidata += '\n[[Category:Apps that are disabled]]\n'
 218         if app.UpdateCheckMode == 'None' and not app.Disabled:
 219             wikidata += '\n[[Category:Apps with no update check]]\n'
 220         for appcat in app.Categories:
 221             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 222
 223         # We can't have underscores in the page name, even if they're in
 224         # the package ID, because MediaWiki messes with them...
 225         pagename = appid.replace('_', ' ')
 226
 227         # Drop a trailing newline, because mediawiki is going to drop it anyway
 228         # and it we don't we'll think the page has changed when it hasn't...
 229         if wikidata.endswith('\n'):
 230             wikidata = wikidata[:-1]
 231
 232         generated_pages[pagename] = wikidata
 233
 234         # Make a redirect from the name to the ID too, unless there's
 235         # already an existing page with the name and it isn't a redirect.
 236         noclobber = False
 237         apppagename = app.Name.replace('_', ' ')
 238         apppagename = apppagename.replace('{', '')
 239         apppagename = apppagename.replace('}', ' ')
 240         apppagename = apppagename.replace(':', ' ')
 241         # Drop double spaces caused mostly by replacing ':' above
 242         apppagename = apppagename.replace('  ', ' ')
 243         for expagename in site.allpages(prefix=apppagename,
 244                                         filterredir='nonredirects',
 245                                         generator=False):
 246             if expagename == apppagename:
 247                 noclobber = True
 248         # Another reason not to make the redirect page is if the app name
 249         # is the same as it's ID, because that will overwrite the real page
 250         # with an redirect to itself! (Although it seems like an odd
 251         # scenario this happens a lot, e.g. where there is metadata but no
 252         # builds or binaries to extract a name from.
 253         if apppagename == pagename:
 254             noclobber = True
 255         if not noclobber:
 256             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 257
 258     for tcat, genp in [(wikicat, generated_pages),
 259                        (wikiredircat, generated_redirects)]:
 260         catpages = site.Pages['Category:' + tcat]
 261         existingpages = []
 262         for page in catpages:
 263             existingpages.append(page.name)
 264             if page.name in genp:
 265                 pagetxt = page.edit()
 266                 if pagetxt != genp[page.name]:
 267                     logging.debug("Updating modified page " + page.name)
 268                     page.save(genp[page.name], summary='Auto-updated')
 269                 else:
 270                     logging.debug("Page " + page.name + " is unchanged")
 271             else:
 272                 logging.warn("Deleting page " + page.name)
 273                 page.delete('No longer published')
 274         for pagename, text in genp.items():
 275             logging.debug("Checking " + pagename)
 276             if pagename not in existingpages:
 277                 logging.debug("Creating page " + pagename)
 278                 try:
 279                     newpage = site.Pages[pagename]
 280                     newpage.save(text, summary='Auto-created')
 281                 except:
 282                     logging.error("...FAILED to create page '{0}'".format(pagename))
 283
 284     # Purge server cache to ensure counts are up to date
 285     site.pages['Repository Maintenance'].purge()
 286
 287
 288 def delete_disabled_builds(apps, apkcache, repodirs):
 289     """Delete disabled build outputs.
 290
 291     :param apps: list of all applications, as per metadata.read_metadata
 292     :param apkcache: current apk cache information
 293     :param repodirs: the repo directories to process
 294     """
 295     for appid, app in apps.iteritems():
 296         for build in app.builds:
 297             if not build.disable:
 298                 continue
 299             apkfilename = appid + '_' + str(build.vercode) + '.apk'
 300             iconfilename = "%s.%s.png" % (
 301                 appid,
 302                 build.vercode)
 303             for repodir in repodirs:
 304                 files = [
 305                     os.path.join(repodir, apkfilename),
 306                     os.path.join(repodir, apkfilename + '.asc'),
 307                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 308                 ]
 309                 for density in all_screen_densities:
 310                     repo_dir = get_icon_dir(repodir, density)
 311                     files.append(os.path.join(repo_dir, iconfilename))
 312
 313                 for f in files:
 314                     if os.path.exists(f):
 315                         logging.info("Deleting disabled build output " + f)
 316                         os.remove(f)
 317             if apkfilename in apkcache:
 318                 del apkcache[apkfilename]
 319
 320
 321 def resize_icon(iconpath, density):
 322
 323     if not os.path.isfile(iconpath):
 324         return
 325
 326     try:
 327         im = Image.open(iconpath)
 328         size = dpi_to_px(density)
 329
 330         if any(length > size for length in im.size):
 331             oldsize = im.size
 332             im.thumbnail((size, size), Image.ANTIALIAS)
 333             logging.debug("%s was too large at %s - new size is %s" % (
 334                 iconpath, oldsize, im.size))
 335             im.save(iconpath, "PNG")
 336
 337     except Exception as e:
 338         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 339
 340
 341 def resize_all_icons(repodirs):
 342     """Resize all icons that exceed the max size
 343
 344     :param repodirs: the repo directories to process
 345     """
 346     for repodir in repodirs:
 347         for density in screen_densities:
 348             icon_dir = get_icon_dir(repodir, density)
 349             icon_glob = os.path.join(icon_dir, '*.png')
 350             for iconpath in glob.glob(icon_glob):
 351                 resize_icon(iconpath, density)
 352
 353
 354 # A signature block file with a .DSA, .RSA, or .EC extension
 355 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 356
 357
 358 def getsig(apkpath):
 359     """ Get the signing certificate of an apk. To get the same md5 has that
 360     Android gets, we encode the .RSA certificate in a specific format and pass
 361     it hex-encoded to the md5 digest algorithm.
 362
 363     :param apkpath: path to the apk
 364     :returns: A string containing the md5 of the signature of the apk or None
 365               if an error occurred.
 366     """
 367
 368     cert = None
 369
 370     # verify the jar signature is correct
 371     args = [config['jarsigner'], '-verify', apkpath]
 372     p = FDroidPopen(args)
 373     if p.returncode != 0:
 374         logging.critical(apkpath + " has a bad signature!")
 375         return None
 376
 377     with zipfile.ZipFile(apkpath, 'r') as apk:
 378
 379         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 380
 381         if len(certs) < 1:
 382             logging.error("Found no signing certificates on %s" % apkpath)
 383             return None
 384         if len(certs) > 1:
 385             logging.error("Found multiple signing certificates on %s" % apkpath)
 386             return None
 387
 388         cert = apk.read(certs[0])
 389
 390     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 391     if content.getComponentByName('contentType') != rfc2315.signedData:
 392         logging.error("Unexpected format.")
 393         return None
 394
 395     content = decoder.decode(content.getComponentByName('content'),
 396                              asn1Spec=rfc2315.SignedData())[0]
 397     try:
 398         certificates = content.getComponentByName('certificates')
 399     except PyAsn1Error:
 400         logging.error("Certificates not found.")
 401         return None
 402
 403     cert_encoded = encoder.encode(certificates)[4:]
 404
 405     return md5(cert_encoded.encode('hex')).hexdigest()
 406
 407
 408 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
 409     """Scan the apks in the given repo directory.
 410
 411     This also extracts the icons.
 412
 413     :param apps: list of all applications, as per metadata.read_metadata
 414     :param apkcache: current apk cache information
 415     :param repodir: repo directory to scan
 416     :param knownapks: known apks info
 417     :param use_date_from_apk: use date from APK (instead of current date)
 418                               for newly added APKs
 419     :returns: (apks, cachechanged) where apks is a list of apk information,
 420               and cachechanged is True if the apkcache got changed.
 421     """
 422
 423     cachechanged = False
 424
 425     for icon_dir in get_all_icon_dirs(repodir):
 426         if os.path.exists(icon_dir):
 427             if options.clean:
 428                 shutil.rmtree(icon_dir)
 429                 os.makedirs(icon_dir)
 430         else:
 431             os.makedirs(icon_dir)
 432
 433     apks = []
 434     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 435     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 436     vername_pat = re.compile(".*versionName='([^']*)'.*")
 437     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 438     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 439     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 440     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 441     string_pat = re.compile(".*'([^']*)'.*")
 442     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 443
 444         apkfilename = apkfile[len(repodir) + 1:]
 445         if ' ' in apkfilename:
 446             logging.critical("Spaces in filenames are not allowed.")
 447             sys.exit(1)
 448
 449         # Calculate the sha256...
 450         sha = hashlib.sha256()
 451         with open(apkfile, 'rb') as f:
 452             while True:
 453                 t = f.read(16384)
 454                 if len(t) == 0:
 455                     break
 456                 sha.update(t)
 457             shasum = sha.hexdigest()
 458
 459         usecache = False
 460         if apkfilename in apkcache:
 461             apk = apkcache[apkfilename]
 462             if apk['sha256'] == shasum:
 463                 logging.debug("Reading " + apkfilename + " from cache")
 464                 usecache = True
 465             else:
 466                 logging.debug("Ignoring stale cache data for " + apkfilename)
 467
 468         if not usecache:
 469             logging.debug("Processing " + apkfilename)
 470             apk = {}
 471             apk['apkname'] = apkfilename
 472             apk['sha256'] = shasum
 473             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 474             if os.path.exists(os.path.join(repodir, srcfilename)):
 475                 apk['srcname'] = srcfilename
 476             apk['size'] = os.path.getsize(apkfile)
 477             apk['permissions'] = set()
 478             apk['features'] = set()
 479             apk['icons_src'] = {}
 480             apk['icons'] = {}
 481             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 482             if p.returncode != 0:
 483                 if options.delete_unknown:
 484                     if os.path.exists(apkfile):
 485                         logging.error("Failed to get apk information, deleting " + apkfile)
 486                         os.remove(apkfile)
 487                     else:
 488                         logging.error("Could not find {0} to remove it".format(apkfile))
 489                 else:
 490                     logging.error("Failed to get apk information, skipping " + apkfile)
 491                 continue
 492             for line in p.output.splitlines():
 493                 if line.startswith("package:"):
 494                     try:
 495                         apk['id'] = re.match(name_pat, line).group(1)
 496                         apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
 497                         apk['version'] = re.match(vername_pat, line).group(1)
 498                     except Exception as e:
 499                         logging.error("Package matching failed: " + str(e))
 500                         logging.info("Line was: " + line)
 501                         sys.exit(1)
 502                 elif line.startswith("application:"):
 503                     apk['name'] = re.match(label_pat, line).group(1)
 504                     # Keep path to non-dpi icon in case we need it
 505                     match = re.match(icon_pat_nodpi, line)
 506                     if match:
 507                         apk['icons_src']['-1'] = match.group(1)
 508                 elif line.startswith("launchable-activity:"):
 509                     # Only use launchable-activity as fallback to application
 510                     if not apk['name']:
 511                         apk['name'] = re.match(label_pat, line).group(1)
 512                     if '-1' not in apk['icons_src']:
 513                         match = re.match(icon_pat_nodpi, line)
 514                         if match:
 515                             apk['icons_src']['-1'] = match.group(1)
 516                 elif line.startswith("application-icon-"):
 517                     match = re.match(icon_pat, line)
 518                     if match:
 519                         density = match.group(1)
 520                         path = match.group(2)
 521                         apk['icons_src'][density] = path
 522                 elif line.startswith("sdkVersion:"):
 523                     m = re.match(sdkversion_pat, line)
 524                     if m is None:
 525                         logging.error(line.replace('sdkVersion:', '')
 526                                       + ' is not a valid minSdkVersion!')
 527                     else:
 528                         apk['sdkversion'] = m.group(1)
 529                 elif line.startswith("maxSdkVersion:"):
 530                     apk['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 531                 elif line.startswith("native-code:"):
 532                     apk['nativecode'] = []
 533                     for arch in line[13:].split(' '):
 534                         apk['nativecode'].append(arch[1:-1])
 535                 elif line.startswith("uses-permission:"):
 536                     perm = re.match(string_pat, line).group(1)
 537                     if perm.startswith("android.permission."):
 538                         perm = perm[19:]
 539                     apk['permissions'].add(perm)
 540                 elif line.startswith("uses-feature:"):
 541                     perm = re.match(string_pat, line).group(1)
 542                     # Filter out this, it's only added with the latest SDK tools and
 543                     # causes problems for lots of apps.
 544                     if perm != "android.hardware.screen.portrait" \
 545                             and perm != "android.hardware.screen.landscape":
 546                         if perm.startswith("android.feature."):
 547                             perm = perm[16:]
 548                         apk['features'].add(perm)
 549
 550             if 'sdkversion' not in apk:
 551                 logging.warn("No SDK version information found in {0}".format(apkfile))
 552                 apk['sdkversion'] = 0
 553
 554             # Check for debuggable apks...
 555             if common.isApkDebuggable(apkfile, config):
 556                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 557
 558             # Get the signature (or md5 of, to be precise)...
 559             logging.debug('Getting signature of {0}'.format(apkfile))
 560             apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 561             if not apk['sig']:
 562                 logging.critical("Failed to get apk signature")
 563                 sys.exit(1)
 564
 565             apkzip = zipfile.ZipFile(apkfile, 'r')
 566
 567             # if an APK has files newer than the system time, suggest updating
 568             # the system clock.  This is useful for offline systems, used for
 569             # signing, which do not have another source of clock sync info. It
 570             # has to be more than 24 hours newer because ZIP/APK files do not
 571             # store timezone info
 572             manifest = apkzip.getinfo('AndroidManifest.xml')
 573             if manifest.date_time[1] == 0:  # month can't be zero
 574                 logging.debug('AndroidManifest.xml has no date')
 575             else:
 576                 dt_obj = datetime(*manifest.date_time)
 577                 checkdt = dt_obj - timedelta(1)
 578                 if datetime.today() < checkdt:
 579                     logging.warn('System clock is older than manifest in: '
 580                                  + apkfilename
 581                                  + '\nSet clock to that time using:\n'
 582                                  + 'sudo date -s "' + str(dt_obj) + '"')
 583
 584             iconfilename = "%s.%s.png" % (
 585                 apk['id'],
 586                 apk['versioncode'])
 587
 588             # Extract the icon file...
 589             empty_densities = []
 590             for density in screen_densities:
 591                 if density not in apk['icons_src']:
 592                     empty_densities.append(density)
 593                     continue
 594                 iconsrc = apk['icons_src'][density]
 595                 icon_dir = get_icon_dir(repodir, density)
 596                 icondest = os.path.join(icon_dir, iconfilename)
 597
 598                 try:
 599                     with open(icondest, 'wb') as f:
 600                         f.write(apkzip.read(iconsrc))
 601                     apk['icons'][density] = iconfilename
 602
 603                 except:
 604                     logging.warn("Error retrieving icon file")
 605                     del apk['icons'][density]
 606                     del apk['icons_src'][density]
 607                     empty_densities.append(density)
 608
 609             if '-1' in apk['icons_src']:
 610                 iconsrc = apk['icons_src']['-1']
 611                 iconpath = os.path.join(
 612                     get_icon_dir(repodir, '0'), iconfilename)
 613                 with open(iconpath, 'wb') as f:
 614                     f.write(apkzip.read(iconsrc))
 615                 try:
 616                     im = Image.open(iconpath)
 617                     dpi = px_to_dpi(im.size[0])
 618                     for density in screen_densities:
 619                         if density in apk['icons']:
 620                             break
 621                         if density == screen_densities[-1] or dpi >= int(density):
 622                             apk['icons'][density] = iconfilename
 623                             shutil.move(iconpath,
 624                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 625                             empty_densities.remove(density)
 626                             break
 627                 except Exception as e:
 628                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 629
 630             if apk['icons']:
 631                 apk['icon'] = iconfilename
 632
 633             apkzip.close()
 634
 635             # First try resizing down to not lose quality
 636             last_density = None
 637             for density in screen_densities:
 638                 if density not in empty_densities:
 639                     last_density = density
 640                     continue
 641                 if last_density is None:
 642                     continue
 643                 logging.debug("Density %s not available, resizing down from %s"
 644                               % (density, last_density))
 645
 646                 last_iconpath = os.path.join(
 647                     get_icon_dir(repodir, last_density), iconfilename)
 648                 iconpath = os.path.join(
 649                     get_icon_dir(repodir, density), iconfilename)
 650                 try:
 651                     im = Image.open(last_iconpath)
 652                 except:
 653                     logging.warn("Invalid image file at %s" % last_iconpath)
 654                     continue
 655
 656                 size = dpi_to_px(density)
 657
 658                 im.thumbnail((size, size), Image.ANTIALIAS)
 659                 im.save(iconpath, "PNG")
 660                 empty_densities.remove(density)
 661
 662             # Then just copy from the highest resolution available
 663             last_density = None
 664             for density in reversed(screen_densities):
 665                 if density not in empty_densities:
 666                     last_density = density
 667                     continue
 668                 if last_density is None:
 669                     continue
 670                 logging.debug("Density %s not available, copying from lower density %s"
 671                               % (density, last_density))
 672
 673                 shutil.copyfile(
 674                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 675                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 676
 677                 empty_densities.remove(density)
 678
 679             for density in screen_densities:
 680                 icon_dir = get_icon_dir(repodir, density)
 681                 icondest = os.path.join(icon_dir, iconfilename)
 682                 resize_icon(icondest, density)
 683
 684             # Copy from icons-mdpi to icons since mdpi is the baseline density
 685             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 686             if os.path.isfile(baseline):
 687                 apk['icons']['0'] = iconfilename
 688                 shutil.copyfile(baseline,
 689                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 690
 691             # Record in known apks, getting the added date at the same time..
 692             added = knownapks.recordapk(apk['apkname'], apk['id'])
 693             if added:
 694                 if use_date_from_apk and manifest.date_time[1] != 0:
 695                     added = datetime(*manifest.date_time).timetuple()
 696                     logging.debug("Using date from APK")
 697
 698                 apk['added'] = added
 699
 700             apkcache[apkfilename] = apk
 701             cachechanged = True
 702
 703         apks.append(apk)
 704
 705     return apks, cachechanged
 706
 707
 708 repo_pubkey_fingerprint = None
 709
 710
 711 # Generate a certificate fingerprint the same way keytool does it
 712 # (but with slightly different formatting)
 713 def cert_fingerprint(data):
 714     digest = hashlib.sha256(data).digest()
 715     ret = []
 716     ret.append(' '.join("%02X" % ord(b) for b in digest))
 717     return " ".join(ret)
 718
 719
 720 def extract_pubkey():
 721     global repo_pubkey_fingerprint
 722     if 'repo_pubkey' in config:
 723         pubkey = unhexlify(config['repo_pubkey'])
 724     else:
 725         p = FDroidPopen([config['keytool'], '-exportcert',
 726                          '-alias', config['repo_keyalias'],
 727                          '-keystore', config['keystore'],
 728                          '-storepass:file', config['keystorepassfile']]
 729                         + config['smartcardoptions'],
 730                         output=False, stderr_to_stdout=False)
 731         if p.returncode != 0 or len(p.output) < 20:
 732             msg = "Failed to get repo pubkey!"
 733             if config['keystore'] == 'NONE':
 734                 msg += ' Is your crypto smartcard plugged in?'
 735             logging.critical(msg)
 736             sys.exit(1)
 737         pubkey = p.output
 738     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 739     return hexlify(pubkey)
 740
 741
 742 def make_index(apps, sortedids, apks, repodir, archive, categories):
 743     """Make a repo index.
 744
 745     :param apps: fully populated apps list
 746     :param apks: full populated apks list
 747     :param repodir: the repo directory
 748     :param archive: True if this is the archive repo, False if it's the
 749                     main one.
 750     :param categories: list of categories
 751     """
 752
 753     doc = Document()
 754
 755     def addElement(name, value, doc, parent):
 756         el = doc.createElement(name)
 757         el.appendChild(doc.createTextNode(value))
 758         parent.appendChild(el)
 759
 760     def addElementNonEmpty(name, value, doc, parent):
 761         if not value:
 762             return
 763         addElement(name, value, doc, parent)
 764
 765     def addElementCDATA(name, value, doc, parent):
 766         el = doc.createElement(name)
 767         el.appendChild(doc.createCDATASection(value))
 768         parent.appendChild(el)
 769
 770     root = doc.createElement("fdroid")
 771     doc.appendChild(root)
 772
 773     repoel = doc.createElement("repo")
 774
 775     mirrorcheckfailed = False
 776     for mirror in config.get('mirrors', []):
 777         base = os.path.basename(urlparse.urlparse(mirror).path.rstrip('/'))
 778         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
 779             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 780             mirrorcheckfailed = True
 781     if mirrorcheckfailed:
 782         sys.exit(1)
 783
 784     if archive:
 785         repoel.setAttribute("name", config['archive_name'])
 786         if config['repo_maxage'] != 0:
 787             repoel.setAttribute("maxage", str(config['repo_maxage']))
 788         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 789         repoel.setAttribute("url", config['archive_url'])
 790         addElement('description', config['archive_description'], doc, repoel)
 791         urlbasepath = os.path.basename(urlparse.urlparse(config['archive_url']).path)
 792         for mirror in config.get('mirrors', []):
 793             addElement('mirror', urlparse.urljoin(mirror, urlbasepath), doc, repoel)
 794
 795     else:
 796         repoel.setAttribute("name", config['repo_name'])
 797         if config['repo_maxage'] != 0:
 798             repoel.setAttribute("maxage", str(config['repo_maxage']))
 799         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 800         repoel.setAttribute("url", config['repo_url'])
 801         addElement('description', config['repo_description'], doc, repoel)
 802         urlbasepath = os.path.basename(urlparse.urlparse(config['repo_url']).path)
 803         for mirror in config.get('mirrors', []):
 804             addElement('mirror', urlparse.urljoin(mirror, urlbasepath), doc, repoel)
 805
 806     repoel.setAttribute("version", "15")
 807     repoel.setAttribute("timestamp", str(int(time.time())))
 808
 809     nosigningkey = False
 810     if not options.nosign:
 811         if 'repo_keyalias' not in config:
 812             nosigningkey = True
 813             logging.critical("'repo_keyalias' not found in config.py!")
 814         if 'keystore' not in config:
 815             nosigningkey = True
 816             logging.critical("'keystore' not found in config.py!")
 817         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 818             nosigningkey = True
 819             logging.critical("'keystorepass' not found in config.py!")
 820         if 'keypass' not in config and 'keypassfile' not in config:
 821             nosigningkey = True
 822             logging.critical("'keypass' not found in config.py!")
 823         if not os.path.exists(config['keystore']):
 824             nosigningkey = True
 825             logging.critical("'" + config['keystore'] + "' does not exist!")
 826         if nosigningkey:
 827             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 828             logging.warning("\tfdroid update --create-key")
 829             sys.exit(1)
 830
 831     repoel.setAttribute("pubkey", extract_pubkey())
 832     root.appendChild(repoel)
 833
 834     for appid in sortedids:
 835         app = apps[appid]
 836
 837         if app.Disabled is not None:
 838             continue
 839
 840         # Get a list of the apks for this app...
 841         apklist = []
 842         for apk in apks:
 843             if apk['id'] == appid:
 844                 apklist.append(apk)
 845
 846         if len(apklist) == 0:
 847             continue
 848
 849         apel = doc.createElement("application")
 850         apel.setAttribute("id", app.id)
 851         root.appendChild(apel)
 852
 853         addElement('id', app.id, doc, apel)
 854         if app.added:
 855             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
 856         if app.lastupdated:
 857             addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
 858         addElement('name', app.Name, doc, apel)
 859         addElement('summary', app.Summary, doc, apel)
 860         if app.icon:
 861             addElement('icon', app.icon, doc, apel)
 862
 863         def linkres(appid):
 864             if appid in apps:
 865                 return ("fdroid.app:" + appid, apps[appid].Name)
 866             raise MetaDataException("Cannot resolve app id " + appid)
 867
 868         addElement('desc',
 869                    metadata.description_html(app.Description, linkres),
 870                    doc, apel)
 871         addElement('license', app.License, doc, apel)
 872         if app.Categories:
 873             addElement('categories', ','.join(app.Categories), doc, apel)
 874             # We put the first (primary) category in LAST, which will have
 875             # the desired effect of making clients that only understand one
 876             # category see that one.
 877             addElement('category', app.Categories[0], doc, apel)
 878         addElement('web', app.WebSite, doc, apel)
 879         addElement('source', app.SourceCode, doc, apel)
 880         addElement('tracker', app.IssueTracker, doc, apel)
 881         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 882         addElementNonEmpty('author', app.AuthorName, doc, apel)
 883         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 884         addElementNonEmpty('donate', app.Donate, doc, apel)
 885         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 886         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 887         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 888
 889         # These elements actually refer to the current version (i.e. which
 890         # one is recommended. They are historically mis-named, and need
 891         # changing, but stay like this for now to support existing clients.
 892         addElement('marketversion', app.CurrentVersion, doc, apel)
 893         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 894
 895         if app.AntiFeatures:
 896             af = app.AntiFeatures
 897             if af:
 898                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 899         if app.Provides:
 900             pv = app.Provides.split(',')
 901             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 902         if app.RequiresRoot:
 903             addElement('requirements', 'root', doc, apel)
 904
 905         # Sort the apk list into version order, just so the web site
 906         # doesn't have to do any work by default...
 907         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 908
 909         # Check for duplicates - they will make the client unhappy...
 910         for i in range(len(apklist) - 1):
 911             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 912                 logging.critical("duplicate versions: '%s' - '%s'" % (
 913                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 914                 sys.exit(1)
 915
 916         current_version_code = 0
 917         current_version_file = None
 918         for apk in apklist:
 919             # find the APK for the "Current Version"
 920             if current_version_code < apk['versioncode']:
 921                 current_version_code = apk['versioncode']
 922             if current_version_code < int(app.CurrentVersionCode):
 923                 current_version_file = apk['apkname']
 924
 925             apkel = doc.createElement("package")
 926             apel.appendChild(apkel)
 927             addElement('version', apk['version'], doc, apkel)
 928             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 929             addElement('apkname', apk['apkname'], doc, apkel)
 930             if 'srcname' in apk:
 931                 addElement('srcname', apk['srcname'], doc, apkel)
 932             for hash_type in ['sha256']:
 933                 if hash_type not in apk:
 934                     continue
 935                 hashel = doc.createElement("hash")
 936                 hashel.setAttribute("type", hash_type)
 937                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 938                 apkel.appendChild(hashel)
 939             addElement('sig', apk['sig'], doc, apkel)
 940             addElement('size', str(apk['size']), doc, apkel)
 941             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 942             if 'maxsdkversion' in apk:
 943                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 944             if 'added' in apk:
 945                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 946             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
 947             if 'nativecode' in apk:
 948                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 949             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
 950
 951         if current_version_file is not None \
 952                 and config['make_current_version_link'] \
 953                 and repodir == 'repo':  # only create these
 954             namefield = config['current_version_name_source']
 955             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
 956             apklinkname = sanitized_name + '.apk'
 957             current_version_path = os.path.join(repodir, current_version_file)
 958             if os.path.islink(apklinkname):
 959                 os.remove(apklinkname)
 960             os.symlink(current_version_path, apklinkname)
 961             # also symlink gpg signature, if it exists
 962             for extension in ('.asc', '.sig'):
 963                 sigfile_path = current_version_path + extension
 964                 if os.path.exists(sigfile_path):
 965                     siglinkname = apklinkname + extension
 966                     if os.path.islink(siglinkname):
 967                         os.remove(siglinkname)
 968                     os.symlink(sigfile_path, siglinkname)
 969
 970     if options.pretty:
 971         output = doc.toprettyxml()
 972     else:
 973         output = doc.toxml()
 974
 975     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 976         f.write(output)
 977
 978     if 'repo_keyalias' in config:
 979
 980         if options.nosign:
 981             logging.info("Creating unsigned index in preparation for signing")
 982         else:
 983             logging.info("Creating signed index with this key (SHA256):")
 984             logging.info("%s" % repo_pubkey_fingerprint)
 985
 986         # Create a jar of the index...
 987         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
 988         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 989         if p.returncode != 0:
 990             logging.critical("Failed to create {0}".format(jar_output))
 991             sys.exit(1)
 992
 993         # Sign the index...
 994         signed = os.path.join(repodir, 'index.jar')
 995         if options.nosign:
 996             # Remove old signed index if not signing
 997             if os.path.exists(signed):
 998                 os.remove(signed)
 999         else:
1000             args = [config['jarsigner'], '-keystore', config['keystore'],
1001                     '-storepass:file', config['keystorepassfile'],
1002                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1003                     signed, config['repo_keyalias']]
1004             if config['keystore'] == 'NONE':
1005                 args += config['smartcardoptions']
1006             else:  # smardcards never use -keypass
1007                 args += ['-keypass:file', config['keypassfile']]
1008             p = FDroidPopen(args)
1009             if p.returncode != 0:
1010                 logging.critical("Failed to sign index")
1011                 sys.exit(1)
1012
1013     # Copy the repo icon into the repo directory...
1014     icon_dir = os.path.join(repodir, 'icons')
1015     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1016     shutil.copyfile(config['repo_icon'], iconfilename)
1017
1018     # Write a category list in the repo to allow quick access...
1019     catdata = ''
1020     for cat in categories:
1021         catdata += cat + '\n'
1022     with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
1023         f.write(catdata)
1024
1025
1026 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1027
1028     for appid, app in apps.iteritems():
1029
1030         if app.ArchivePolicy:
1031             keepversions = int(app.ArchivePolicy[:-9])
1032         else:
1033             keepversions = defaultkeepversions
1034
1035         def filter_apk_list_sorted(apk_list):
1036             res = []
1037             for apk in apk_list:
1038                 if apk['id'] == appid:
1039                     res.append(apk)
1040
1041             # Sort the apk list by version code. First is highest/newest.
1042             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1043
1044         def move_file(from_dir, to_dir, filename, ignore_missing):
1045             from_path = os.path.join(from_dir, filename)
1046             if ignore_missing and not os.path.exists(from_path):
1047                 return
1048             to_path = os.path.join(to_dir, filename)
1049             shutil.move(from_path, to_path)
1050
1051         if len(apks) > keepversions:
1052             apklist = filter_apk_list_sorted(apks)
1053             # Move back the ones we don't want.
1054             for apk in apklist[keepversions:]:
1055                 logging.info("Moving " + apk['apkname'] + " to archive")
1056                 move_file(repodir, archivedir, apk['apkname'], False)
1057                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1058                 for density in all_screen_densities:
1059                     repo_icon_dir = get_icon_dir(repodir, density)
1060                     archive_icon_dir = get_icon_dir(archivedir, density)
1061                     if density not in apk['icons']:
1062                         continue
1063                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1064                 if 'srcname' in apk:
1065                     move_file(repodir, archivedir, apk['srcname'], False)
1066                 archapks.append(apk)
1067                 apks.remove(apk)
1068         elif len(apks) < keepversions and len(archapks) > 0:
1069             required = keepversions - len(apks)
1070             archapklist = filter_apk_list_sorted(archapks)
1071             # Move forward the ones we want again.
1072             for apk in archapklist[:required]:
1073                 logging.info("Moving " + apk['apkname'] + " from archive")
1074                 move_file(archivedir, repodir, apk['apkname'], False)
1075                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1076                 for density in all_screen_densities:
1077                     repo_icon_dir = get_icon_dir(repodir, density)
1078                     archive_icon_dir = get_icon_dir(archivedir, density)
1079                     if density not in apk['icons']:
1080                         continue
1081                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1082                 if 'srcname' in apk:
1083                     move_file(archivedir, repodir, apk['srcname'], False)
1084                 archapks.remove(apk)
1085                 apks.append(apk)
1086
1087
1088 def add_apks_to_per_app_repos(repodir, apks):
1089     apks_per_app = dict()
1090     for apk in apks:
1091         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1092         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1093         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1094         apks_per_app[apk['id']] = apk
1095
1096         if not os.path.exists(apk['per_app_icons']):
1097             logging.info('Adding new repo for only ' + apk['id'])
1098             os.makedirs(apk['per_app_icons'])
1099
1100         apkpath = os.path.join(repodir, apk['apkname'])
1101         shutil.copy(apkpath, apk['per_app_repo'])
1102         apksigpath = apkpath + '.sig'
1103         if os.path.exists(apksigpath):
1104             shutil.copy(apksigpath, apk['per_app_repo'])
1105         apkascpath = apkpath + '.asc'
1106         if os.path.exists(apkascpath):
1107             shutil.copy(apkascpath, apk['per_app_repo'])
1108
1109
1110 config = None
1111 options = None
1112
1113
1114 def main():
1115
1116     global config, options
1117
1118     # Parse command line...
1119     parser = ArgumentParser()
1120     common.setup_global_opts(parser)
1121     parser.add_argument("--create-key", action="store_true", default=False,
1122                         help="Create a repo signing key in a keystore")
1123     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1124                         help="Create skeleton metadata files that are missing")
1125     parser.add_argument("--delete-unknown", action="store_true", default=False,
1126                         help="Delete APKs without metadata from the repo")
1127     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1128                         help="Report on build data status")
1129     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1130                         help="Interactively ask about things that need updating.")
1131     parser.add_argument("-I", "--icons", action="store_true", default=False,
1132                         help="Resize all the icons exceeding the max pixel size and exit")
1133     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1134                         help="Specify editor to use in interactive mode. Default " +
1135                         "is /etc/alternatives/editor")
1136     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1137                         help="Update the wiki")
1138     parser.add_argument("--pretty", action="store_true", default=False,
1139                         help="Produce human-readable index.xml")
1140     parser.add_argument("--clean", action="store_true", default=False,
1141                         help="Clean update - don't uses caches, reprocess all apks")
1142     parser.add_argument("--nosign", action="store_true", default=False,
1143                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1144     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1145                         help="Use date from apk instead of current time for newly added apks")
1146     options = parser.parse_args()
1147
1148     config = common.read_config(options)
1149
1150     if not ('jarsigner' in config and 'keytool' in config):
1151         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1152         sys.exit(1)
1153
1154     repodirs = ['repo']
1155     if config['archive_older'] != 0:
1156         repodirs.append('archive')
1157         if not os.path.exists('archive'):
1158             os.mkdir('archive')
1159
1160     if options.icons:
1161         resize_all_icons(repodirs)
1162         sys.exit(0)
1163
1164     # check that icons exist now, rather than fail at the end of `fdroid update`
1165     for k in ['repo_icon', 'archive_icon']:
1166         if k in config:
1167             if not os.path.exists(config[k]):
1168                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1169                 sys.exit(1)
1170
1171     # if the user asks to create a keystore, do it now, reusing whatever it can
1172     if options.create_key:
1173         if os.path.exists(config['keystore']):
1174             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1175             logging.critical("\t'" + config['keystore'] + "'")
1176             sys.exit(1)
1177
1178         if 'repo_keyalias' not in config:
1179             config['repo_keyalias'] = socket.getfqdn()
1180             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1181         if 'keydname' not in config:
1182             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1183             common.write_to_config(config, 'keydname', config['keydname'])
1184         if 'keystore' not in config:
1185             config['keystore'] = common.default_config.keystore
1186             common.write_to_config(config, 'keystore', config['keystore'])
1187
1188         password = common.genpassword()
1189         if 'keystorepass' not in config:
1190             config['keystorepass'] = password
1191             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1192         if 'keypass' not in config:
1193             config['keypass'] = password
1194             common.write_to_config(config, 'keypass', config['keypass'])
1195         common.genkeystore(config)
1196
1197     # Get all apps...
1198     apps = metadata.read_metadata()
1199
1200     # Generate a list of categories...
1201     categories = set()
1202     for app in apps.itervalues():
1203         categories.update(app.Categories)
1204
1205     # Read known apks data (will be updated and written back when we've finished)
1206     knownapks = common.KnownApks()
1207
1208     # Gather information about all the apk files in the repo directory, using
1209     # cached data if possible.
1210     apkcachefile = os.path.join('tmp', 'apkcache')
1211     if not options.clean and os.path.exists(apkcachefile):
1212         with open(apkcachefile, 'rb') as cf:
1213             apkcache = pickle.load(cf)
1214     else:
1215         apkcache = {}
1216
1217     delete_disabled_builds(apps, apkcache, repodirs)
1218
1219     # Scan all apks in the main repo
1220     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1221
1222     # Generate warnings for apk's with no metadata (or create skeleton
1223     # metadata files, if requested on the command line)
1224     newmetadata = False
1225     for apk in apks:
1226         if apk['id'] not in apps:
1227             if options.create_metadata:
1228                 if 'name' not in apk:
1229                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1230                     continue
1231                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1232                 f.write("License:Unknown\n")
1233                 f.write("Web Site:\n")
1234                 f.write("Source Code:\n")
1235                 f.write("Issue Tracker:\n")
1236                 f.write("Changelog:\n")
1237                 f.write("Summary:" + apk['name'] + "\n")
1238                 f.write("Description:\n")
1239                 f.write(apk['name'] + "\n")
1240                 f.write(".\n")
1241                 f.close()
1242                 logging.info("Generated skeleton metadata for " + apk['id'])
1243                 newmetadata = True
1244             else:
1245                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1246                 if options.delete_unknown:
1247                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1248                     rmf = os.path.join(repodirs[0], apk['apkname'])
1249                     if not os.path.exists(rmf):
1250                         logging.error("Could not find {0} to remove it".format(rmf))
1251                     else:
1252                         os.remove(rmf)
1253                 else:
1254                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1255
1256     # update the metadata with the newly created ones included
1257     if newmetadata:
1258         apps = metadata.read_metadata()
1259
1260     # Scan the archive repo for apks as well
1261     if len(repodirs) > 1:
1262         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1263         if cc:
1264             cachechanged = True
1265     else:
1266         archapks = []
1267
1268     # Some information from the apks needs to be applied up to the application
1269     # level. When doing this, we use the info from the most recent version's apk.
1270     # We deal with figuring out when the app was added and last updated at the
1271     # same time.
1272     for appid, app in apps.iteritems():
1273         bestver = 0
1274         for apk in apks + archapks:
1275             if apk['id'] == appid:
1276                 if apk['versioncode'] > bestver:
1277                     bestver = apk['versioncode']
1278                     bestapk = apk
1279
1280                 if 'added' in apk:
1281                     if not app.added or apk['added'] < app.added:
1282                         app.added = apk['added']
1283                     if not app.lastupdated or apk['added'] > app.lastupdated:
1284                         app.lastupdated = apk['added']
1285
1286         if not app.added:
1287             logging.debug("Don't know when " + appid + " was added")
1288         if not app.lastupdated:
1289             logging.debug("Don't know when " + appid + " was last updated")
1290
1291         if bestver == 0:
1292             if app.Name is None:
1293                 app.Name = app.AutoName or appid
1294             app.icon = None
1295             logging.debug("Application " + appid + " has no packages")
1296         else:
1297             if app.Name is None:
1298                 app.Name = bestapk['name']
1299             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1300             if app.CurrentVersionCode is None:
1301                 app.CurrentVersionCode = str(bestver)
1302
1303     # Sort the app list by name, then the web site doesn't have to by default.
1304     # (we had to wait until we'd scanned the apks to do this, because mostly the
1305     # name comes from there!)
1306     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid].Name.upper())
1307
1308     # APKs are placed into multiple repos based on the app package, providing
1309     # per-app subscription feeds for nightly builds and things like it
1310     if config['per_app_repos']:
1311         add_apks_to_per_app_repos(repodirs[0], apks)
1312         for appid, app in apps.iteritems():
1313             repodir = os.path.join(appid, 'fdroid', 'repo')
1314             appdict = dict()
1315             appdict[appid] = app
1316             if os.path.isdir(repodir):
1317                 make_index(appdict, [appid], apks, repodir, False, categories)
1318             else:
1319                 logging.info('Skipping index generation for ' + appid)
1320         return
1321
1322     if len(repodirs) > 1:
1323         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1324
1325     # Make the index for the main repo...
1326     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1327
1328     # If there's an archive repo,  make the index for it. We already scanned it
1329     # earlier on.
1330     if len(repodirs) > 1:
1331         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1332
1333     if config['update_stats']:
1334
1335         # Update known apks info...
1336         knownapks.writeifchanged()
1337
1338         # Generate latest apps data for widget
1339         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1340             data = ''
1341             for line in file(os.path.join('stats', 'latestapps.txt')):
1342                 appid = line.rstrip()
1343                 data += appid + "\t"
1344                 app = apps[appid]
1345                 data += app.Name + "\t"
1346                 if app.icon is not None:
1347                     data += app.icon + "\t"
1348                 data += app.License + "\n"
1349             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
1350                 f.write(data)
1351
1352     if cachechanged:
1353         with open(apkcachefile, 'wb') as cf:
1354             pickle.dump(apkcache, cf)
1355
1356     # Update the wiki...
1357     if options.wiki:
1358         update_wiki(apps, sortedids, apks + archapks)
1359
1360     logging.info("Finished.")
1361
1362 if __name__ == "__main__":
1363     main()