fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   5 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU Affero General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Affero General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Affero General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 import sys
  21 import os
  22 import shutil
  23 import glob
  24 import re
  25 import socket
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 import urllib.parse
  30 from datetime import datetime, timedelta
  31 from xml.dom.minidom import Document
  32 from argparse import ArgumentParser
  33 import time
  34 from pyasn1.error import PyAsn1Error
  35 from pyasn1.codec.der import decoder, encoder
  36 from pyasn1_modules import rfc2315
  37 from binascii import hexlify, unhexlify
  38
  39 from PIL import Image
  40 import logging
  41
  42 from . import common
  43 from . import metadata
  44 from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
  45 from .metadata import MetaDataException
  46
  47 screen_densities = ['640', '480', '320', '240', '160', '120']
  48
  49 all_screen_densities = ['0'] + screen_densities
  50
  51
  52 def dpi_to_px(density):
  53     return (int(density) * 48) / 160
  54
  55
  56 def px_to_dpi(px):
  57     return (int(px) * 160) / 48
  58
  59
  60 def get_icon_dir(repodir, density):
  61     if density == '0':
  62         return os.path.join(repodir, "icons")
  63     return os.path.join(repodir, "icons-%s" % density)
  64
  65
  66 def get_icon_dirs(repodir):
  67     for density in screen_densities:
  68         yield get_icon_dir(repodir, density)
  69
  70
  71 def get_all_icon_dirs(repodir):
  72     for density in all_screen_densities:
  73         yield get_icon_dir(repodir, density)
  74
  75
  76 def update_wiki(apps, sortedids, apks):
  77     """Update the wiki
  78
  79     :param apps: fully populated list of all applications
  80     :param apks: all apks, except...
  81     """
  82     logging.info("Updating wiki")
  83     wikicat = 'Apps'
  84     wikiredircat = 'App Redirects'
  85     import mwclient
  86     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  87                          path=config['wiki_path'])
  88     site.login(config['wiki_user'], config['wiki_password'])
  89     generated_pages = {}
  90     generated_redirects = {}
  91
  92     for appid in sortedids:
  93         app = apps[appid]
  94
  95         wikidata = ''
  96         if app.Disabled:
  97             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
  98         if app.AntiFeatures:
  99             for af in app.AntiFeatures:
 100                 wikidata += '{{AntiFeature|' + af + '}}\n'
 101         if app.RequiresRoot:
 102             requiresroot = 'Yes'
 103         else:
 104             requiresroot = 'No'
 105         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 106             appid,
 107             app.Name,
 108             time.strftime('%Y-%m-%d', app.added) if app.added else '',
 109             time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
 110             app.SourceCode,
 111             app.IssueTracker,
 112             app.WebSite,
 113             app.Changelog,
 114             app.Donate,
 115             app.FlattrID,
 116             app.Bitcoin,
 117             app.Litecoin,
 118             app.License,
 119             requiresroot,
 120             app.AuthorName,
 121             app.AuthorEmail)
 122
 123         if app.Provides:
 124             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 125
 126         wikidata += app.Summary
 127         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 128
 129         wikidata += "=Description=\n"
 130         wikidata += metadata.description_wiki(app.Description) + "\n"
 131
 132         wikidata += "=Maintainer Notes=\n"
 133         if app.MaintainerNotes:
 134             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 135         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 136
 137         # Get a list of all packages for this application...
 138         apklist = []
 139         gotcurrentver = False
 140         cantupdate = False
 141         buildfails = False
 142         for apk in apks:
 143             if apk['id'] == appid:
 144                 if str(apk['versioncode']) == app.CurrentVersionCode:
 145                     gotcurrentver = True
 146                 apklist.append(apk)
 147         # Include ones we can't build, as a special case...
 148         for build in app.builds:
 149             if build.disable:
 150                 if build.vercode == app.CurrentVersionCode:
 151                     cantupdate = True
 152                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 153                 apklist.append({'versioncode': int(build.vercode),
 154                                 'version': build.version,
 155                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 156                                 })
 157             else:
 158                 builtit = False
 159                 for apk in apklist:
 160                     if apk['versioncode'] == int(build.vercode):
 161                         builtit = True
 162                         break
 163                 if not builtit:
 164                     buildfails = True
 165                     apklist.append({'versioncode': int(build.vercode),
 166                                     'version': build.version,
 167                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
 168                                     })
 169         if app.CurrentVersionCode == '0':
 170             cantupdate = True
 171         # Sort with most recent first...
 172         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 173
 174         wikidata += "=Versions=\n"
 175         if len(apklist) == 0:
 176             wikidata += "We currently have no versions of this app available."
 177         elif not gotcurrentver:
 178             wikidata += "We don't have the current version of this app."
 179         else:
 180             wikidata += "We have the current version of this app."
 181         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 182         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 183         if len(app.NoSourceSince) > 0:
 184             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 185         if len(app.CurrentVersion) > 0:
 186             wikidata += "The current (recommended) version is " + app.CurrentVersion
 187             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 188         validapks = 0
 189         for apk in apklist:
 190             wikidata += "==" + apk['version'] + "==\n"
 191
 192             if 'buildproblem' in apk:
 193                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 194             else:
 195                 validapks += 1
 196                 wikidata += "This version is built and signed by "
 197                 if 'srcname' in apk:
 198                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 199                 else:
 200                     wikidata += "the original developer.\n\n"
 201             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 202
 203         wikidata += '\n[[Category:' + wikicat + ']]\n'
 204         if len(app.NoSourceSince) > 0:
 205             wikidata += '\n[[Category:Apps missing source code]]\n'
 206         if validapks == 0 and not app.Disabled:
 207             wikidata += '\n[[Category:Apps with no packages]]\n'
 208         if cantupdate and not app.Disabled:
 209             wikidata += "\n[[Category:Apps we cannot update]]\n"
 210         if buildfails and not app.Disabled:
 211             wikidata += "\n[[Category:Apps with failing builds]]\n"
 212         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 213             wikidata += '\n[[Category:Apps to Update]]\n'
 214         if app.Disabled:
 215             wikidata += '\n[[Category:Apps that are disabled]]\n'
 216         if app.UpdateCheckMode == 'None' and not app.Disabled:
 217             wikidata += '\n[[Category:Apps with no update check]]\n'
 218         for appcat in app.Categories:
 219             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 220
 221         # We can't have underscores in the page name, even if they're in
 222         # the package ID, because MediaWiki messes with them...
 223         pagename = appid.replace('_', ' ')
 224
 225         # Drop a trailing newline, because mediawiki is going to drop it anyway
 226         # and it we don't we'll think the page has changed when it hasn't...
 227         if wikidata.endswith('\n'):
 228             wikidata = wikidata[:-1]
 229
 230         generated_pages[pagename] = wikidata
 231
 232         # Make a redirect from the name to the ID too, unless there's
 233         # already an existing page with the name and it isn't a redirect.
 234         noclobber = False
 235         apppagename = app.Name.replace('_', ' ')
 236         apppagename = apppagename.replace('{', '')
 237         apppagename = apppagename.replace('}', ' ')
 238         apppagename = apppagename.replace(':', ' ')
 239         # Drop double spaces caused mostly by replacing ':' above
 240         apppagename = apppagename.replace('  ', ' ')
 241         for expagename in site.allpages(prefix=apppagename,
 242                                         filterredir='nonredirects',
 243                                         generator=False):
 244             if expagename == apppagename:
 245                 noclobber = True
 246         # Another reason not to make the redirect page is if the app name
 247         # is the same as it's ID, because that will overwrite the real page
 248         # with an redirect to itself! (Although it seems like an odd
 249         # scenario this happens a lot, e.g. where there is metadata but no
 250         # builds or binaries to extract a name from.
 251         if apppagename == pagename:
 252             noclobber = True
 253         if not noclobber:
 254             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 255
 256     for tcat, genp in [(wikicat, generated_pages),
 257                        (wikiredircat, generated_redirects)]:
 258         catpages = site.Pages['Category:' + tcat]
 259         existingpages = []
 260         for page in catpages:
 261             existingpages.append(page.name)
 262             if page.name in genp:
 263                 pagetxt = page.edit()
 264                 if pagetxt != genp[page.name]:
 265                     logging.debug("Updating modified page " + page.name)
 266                     page.save(genp[page.name], summary='Auto-updated')
 267                 else:
 268                     logging.debug("Page " + page.name + " is unchanged")
 269             else:
 270                 logging.warn("Deleting page " + page.name)
 271                 page.delete('No longer published')
 272         for pagename, text in genp.items():
 273             logging.debug("Checking " + pagename)
 274             if pagename not in existingpages:
 275                 logging.debug("Creating page " + pagename)
 276                 try:
 277                     newpage = site.Pages[pagename]
 278                     newpage.save(text, summary='Auto-created')
 279                 except:
 280                     logging.error("...FAILED to create page '{0}'".format(pagename))
 281
 282     # Purge server cache to ensure counts are up to date
 283     site.pages['Repository Maintenance'].purge()
 284
 285
 286 def delete_disabled_builds(apps, apkcache, repodirs):
 287     """Delete disabled build outputs.
 288
 289     :param apps: list of all applications, as per metadata.read_metadata
 290     :param apkcache: current apk cache information
 291     :param repodirs: the repo directories to process
 292     """
 293     for appid, app in apps.items():
 294         for build in app.builds:
 295             if not build.disable:
 296                 continue
 297             apkfilename = appid + '_' + str(build.vercode) + '.apk'
 298             iconfilename = "%s.%s.png" % (
 299                 appid,
 300                 build.vercode)
 301             for repodir in repodirs:
 302                 files = [
 303                     os.path.join(repodir, apkfilename),
 304                     os.path.join(repodir, apkfilename + '.asc'),
 305                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 306                 ]
 307                 for density in all_screen_densities:
 308                     repo_dir = get_icon_dir(repodir, density)
 309                     files.append(os.path.join(repo_dir, iconfilename))
 310
 311                 for f in files:
 312                     if os.path.exists(f):
 313                         logging.info("Deleting disabled build output " + f)
 314                         os.remove(f)
 315             if apkfilename in apkcache:
 316                 del apkcache[apkfilename]
 317
 318
 319 def resize_icon(iconpath, density):
 320
 321     if not os.path.isfile(iconpath):
 322         return
 323
 324     try:
 325         im = Image.open(iconpath)
 326         size = dpi_to_px(density)
 327
 328         if any(length > size for length in im.size):
 329             oldsize = im.size
 330             im.thumbnail((size, size), Image.ANTIALIAS)
 331             logging.debug("%s was too large at %s - new size is %s" % (
 332                 iconpath, oldsize, im.size))
 333             im.save(iconpath, "PNG")
 334
 335     except Exception as e:
 336         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 337
 338
 339 def resize_all_icons(repodirs):
 340     """Resize all icons that exceed the max size
 341
 342     :param repodirs: the repo directories to process
 343     """
 344     for repodir in repodirs:
 345         for density in screen_densities:
 346             icon_dir = get_icon_dir(repodir, density)
 347             icon_glob = os.path.join(icon_dir, '*.png')
 348             for iconpath in glob.glob(icon_glob):
 349                 resize_icon(iconpath, density)
 350
 351
 352 # A signature block file with a .DSA, .RSA, or .EC extension
 353 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 354
 355
 356 def getsig(apkpath):
 357     """ Get the signing certificate of an apk. To get the same md5 has that
 358     Android gets, we encode the .RSA certificate in a specific format and pass
 359     it hex-encoded to the md5 digest algorithm.
 360
 361     :param apkpath: path to the apk
 362     :returns: A string containing the md5 of the signature of the apk or None
 363               if an error occurred.
 364     """
 365
 366     cert = None
 367
 368     # verify the jar signature is correct
 369     args = [config['jarsigner'], '-verify', apkpath]
 370     p = FDroidPopen(args)
 371     if p.returncode != 0:
 372         logging.critical(apkpath + " has a bad signature!")
 373         return None
 374
 375     with zipfile.ZipFile(apkpath, 'r') as apk:
 376
 377         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 378
 379         if len(certs) < 1:
 380             logging.error("Found no signing certificates on %s" % apkpath)
 381             return None
 382         if len(certs) > 1:
 383             logging.error("Found multiple signing certificates on %s" % apkpath)
 384             return None
 385
 386         cert = apk.read(certs[0])
 387
 388     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 389     if content.getComponentByName('contentType') != rfc2315.signedData:
 390         logging.error("Unexpected format.")
 391         return None
 392
 393     content = decoder.decode(content.getComponentByName('content'),
 394                              asn1Spec=rfc2315.SignedData())[0]
 395     try:
 396         certificates = content.getComponentByName('certificates')
 397     except PyAsn1Error:
 398         logging.error("Certificates not found.")
 399         return None
 400
 401     cert_encoded = encoder.encode(certificates)[4:]
 402
 403     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 404
 405
 406 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
 407     """Scan the apks in the given repo directory.
 408
 409     This also extracts the icons.
 410
 411     :param apps: list of all applications, as per metadata.read_metadata
 412     :param apkcache: current apk cache information
 413     :param repodir: repo directory to scan
 414     :param knownapks: known apks info
 415     :param use_date_from_apk: use date from APK (instead of current date)
 416                               for newly added APKs
 417     :returns: (apks, cachechanged) where apks is a list of apk information,
 418               and cachechanged is True if the apkcache got changed.
 419     """
 420
 421     cachechanged = False
 422
 423     for icon_dir in get_all_icon_dirs(repodir):
 424         if os.path.exists(icon_dir):
 425             if options.clean:
 426                 shutil.rmtree(icon_dir)
 427                 os.makedirs(icon_dir)
 428         else:
 429             os.makedirs(icon_dir)
 430
 431     apks = []
 432     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 433     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 434     vername_pat = re.compile(".*versionName='([^']*)'.*")
 435     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 436     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 437     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 438     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 439     string_pat = re.compile(".*'([^']*)'.*")
 440     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 441
 442         apkfilename = apkfile[len(repodir) + 1:]
 443         if ' ' in apkfilename:
 444             logging.critical("Spaces in filenames are not allowed.")
 445             sys.exit(1)
 446
 447         # Calculate the sha256...
 448         sha = hashlib.sha256()
 449         with open(apkfile, 'rb') as f:
 450             while True:
 451                 t = f.read(16384)
 452                 if len(t) == 0:
 453                     break
 454                 sha.update(t)
 455             shasum = sha.hexdigest()
 456
 457         usecache = False
 458         if apkfilename in apkcache:
 459             apk = apkcache[apkfilename]
 460             if apk['sha256'] == shasum:
 461                 logging.debug("Reading " + apkfilename + " from cache")
 462                 usecache = True
 463             else:
 464                 logging.debug("Ignoring stale cache data for " + apkfilename)
 465
 466         if not usecache:
 467             logging.debug("Processing " + apkfilename)
 468             apk = {}
 469             apk['apkname'] = apkfilename
 470             apk['sha256'] = shasum
 471             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 472             if os.path.exists(os.path.join(repodir, srcfilename)):
 473                 apk['srcname'] = srcfilename
 474             apk['size'] = os.path.getsize(apkfile)
 475             apk['permissions'] = set()
 476             apk['features'] = set()
 477             apk['icons_src'] = {}
 478             apk['icons'] = {}
 479             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 480             if p.returncode != 0:
 481                 if options.delete_unknown:
 482                     if os.path.exists(apkfile):
 483                         logging.error("Failed to get apk information, deleting " + apkfile)
 484                         os.remove(apkfile)
 485                     else:
 486                         logging.error("Could not find {0} to remove it".format(apkfile))
 487                 else:
 488                     logging.error("Failed to get apk information, skipping " + apkfile)
 489                 continue
 490             for line in p.output.splitlines():
 491                 if line.startswith("package:"):
 492                     try:
 493                         apk['id'] = re.match(name_pat, line).group(1)
 494                         apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
 495                         apk['version'] = re.match(vername_pat, line).group(1)
 496                     except Exception as e:
 497                         logging.error("Package matching failed: " + str(e))
 498                         logging.info("Line was: " + line)
 499                         sys.exit(1)
 500                 elif line.startswith("application:"):
 501                     apk['name'] = re.match(label_pat, line).group(1)
 502                     # Keep path to non-dpi icon in case we need it
 503                     match = re.match(icon_pat_nodpi, line)
 504                     if match:
 505                         apk['icons_src']['-1'] = match.group(1)
 506                 elif line.startswith("launchable-activity:"):
 507                     # Only use launchable-activity as fallback to application
 508                     if not apk['name']:
 509                         apk['name'] = re.match(label_pat, line).group(1)
 510                     if '-1' not in apk['icons_src']:
 511                         match = re.match(icon_pat_nodpi, line)
 512                         if match:
 513                             apk['icons_src']['-1'] = match.group(1)
 514                 elif line.startswith("application-icon-"):
 515                     match = re.match(icon_pat, line)
 516                     if match:
 517                         density = match.group(1)
 518                         path = match.group(2)
 519                         apk['icons_src'][density] = path
 520                 elif line.startswith("sdkVersion:"):
 521                     m = re.match(sdkversion_pat, line)
 522                     if m is None:
 523                         logging.error(line.replace('sdkVersion:', '')
 524                                       + ' is not a valid minSdkVersion!')
 525                     else:
 526                         apk['sdkversion'] = m.group(1)
 527                 elif line.startswith("maxSdkVersion:"):
 528                     apk['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 529                 elif line.startswith("native-code:"):
 530                     apk['nativecode'] = []
 531                     for arch in line[13:].split(' '):
 532                         apk['nativecode'].append(arch[1:-1])
 533                 elif line.startswith("uses-permission:"):
 534                     perm = re.match(string_pat, line).group(1)
 535                     if perm.startswith("android.permission."):
 536                         perm = perm[19:]
 537                     apk['permissions'].add(perm)
 538                 elif line.startswith("uses-feature:"):
 539                     perm = re.match(string_pat, line).group(1)
 540                     # Filter out this, it's only added with the latest SDK tools and
 541                     # causes problems for lots of apps.
 542                     if perm != "android.hardware.screen.portrait" \
 543                             and perm != "android.hardware.screen.landscape":
 544                         if perm.startswith("android.feature."):
 545                             perm = perm[16:]
 546                         apk['features'].add(perm)
 547
 548             if 'sdkversion' not in apk:
 549                 logging.warn("No SDK version information found in {0}".format(apkfile))
 550                 apk['sdkversion'] = 0
 551
 552             # Check for debuggable apks...
 553             if common.isApkDebuggable(apkfile, config):
 554                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 555
 556             # Get the signature (or md5 of, to be precise)...
 557             logging.debug('Getting signature of {0}'.format(apkfile))
 558             apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 559             if not apk['sig']:
 560                 logging.critical("Failed to get apk signature")
 561                 sys.exit(1)
 562
 563             apkzip = zipfile.ZipFile(apkfile, 'r')
 564
 565             # if an APK has files newer than the system time, suggest updating
 566             # the system clock.  This is useful for offline systems, used for
 567             # signing, which do not have another source of clock sync info. It
 568             # has to be more than 24 hours newer because ZIP/APK files do not
 569             # store timezone info
 570             manifest = apkzip.getinfo('AndroidManifest.xml')
 571             if manifest.date_time[1] == 0:  # month can't be zero
 572                 logging.debug('AndroidManifest.xml has no date')
 573             else:
 574                 dt_obj = datetime(*manifest.date_time)
 575                 checkdt = dt_obj - timedelta(1)
 576                 if datetime.today() < checkdt:
 577                     logging.warn('System clock is older than manifest in: '
 578                                  + apkfilename
 579                                  + '\nSet clock to that time using:\n'
 580                                  + 'sudo date -s "' + str(dt_obj) + '"')
 581
 582             iconfilename = "%s.%s.png" % (
 583                 apk['id'],
 584                 apk['versioncode'])
 585
 586             # Extract the icon file...
 587             empty_densities = []
 588             for density in screen_densities:
 589                 if density not in apk['icons_src']:
 590                     empty_densities.append(density)
 591                     continue
 592                 iconsrc = apk['icons_src'][density]
 593                 icon_dir = get_icon_dir(repodir, density)
 594                 icondest = os.path.join(icon_dir, iconfilename)
 595
 596                 try:
 597                     with open(icondest, 'wb') as f:
 598                         f.write(apkzip.read(iconsrc))
 599                     apk['icons'][density] = iconfilename
 600
 601                 except:
 602                     logging.warn("Error retrieving icon file")
 603                     del apk['icons'][density]
 604                     del apk['icons_src'][density]
 605                     empty_densities.append(density)
 606
 607             if '-1' in apk['icons_src']:
 608                 iconsrc = apk['icons_src']['-1']
 609                 iconpath = os.path.join(
 610                     get_icon_dir(repodir, '0'), iconfilename)
 611                 with open(iconpath, 'wb') as f:
 612                     f.write(apkzip.read(iconsrc))
 613                 try:
 614                     im = Image.open(iconpath)
 615                     dpi = px_to_dpi(im.size[0])
 616                     for density in screen_densities:
 617                         if density in apk['icons']:
 618                             break
 619                         if density == screen_densities[-1] or dpi >= int(density):
 620                             apk['icons'][density] = iconfilename
 621                             shutil.move(iconpath,
 622                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 623                             empty_densities.remove(density)
 624                             break
 625                 except Exception as e:
 626                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 627
 628             if apk['icons']:
 629                 apk['icon'] = iconfilename
 630
 631             apkzip.close()
 632
 633             # First try resizing down to not lose quality
 634             last_density = None
 635             for density in screen_densities:
 636                 if density not in empty_densities:
 637                     last_density = density
 638                     continue
 639                 if last_density is None:
 640                     continue
 641                 logging.debug("Density %s not available, resizing down from %s"
 642                               % (density, last_density))
 643
 644                 last_iconpath = os.path.join(
 645                     get_icon_dir(repodir, last_density), iconfilename)
 646                 iconpath = os.path.join(
 647                     get_icon_dir(repodir, density), iconfilename)
 648                 try:
 649                     im = Image.open(last_iconpath)
 650                 except:
 651                     logging.warn("Invalid image file at %s" % last_iconpath)
 652                     continue
 653
 654                 size = dpi_to_px(density)
 655
 656                 im.thumbnail((size, size), Image.ANTIALIAS)
 657                 im.save(iconpath, "PNG")
 658                 empty_densities.remove(density)
 659
 660             # Then just copy from the highest resolution available
 661             last_density = None
 662             for density in reversed(screen_densities):
 663                 if density not in empty_densities:
 664                     last_density = density
 665                     continue
 666                 if last_density is None:
 667                     continue
 668                 logging.debug("Density %s not available, copying from lower density %s"
 669                               % (density, last_density))
 670
 671                 shutil.copyfile(
 672                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 673                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 674
 675                 empty_densities.remove(density)
 676
 677             for density in screen_densities:
 678                 icon_dir = get_icon_dir(repodir, density)
 679                 icondest = os.path.join(icon_dir, iconfilename)
 680                 resize_icon(icondest, density)
 681
 682             # Copy from icons-mdpi to icons since mdpi is the baseline density
 683             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 684             if os.path.isfile(baseline):
 685                 apk['icons']['0'] = iconfilename
 686                 shutil.copyfile(baseline,
 687                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 688
 689             # Record in known apks, getting the added date at the same time..
 690             added = knownapks.recordapk(apk['apkname'], apk['id'])
 691             if added:
 692                 if use_date_from_apk and manifest.date_time[1] != 0:
 693                     added = datetime(*manifest.date_time).timetuple()
 694                     logging.debug("Using date from APK")
 695
 696                 apk['added'] = added
 697
 698             apkcache[apkfilename] = apk
 699             cachechanged = True
 700
 701         apks.append(apk)
 702
 703     return apks, cachechanged
 704
 705
 706 repo_pubkey_fingerprint = None
 707
 708
 709 # Generate a certificate fingerprint the same way keytool does it
 710 # (but with slightly different formatting)
 711 def cert_fingerprint(data):
 712     digest = hashlib.sha256(data).digest()
 713     ret = []
 714     ret.append(' '.join("%02X" % b for b in bytearray(digest)))
 715     return " ".join(ret)
 716
 717
 718 def extract_pubkey():
 719     global repo_pubkey_fingerprint
 720     if 'repo_pubkey' in config:
 721         pubkey = unhexlify(config['repo_pubkey'])
 722     else:
 723         p = FDroidPopenBytes([config['keytool'], '-exportcert',
 724                               '-alias', config['repo_keyalias'],
 725                               '-keystore', config['keystore'],
 726                               '-storepass:file', config['keystorepassfile']]
 727                              + config['smartcardoptions'],
 728                              output=False, stderr_to_stdout=False)
 729         if p.returncode != 0 or len(p.output) < 20:
 730             msg = "Failed to get repo pubkey!"
 731             if config['keystore'] == 'NONE':
 732                 msg += ' Is your crypto smartcard plugged in?'
 733             logging.critical(msg)
 734             sys.exit(1)
 735         pubkey = p.output
 736     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 737     return hexlify(pubkey)
 738
 739
 740 def make_index(apps, sortedids, apks, repodir, archive, categories):
 741     """Make a repo index.
 742
 743     :param apps: fully populated apps list
 744     :param apks: full populated apks list
 745     :param repodir: the repo directory
 746     :param archive: True if this is the archive repo, False if it's the
 747                     main one.
 748     :param categories: list of categories
 749     """
 750
 751     doc = Document()
 752
 753     def addElement(name, value, doc, parent):
 754         el = doc.createElement(name)
 755         el.appendChild(doc.createTextNode(value))
 756         parent.appendChild(el)
 757
 758     def addElementNonEmpty(name, value, doc, parent):
 759         if not value:
 760             return
 761         addElement(name, value, doc, parent)
 762
 763     def addElementCDATA(name, value, doc, parent):
 764         el = doc.createElement(name)
 765         el.appendChild(doc.createCDATASection(value))
 766         parent.appendChild(el)
 767
 768     root = doc.createElement("fdroid")
 769     doc.appendChild(root)
 770
 771     repoel = doc.createElement("repo")
 772
 773     mirrorcheckfailed = False
 774     for mirror in config.get('mirrors', []):
 775         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
 776         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
 777             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 778             mirrorcheckfailed = True
 779     if mirrorcheckfailed:
 780         sys.exit(1)
 781
 782     if archive:
 783         repoel.setAttribute("name", config['archive_name'])
 784         if config['repo_maxage'] != 0:
 785             repoel.setAttribute("maxage", str(config['repo_maxage']))
 786         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 787         repoel.setAttribute("url", config['archive_url'])
 788         addElement('description', config['archive_description'], doc, repoel)
 789         urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
 790         for mirror in config.get('mirrors', []):
 791             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 792
 793     else:
 794         repoel.setAttribute("name", config['repo_name'])
 795         if config['repo_maxage'] != 0:
 796             repoel.setAttribute("maxage", str(config['repo_maxage']))
 797         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 798         repoel.setAttribute("url", config['repo_url'])
 799         addElement('description', config['repo_description'], doc, repoel)
 800         urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
 801         for mirror in config.get('mirrors', []):
 802             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 803
 804     repoel.setAttribute("version", "15")
 805     repoel.setAttribute("timestamp", str(int(time.time())))
 806
 807     nosigningkey = False
 808     if not options.nosign:
 809         if 'repo_keyalias' not in config:
 810             nosigningkey = True
 811             logging.critical("'repo_keyalias' not found in config.py!")
 812         if 'keystore' not in config:
 813             nosigningkey = True
 814             logging.critical("'keystore' not found in config.py!")
 815         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 816             nosigningkey = True
 817             logging.critical("'keystorepass' not found in config.py!")
 818         if 'keypass' not in config and 'keypassfile' not in config:
 819             nosigningkey = True
 820             logging.critical("'keypass' not found in config.py!")
 821         if not os.path.exists(config['keystore']):
 822             nosigningkey = True
 823             logging.critical("'" + config['keystore'] + "' does not exist!")
 824         if nosigningkey:
 825             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 826             logging.warning("\tfdroid update --create-key")
 827             sys.exit(1)
 828
 829     repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
 830     root.appendChild(repoel)
 831
 832     for appid in sortedids:
 833         app = apps[appid]
 834
 835         if app.Disabled is not None:
 836             continue
 837
 838         # Get a list of the apks for this app...
 839         apklist = []
 840         for apk in apks:
 841             if apk['id'] == appid:
 842                 apklist.append(apk)
 843
 844         if len(apklist) == 0:
 845             continue
 846
 847         apel = doc.createElement("application")
 848         apel.setAttribute("id", app.id)
 849         root.appendChild(apel)
 850
 851         addElement('id', app.id, doc, apel)
 852         if app.added:
 853             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
 854         if app.lastupdated:
 855             addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
 856         addElement('name', app.Name, doc, apel)
 857         addElement('summary', app.Summary, doc, apel)
 858         if app.icon:
 859             addElement('icon', app.icon, doc, apel)
 860
 861         def linkres(appid):
 862             if appid in apps:
 863                 return ("fdroid.app:" + appid, apps[appid].Name)
 864             raise MetaDataException("Cannot resolve app id " + appid)
 865
 866         addElement('desc',
 867                    metadata.description_html(app.Description, linkres),
 868                    doc, apel)
 869         addElement('license', app.License, doc, apel)
 870         if app.Categories:
 871             addElement('categories', ','.join(app.Categories), doc, apel)
 872             # We put the first (primary) category in LAST, which will have
 873             # the desired effect of making clients that only understand one
 874             # category see that one.
 875             addElement('category', app.Categories[0], doc, apel)
 876         addElement('web', app.WebSite, doc, apel)
 877         addElement('source', app.SourceCode, doc, apel)
 878         addElement('tracker', app.IssueTracker, doc, apel)
 879         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 880         addElementNonEmpty('author', app.AuthorName, doc, apel)
 881         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 882         addElementNonEmpty('donate', app.Donate, doc, apel)
 883         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 884         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 885         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 886
 887         # These elements actually refer to the current version (i.e. which
 888         # one is recommended. They are historically mis-named, and need
 889         # changing, but stay like this for now to support existing clients.
 890         addElement('marketversion', app.CurrentVersion, doc, apel)
 891         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 892
 893         if app.AntiFeatures:
 894             af = app.AntiFeatures
 895             if af:
 896                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 897         if app.Provides:
 898             pv = app.Provides.split(',')
 899             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 900         if app.RequiresRoot:
 901             addElement('requirements', 'root', doc, apel)
 902
 903         # Sort the apk list into version order, just so the web site
 904         # doesn't have to do any work by default...
 905         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 906
 907         # Check for duplicates - they will make the client unhappy...
 908         for i in range(len(apklist) - 1):
 909             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 910                 logging.critical("duplicate versions: '%s' - '%s'" % (
 911                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 912                 sys.exit(1)
 913
 914         current_version_code = 0
 915         current_version_file = None
 916         for apk in apklist:
 917             # find the APK for the "Current Version"
 918             if current_version_code < apk['versioncode']:
 919                 current_version_code = apk['versioncode']
 920             if current_version_code < int(app.CurrentVersionCode):
 921                 current_version_file = apk['apkname']
 922
 923             apkel = doc.createElement("package")
 924             apel.appendChild(apkel)
 925             addElement('version', apk['version'], doc, apkel)
 926             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 927             addElement('apkname', apk['apkname'], doc, apkel)
 928             if 'srcname' in apk:
 929                 addElement('srcname', apk['srcname'], doc, apkel)
 930             for hash_type in ['sha256']:
 931                 if hash_type not in apk:
 932                     continue
 933                 hashel = doc.createElement("hash")
 934                 hashel.setAttribute("type", hash_type)
 935                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 936                 apkel.appendChild(hashel)
 937             addElement('sig', apk['sig'], doc, apkel)
 938             addElement('size', str(apk['size']), doc, apkel)
 939             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 940             if 'maxsdkversion' in apk:
 941                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 942             if 'added' in apk:
 943                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 944             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
 945             if 'nativecode' in apk:
 946                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 947             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
 948
 949         if current_version_file is not None \
 950                 and config['make_current_version_link'] \
 951                 and repodir == 'repo':  # only create these
 952             namefield = config['current_version_name_source']
 953             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
 954             apklinkname = sanitized_name + '.apk'
 955             current_version_path = os.path.join(repodir, current_version_file)
 956             if os.path.islink(apklinkname):
 957                 os.remove(apklinkname)
 958             os.symlink(current_version_path, apklinkname)
 959             # also symlink gpg signature, if it exists
 960             for extension in ('.asc', '.sig'):
 961                 sigfile_path = current_version_path + extension
 962                 if os.path.exists(sigfile_path):
 963                     siglinkname = apklinkname + extension
 964                     if os.path.islink(siglinkname):
 965                         os.remove(siglinkname)
 966                     os.symlink(sigfile_path, siglinkname)
 967
 968     if options.pretty:
 969         output = doc.toprettyxml(encoding='utf-8')
 970     else:
 971         output = doc.toxml(encoding='utf-8')
 972
 973     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 974         f.write(output)
 975
 976     if 'repo_keyalias' in config:
 977
 978         if options.nosign:
 979             logging.info("Creating unsigned index in preparation for signing")
 980         else:
 981             logging.info("Creating signed index with this key (SHA256):")
 982             logging.info("%s" % repo_pubkey_fingerprint)
 983
 984         # Create a jar of the index...
 985         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
 986         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
 987         if p.returncode != 0:
 988             logging.critical("Failed to create {0}".format(jar_output))
 989             sys.exit(1)
 990
 991         # Sign the index...
 992         signed = os.path.join(repodir, 'index.jar')
 993         if options.nosign:
 994             # Remove old signed index if not signing
 995             if os.path.exists(signed):
 996                 os.remove(signed)
 997         else:
 998             args = [config['jarsigner'], '-keystore', config['keystore'],
 999                     '-storepass:file', config['keystorepassfile'],
1000                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1001                     signed, config['repo_keyalias']]
1002             if config['keystore'] == 'NONE':
1003                 args += config['smartcardoptions']
1004             else:  # smardcards never use -keypass
1005                 args += ['-keypass:file', config['keypassfile']]
1006             p = FDroidPopen(args)
1007             if p.returncode != 0:
1008                 logging.critical("Failed to sign index")
1009                 sys.exit(1)
1010
1011     # Copy the repo icon into the repo directory...
1012     icon_dir = os.path.join(repodir, 'icons')
1013     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1014     shutil.copyfile(config['repo_icon'], iconfilename)
1015
1016     # Write a category list in the repo to allow quick access...
1017     catdata = ''
1018     for cat in categories:
1019         catdata += cat + '\n'
1020     with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
1021         f.write(catdata)
1022
1023
1024 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1025
1026     for appid, app in apps.items():
1027
1028         if app.ArchivePolicy:
1029             keepversions = int(app.ArchivePolicy[:-9])
1030         else:
1031             keepversions = defaultkeepversions
1032
1033         def filter_apk_list_sorted(apk_list):
1034             res = []
1035             for apk in apk_list:
1036                 if apk['id'] == appid:
1037                     res.append(apk)
1038
1039             # Sort the apk list by version code. First is highest/newest.
1040             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1041
1042         def move_file(from_dir, to_dir, filename, ignore_missing):
1043             from_path = os.path.join(from_dir, filename)
1044             if ignore_missing and not os.path.exists(from_path):
1045                 return
1046             to_path = os.path.join(to_dir, filename)
1047             shutil.move(from_path, to_path)
1048
1049         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1050                       .format(appid, len(apks), keepversions, len(archapks)))
1051
1052         if len(apks) > keepversions:
1053             apklist = filter_apk_list_sorted(apks)
1054             # Move back the ones we don't want.
1055             for apk in apklist[keepversions:]:
1056                 logging.info("Moving " + apk['apkname'] + " to archive")
1057                 move_file(repodir, archivedir, apk['apkname'], False)
1058                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1059                 for density in all_screen_densities:
1060                     repo_icon_dir = get_icon_dir(repodir, density)
1061                     archive_icon_dir = get_icon_dir(archivedir, density)
1062                     if density not in apk['icons']:
1063                         continue
1064                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1065                 if 'srcname' in apk:
1066                     move_file(repodir, archivedir, apk['srcname'], False)
1067                 archapks.append(apk)
1068                 apks.remove(apk)
1069         elif len(apks) < keepversions and len(archapks) > 0:
1070             required = keepversions - len(apks)
1071             archapklist = filter_apk_list_sorted(archapks)
1072             # Move forward the ones we want again.
1073             for apk in archapklist[:required]:
1074                 logging.info("Moving " + apk['apkname'] + " from archive")
1075                 move_file(archivedir, repodir, apk['apkname'], False)
1076                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1077                 for density in all_screen_densities:
1078                     repo_icon_dir = get_icon_dir(repodir, density)
1079                     archive_icon_dir = get_icon_dir(archivedir, density)
1080                     if density not in apk['icons']:
1081                         continue
1082                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1083                 if 'srcname' in apk:
1084                     move_file(archivedir, repodir, apk['srcname'], False)
1085                 archapks.remove(apk)
1086                 apks.append(apk)
1087
1088
1089 def add_apks_to_per_app_repos(repodir, apks):
1090     apks_per_app = dict()
1091     for apk in apks:
1092         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1093         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1094         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1095         apks_per_app[apk['id']] = apk
1096
1097         if not os.path.exists(apk['per_app_icons']):
1098             logging.info('Adding new repo for only ' + apk['id'])
1099             os.makedirs(apk['per_app_icons'])
1100
1101         apkpath = os.path.join(repodir, apk['apkname'])
1102         shutil.copy(apkpath, apk['per_app_repo'])
1103         apksigpath = apkpath + '.sig'
1104         if os.path.exists(apksigpath):
1105             shutil.copy(apksigpath, apk['per_app_repo'])
1106         apkascpath = apkpath + '.asc'
1107         if os.path.exists(apkascpath):
1108             shutil.copy(apkascpath, apk['per_app_repo'])
1109
1110
1111 config = None
1112 options = None
1113
1114
1115 def main():
1116
1117     global config, options
1118
1119     # Parse command line...
1120     parser = ArgumentParser()
1121     common.setup_global_opts(parser)
1122     parser.add_argument("--create-key", action="store_true", default=False,
1123                         help="Create a repo signing key in a keystore")
1124     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1125                         help="Create skeleton metadata files that are missing")
1126     parser.add_argument("--delete-unknown", action="store_true", default=False,
1127                         help="Delete APKs without metadata from the repo")
1128     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1129                         help="Report on build data status")
1130     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1131                         help="Interactively ask about things that need updating.")
1132     parser.add_argument("-I", "--icons", action="store_true", default=False,
1133                         help="Resize all the icons exceeding the max pixel size and exit")
1134     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1135                         help="Specify editor to use in interactive mode. Default " +
1136                         "is /etc/alternatives/editor")
1137     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1138                         help="Update the wiki")
1139     parser.add_argument("--pretty", action="store_true", default=False,
1140                         help="Produce human-readable index.xml")
1141     parser.add_argument("--clean", action="store_true", default=False,
1142                         help="Clean update - don't uses caches, reprocess all apks")
1143     parser.add_argument("--nosign", action="store_true", default=False,
1144                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1145     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1146                         help="Use date from apk instead of current time for newly added apks")
1147     options = parser.parse_args()
1148
1149     config = common.read_config(options)
1150
1151     if not ('jarsigner' in config and 'keytool' in config):
1152         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1153         sys.exit(1)
1154
1155     repodirs = ['repo']
1156     if config['archive_older'] != 0:
1157         repodirs.append('archive')
1158         if not os.path.exists('archive'):
1159             os.mkdir('archive')
1160
1161     if options.icons:
1162         resize_all_icons(repodirs)
1163         sys.exit(0)
1164
1165     # check that icons exist now, rather than fail at the end of `fdroid update`
1166     for k in ['repo_icon', 'archive_icon']:
1167         if k in config:
1168             if not os.path.exists(config[k]):
1169                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1170                 sys.exit(1)
1171
1172     # if the user asks to create a keystore, do it now, reusing whatever it can
1173     if options.create_key:
1174         if os.path.exists(config['keystore']):
1175             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1176             logging.critical("\t'" + config['keystore'] + "'")
1177             sys.exit(1)
1178
1179         if 'repo_keyalias' not in config:
1180             config['repo_keyalias'] = socket.getfqdn()
1181             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1182         if 'keydname' not in config:
1183             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1184             common.write_to_config(config, 'keydname', config['keydname'])
1185         if 'keystore' not in config:
1186             config['keystore'] = common.default_config.keystore
1187             common.write_to_config(config, 'keystore', config['keystore'])
1188
1189         password = common.genpassword()
1190         if 'keystorepass' not in config:
1191             config['keystorepass'] = password
1192             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1193         if 'keypass' not in config:
1194             config['keypass'] = password
1195             common.write_to_config(config, 'keypass', config['keypass'])
1196         common.genkeystore(config)
1197
1198     # Get all apps...
1199     apps = metadata.read_metadata()
1200
1201     # Generate a list of categories...
1202     categories = set()
1203     for app in apps.values():
1204         categories.update(app.Categories)
1205
1206     # Read known apks data (will be updated and written back when we've finished)
1207     knownapks = common.KnownApks()
1208
1209     # Gather information about all the apk files in the repo directory, using
1210     # cached data if possible.
1211     apkcachefile = os.path.join('tmp', 'apkcache')
1212     if not options.clean and os.path.exists(apkcachefile):
1213         with open(apkcachefile, 'rb') as cf:
1214             apkcache = pickle.load(cf, encoding='utf-8')
1215     else:
1216         apkcache = {}
1217
1218     delete_disabled_builds(apps, apkcache, repodirs)
1219
1220     # Scan all apks in the main repo
1221     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1222
1223     # Generate warnings for apk's with no metadata (or create skeleton
1224     # metadata files, if requested on the command line)
1225     newmetadata = False
1226     for apk in apks:
1227         if apk['id'] not in apps:
1228             if options.create_metadata:
1229                 if 'name' not in apk:
1230                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1231                     continue
1232                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1233                 f.write("License:Unknown\n")
1234                 f.write("Web Site:\n")
1235                 f.write("Source Code:\n")
1236                 f.write("Issue Tracker:\n")
1237                 f.write("Changelog:\n")
1238                 f.write("Summary:" + apk['name'] + "\n")
1239                 f.write("Description:\n")
1240                 f.write(apk['name'] + "\n")
1241                 f.write(".\n")
1242                 f.close()
1243                 logging.info("Generated skeleton metadata for " + apk['id'])
1244                 newmetadata = True
1245             else:
1246                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1247                 if options.delete_unknown:
1248                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1249                     rmf = os.path.join(repodirs[0], apk['apkname'])
1250                     if not os.path.exists(rmf):
1251                         logging.error("Could not find {0} to remove it".format(rmf))
1252                     else:
1253                         os.remove(rmf)
1254                 else:
1255                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1256
1257     # update the metadata with the newly created ones included
1258     if newmetadata:
1259         apps = metadata.read_metadata()
1260
1261     # Scan the archive repo for apks as well
1262     if len(repodirs) > 1:
1263         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1264         if cc:
1265             cachechanged = True
1266     else:
1267         archapks = []
1268
1269     # Some information from the apks needs to be applied up to the application
1270     # level. When doing this, we use the info from the most recent version's apk.
1271     # We deal with figuring out when the app was added and last updated at the
1272     # same time.
1273     for appid, app in apps.items():
1274         bestver = 0
1275         for apk in apks + archapks:
1276             if apk['id'] == appid:
1277                 if apk['versioncode'] > bestver:
1278                     bestver = apk['versioncode']
1279                     bestapk = apk
1280
1281                 if 'added' in apk:
1282                     if not app.added or apk['added'] < app.added:
1283                         app.added = apk['added']
1284                     if not app.lastupdated or apk['added'] > app.lastupdated:
1285                         app.lastupdated = apk['added']
1286
1287         if not app.added:
1288             logging.debug("Don't know when " + appid + " was added")
1289         if not app.lastupdated:
1290             logging.debug("Don't know when " + appid + " was last updated")
1291
1292         if bestver == 0:
1293             if app.Name is None:
1294                 app.Name = app.AutoName or appid
1295             app.icon = None
1296             logging.debug("Application " + appid + " has no packages")
1297         else:
1298             if app.Name is None:
1299                 app.Name = bestapk['name']
1300             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1301             if app.CurrentVersionCode is None:
1302                 app.CurrentVersionCode = str(bestver)
1303
1304     # Sort the app list by name, then the web site doesn't have to by default.
1305     # (we had to wait until we'd scanned the apks to do this, because mostly the
1306     # name comes from there!)
1307     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1308
1309     # APKs are placed into multiple repos based on the app package, providing
1310     # per-app subscription feeds for nightly builds and things like it
1311     if config['per_app_repos']:
1312         add_apks_to_per_app_repos(repodirs[0], apks)
1313         for appid, app in apps.items():
1314             repodir = os.path.join(appid, 'fdroid', 'repo')
1315             appdict = dict()
1316             appdict[appid] = app
1317             if os.path.isdir(repodir):
1318                 make_index(appdict, [appid], apks, repodir, False, categories)
1319             else:
1320                 logging.info('Skipping index generation for ' + appid)
1321         return
1322
1323     if len(repodirs) > 1:
1324         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1325
1326     # Make the index for the main repo...
1327     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1328
1329     # If there's an archive repo,  make the index for it. We already scanned it
1330     # earlier on.
1331     if len(repodirs) > 1:
1332         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1333
1334     if config['update_stats']:
1335
1336         # Update known apks info...
1337         knownapks.writeifchanged()
1338
1339         # Generate latest apps data for widget
1340         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1341             data = ''
1342             with open(os.path.join('stats', 'latestapps.txt'), 'r') as f:
1343                 for line in f:
1344                     appid = line.rstrip()
1345                     data += appid + "\t"
1346                     app = apps[appid]
1347                     data += app.Name + "\t"
1348                     if app.icon is not None:
1349                         data += app.icon + "\t"
1350                     data += app.License + "\n"
1351             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
1352                 f.write(data)
1353
1354     if cachechanged:
1355         with open(apkcachefile, 'wb') as cf:
1356             pickle.dump(apkcache, cf)
1357
1358     # Update the wiki...
1359     if options.wiki:
1360         update_wiki(apps, sortedids, apks + archapks)
1361
1362     logging.info("Finished.")
1363
1364 if __name__ == "__main__":
1365     main()