fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   5 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU Affero General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Affero General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Affero General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 import sys
  21 import os
  22 import shutil
  23 import glob
  24 import re
  25 import socket
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 import urllib.parse
  30 from datetime import datetime, timedelta
  31 from xml.dom.minidom import Document
  32 from argparse import ArgumentParser
  33 import time
  34 from pyasn1.error import PyAsn1Error
  35 from pyasn1.codec.der import decoder, encoder
  36 from pyasn1_modules import rfc2315
  37 from binascii import hexlify, unhexlify
  38
  39 from PIL import Image
  40 import logging
  41
  42 from . import common
  43 from . import metadata
  44 from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
  45 from .metadata import MetaDataException
  46
  47 METADATA_VERSION = 16
  48
  49 screen_densities = ['640', '480', '320', '240', '160', '120']
  50
  51 all_screen_densities = ['0'] + screen_densities
  52
  53
  54 def dpi_to_px(density):
  55     return (int(density) * 48) / 160
  56
  57
  58 def px_to_dpi(px):
  59     return (int(px) * 160) / 48
  60
  61
  62 def get_icon_dir(repodir, density):
  63     if density == '0':
  64         return os.path.join(repodir, "icons")
  65     return os.path.join(repodir, "icons-%s" % density)
  66
  67
  68 def get_icon_dirs(repodir):
  69     for density in screen_densities:
  70         yield get_icon_dir(repodir, density)
  71
  72
  73 def get_all_icon_dirs(repodir):
  74     for density in all_screen_densities:
  75         yield get_icon_dir(repodir, density)
  76
  77
  78 def update_wiki(apps, sortedids, apks):
  79     """Update the wiki
  80
  81     :param apps: fully populated list of all applications
  82     :param apks: all apks, except...
  83     """
  84     logging.info("Updating wiki")
  85     wikicat = 'Apps'
  86     wikiredircat = 'App Redirects'
  87     import mwclient
  88     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  89                          path=config['wiki_path'])
  90     site.login(config['wiki_user'], config['wiki_password'])
  91     generated_pages = {}
  92     generated_redirects = {}
  93
  94     for appid in sortedids:
  95         app = apps[appid]
  96
  97         wikidata = ''
  98         if app.Disabled:
  99             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 100         if app.AntiFeatures:
 101             for af in app.AntiFeatures:
 102                 wikidata += '{{AntiFeature|' + af + '}}\n'
 103         if app.RequiresRoot:
 104             requiresroot = 'Yes'
 105         else:
 106             requiresroot = 'No'
 107         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 108             appid,
 109             app.Name,
 110             time.strftime('%Y-%m-%d', app.added) if app.added else '',
 111             time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
 112             app.SourceCode,
 113             app.IssueTracker,
 114             app.WebSite,
 115             app.Changelog,
 116             app.Donate,
 117             app.FlattrID,
 118             app.Bitcoin,
 119             app.Litecoin,
 120             app.License,
 121             requiresroot,
 122             app.AuthorName,
 123             app.AuthorEmail)
 124
 125         if app.Provides:
 126             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 127
 128         wikidata += app.Summary
 129         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 130
 131         wikidata += "=Description=\n"
 132         wikidata += metadata.description_wiki(app.Description) + "\n"
 133
 134         wikidata += "=Maintainer Notes=\n"
 135         if app.MaintainerNotes:
 136             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 137         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 138
 139         # Get a list of all packages for this application...
 140         apklist = []
 141         gotcurrentver = False
 142         cantupdate = False
 143         buildfails = False
 144         for apk in apks:
 145             if apk['id'] == appid:
 146                 if str(apk['versioncode']) == app.CurrentVersionCode:
 147                     gotcurrentver = True
 148                 apklist.append(apk)
 149         # Include ones we can't build, as a special case...
 150         for build in app.builds:
 151             if build.disable:
 152                 if build.vercode == app.CurrentVersionCode:
 153                     cantupdate = True
 154                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 155                 apklist.append({'versioncode': int(build.vercode),
 156                                 'version': build.version,
 157                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 158                                 })
 159             else:
 160                 builtit = False
 161                 for apk in apklist:
 162                     if apk['versioncode'] == int(build.vercode):
 163                         builtit = True
 164                         break
 165                 if not builtit:
 166                     buildfails = True
 167                     apklist.append({'versioncode': int(build.vercode),
 168                                     'version': build.version,
 169                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
 170                                     })
 171         if app.CurrentVersionCode == '0':
 172             cantupdate = True
 173         # Sort with most recent first...
 174         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 175
 176         wikidata += "=Versions=\n"
 177         if len(apklist) == 0:
 178             wikidata += "We currently have no versions of this app available."
 179         elif not gotcurrentver:
 180             wikidata += "We don't have the current version of this app."
 181         else:
 182             wikidata += "We have the current version of this app."
 183         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 184         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 185         if len(app.NoSourceSince) > 0:
 186             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 187         if len(app.CurrentVersion) > 0:
 188             wikidata += "The current (recommended) version is " + app.CurrentVersion
 189             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 190         validapks = 0
 191         for apk in apklist:
 192             wikidata += "==" + apk['version'] + "==\n"
 193
 194             if 'buildproblem' in apk:
 195                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 196             else:
 197                 validapks += 1
 198                 wikidata += "This version is built and signed by "
 199                 if 'srcname' in apk:
 200                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 201                 else:
 202                     wikidata += "the original developer.\n\n"
 203             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 204
 205         wikidata += '\n[[Category:' + wikicat + ']]\n'
 206         if len(app.NoSourceSince) > 0:
 207             wikidata += '\n[[Category:Apps missing source code]]\n'
 208         if validapks == 0 and not app.Disabled:
 209             wikidata += '\n[[Category:Apps with no packages]]\n'
 210         if cantupdate and not app.Disabled:
 211             wikidata += "\n[[Category:Apps we cannot update]]\n"
 212         if buildfails and not app.Disabled:
 213             wikidata += "\n[[Category:Apps with failing builds]]\n"
 214         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 215             wikidata += '\n[[Category:Apps to Update]]\n'
 216         if app.Disabled:
 217             wikidata += '\n[[Category:Apps that are disabled]]\n'
 218         if app.UpdateCheckMode == 'None' and not app.Disabled:
 219             wikidata += '\n[[Category:Apps with no update check]]\n'
 220         for appcat in app.Categories:
 221             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 222
 223         # We can't have underscores in the page name, even if they're in
 224         # the package ID, because MediaWiki messes with them...
 225         pagename = appid.replace('_', ' ')
 226
 227         # Drop a trailing newline, because mediawiki is going to drop it anyway
 228         # and it we don't we'll think the page has changed when it hasn't...
 229         if wikidata.endswith('\n'):
 230             wikidata = wikidata[:-1]
 231
 232         generated_pages[pagename] = wikidata
 233
 234         # Make a redirect from the name to the ID too, unless there's
 235         # already an existing page with the name and it isn't a redirect.
 236         noclobber = False
 237         apppagename = app.Name.replace('_', ' ')
 238         apppagename = apppagename.replace('{', '')
 239         apppagename = apppagename.replace('}', ' ')
 240         apppagename = apppagename.replace(':', ' ')
 241         # Drop double spaces caused mostly by replacing ':' above
 242         apppagename = apppagename.replace('  ', ' ')
 243         for expagename in site.allpages(prefix=apppagename,
 244                                         filterredir='nonredirects',
 245                                         generator=False):
 246             if expagename == apppagename:
 247                 noclobber = True
 248         # Another reason not to make the redirect page is if the app name
 249         # is the same as it's ID, because that will overwrite the real page
 250         # with an redirect to itself! (Although it seems like an odd
 251         # scenario this happens a lot, e.g. where there is metadata but no
 252         # builds or binaries to extract a name from.
 253         if apppagename == pagename:
 254             noclobber = True
 255         if not noclobber:
 256             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 257
 258     for tcat, genp in [(wikicat, generated_pages),
 259                        (wikiredircat, generated_redirects)]:
 260         catpages = site.Pages['Category:' + tcat]
 261         existingpages = []
 262         for page in catpages:
 263             existingpages.append(page.name)
 264             if page.name in genp:
 265                 pagetxt = page.edit()
 266                 if pagetxt != genp[page.name]:
 267                     logging.debug("Updating modified page " + page.name)
 268                     page.save(genp[page.name], summary='Auto-updated')
 269                 else:
 270                     logging.debug("Page " + page.name + " is unchanged")
 271             else:
 272                 logging.warn("Deleting page " + page.name)
 273                 page.delete('No longer published')
 274         for pagename, text in genp.items():
 275             logging.debug("Checking " + pagename)
 276             if pagename not in existingpages:
 277                 logging.debug("Creating page " + pagename)
 278                 try:
 279                     newpage = site.Pages[pagename]
 280                     newpage.save(text, summary='Auto-created')
 281                 except:
 282                     logging.error("...FAILED to create page '{0}'".format(pagename))
 283
 284     # Purge server cache to ensure counts are up to date
 285     site.pages['Repository Maintenance'].purge()
 286
 287
 288 def delete_disabled_builds(apps, apkcache, repodirs):
 289     """Delete disabled build outputs.
 290
 291     :param apps: list of all applications, as per metadata.read_metadata
 292     :param apkcache: current apk cache information
 293     :param repodirs: the repo directories to process
 294     """
 295     for appid, app in apps.items():
 296         for build in app.builds:
 297             if not build.disable:
 298                 continue
 299             apkfilename = appid + '_' + str(build.vercode) + '.apk'
 300             iconfilename = "%s.%s.png" % (
 301                 appid,
 302                 build.vercode)
 303             for repodir in repodirs:
 304                 files = [
 305                     os.path.join(repodir, apkfilename),
 306                     os.path.join(repodir, apkfilename + '.asc'),
 307                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 308                 ]
 309                 for density in all_screen_densities:
 310                     repo_dir = get_icon_dir(repodir, density)
 311                     files.append(os.path.join(repo_dir, iconfilename))
 312
 313                 for f in files:
 314                     if os.path.exists(f):
 315                         logging.info("Deleting disabled build output " + f)
 316                         os.remove(f)
 317             if apkfilename in apkcache:
 318                 del apkcache[apkfilename]
 319
 320
 321 def resize_icon(iconpath, density):
 322
 323     if not os.path.isfile(iconpath):
 324         return
 325
 326     try:
 327         im = Image.open(iconpath)
 328         size = dpi_to_px(density)
 329
 330         if any(length > size for length in im.size):
 331             oldsize = im.size
 332             im.thumbnail((size, size), Image.ANTIALIAS)
 333             logging.debug("%s was too large at %s - new size is %s" % (
 334                 iconpath, oldsize, im.size))
 335             im.save(iconpath, "PNG")
 336
 337     except Exception as e:
 338         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 339
 340
 341 def resize_all_icons(repodirs):
 342     """Resize all icons that exceed the max size
 343
 344     :param repodirs: the repo directories to process
 345     """
 346     for repodir in repodirs:
 347         for density in screen_densities:
 348             icon_dir = get_icon_dir(repodir, density)
 349             icon_glob = os.path.join(icon_dir, '*.png')
 350             for iconpath in glob.glob(icon_glob):
 351                 resize_icon(iconpath, density)
 352
 353
 354 # A signature block file with a .DSA, .RSA, or .EC extension
 355 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 356
 357
 358 def getsig(apkpath):
 359     """ Get the signing certificate of an apk. To get the same md5 has that
 360     Android gets, we encode the .RSA certificate in a specific format and pass
 361     it hex-encoded to the md5 digest algorithm.
 362
 363     :param apkpath: path to the apk
 364     :returns: A string containing the md5 of the signature of the apk or None
 365               if an error occurred.
 366     """
 367
 368     cert = None
 369
 370     # verify the jar signature is correct
 371     args = [config['jarsigner'], '-verify', apkpath]
 372     p = FDroidPopen(args)
 373     if p.returncode != 0:
 374         logging.critical(apkpath + " has a bad signature!")
 375         return None
 376
 377     with zipfile.ZipFile(apkpath, 'r') as apk:
 378
 379         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 380
 381         if len(certs) < 1:
 382             logging.error("Found no signing certificates on %s" % apkpath)
 383             return None
 384         if len(certs) > 1:
 385             logging.error("Found multiple signing certificates on %s" % apkpath)
 386             return None
 387
 388         cert = apk.read(certs[0])
 389
 390     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 391     if content.getComponentByName('contentType') != rfc2315.signedData:
 392         logging.error("Unexpected format.")
 393         return None
 394
 395     content = decoder.decode(content.getComponentByName('content'),
 396                              asn1Spec=rfc2315.SignedData())[0]
 397     try:
 398         certificates = content.getComponentByName('certificates')
 399     except PyAsn1Error:
 400         logging.error("Certificates not found.")
 401         return None
 402
 403     cert_encoded = encoder.encode(certificates)[4:]
 404
 405     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 406
 407
 408 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
 409     """Scan the apks in the given repo directory.
 410
 411     This also extracts the icons.
 412
 413     :param apps: list of all applications, as per metadata.read_metadata
 414     :param apkcache: current apk cache information
 415     :param repodir: repo directory to scan
 416     :param knownapks: known apks info
 417     :param use_date_from_apk: use date from APK (instead of current date)
 418                               for newly added APKs
 419     :returns: (apks, cachechanged) where apks is a list of apk information,
 420               and cachechanged is True if the apkcache got changed.
 421     """
 422
 423     cachechanged = False
 424
 425     for icon_dir in get_all_icon_dirs(repodir):
 426         if os.path.exists(icon_dir):
 427             if options.clean:
 428                 shutil.rmtree(icon_dir)
 429                 os.makedirs(icon_dir)
 430         else:
 431             os.makedirs(icon_dir)
 432
 433     apks = []
 434     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 435     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 436     vername_pat = re.compile(".*versionName='([^']*)'.*")
 437     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 438     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 439     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 440     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 441     string_pat = re.compile(".* name='([^']*)'.*")
 442     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 443
 444         apkfilename = apkfile[len(repodir) + 1:]
 445         if ' ' in apkfilename:
 446             logging.critical("Spaces in filenames are not allowed.")
 447             sys.exit(1)
 448
 449         # Calculate the sha256...
 450         sha = hashlib.sha256()
 451         with open(apkfile, 'rb') as f:
 452             while True:
 453                 t = f.read(16384)
 454                 if len(t) == 0:
 455                     break
 456                 sha.update(t)
 457             shasum = sha.hexdigest()
 458
 459         usecache = False
 460         if apkfilename in apkcache:
 461             apk = apkcache[apkfilename]
 462             if apk['sha256'] == shasum:
 463                 logging.debug("Reading " + apkfilename + " from cache")
 464                 usecache = True
 465             else:
 466                 logging.debug("Ignoring stale cache data for " + apkfilename)
 467
 468         if not usecache:
 469             logging.debug("Processing " + apkfilename)
 470             apk = {}
 471             apk['apkname'] = apkfilename
 472             apk['sha256'] = shasum
 473             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 474             if os.path.exists(os.path.join(repodir, srcfilename)):
 475                 apk['srcname'] = srcfilename
 476             apk['size'] = os.path.getsize(apkfile)
 477             apk['permissions'] = set()
 478             apk['features'] = set()
 479             apk['icons_src'] = {}
 480             apk['icons'] = {}
 481             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 482             if p.returncode != 0:
 483                 if options.delete_unknown:
 484                     if os.path.exists(apkfile):
 485                         logging.error("Failed to get apk information, deleting " + apkfile)
 486                         os.remove(apkfile)
 487                     else:
 488                         logging.error("Could not find {0} to remove it".format(apkfile))
 489                 else:
 490                     logging.error("Failed to get apk information, skipping " + apkfile)
 491                 continue
 492             for line in p.output.splitlines():
 493                 if line.startswith("package:"):
 494                     try:
 495                         apk['id'] = re.match(name_pat, line).group(1)
 496                         apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
 497                         apk['version'] = re.match(vername_pat, line).group(1)
 498                     except Exception as e:
 499                         logging.error("Package matching failed: " + str(e))
 500                         logging.info("Line was: " + line)
 501                         sys.exit(1)
 502                 elif line.startswith("application:"):
 503                     apk['name'] = re.match(label_pat, line).group(1)
 504                     # Keep path to non-dpi icon in case we need it
 505                     match = re.match(icon_pat_nodpi, line)
 506                     if match:
 507                         apk['icons_src']['-1'] = match.group(1)
 508                 elif line.startswith("launchable-activity:"):
 509                     # Only use launchable-activity as fallback to application
 510                     if not apk['name']:
 511                         apk['name'] = re.match(label_pat, line).group(1)
 512                     if '-1' not in apk['icons_src']:
 513                         match = re.match(icon_pat_nodpi, line)
 514                         if match:
 515                             apk['icons_src']['-1'] = match.group(1)
 516                 elif line.startswith("application-icon-"):
 517                     match = re.match(icon_pat, line)
 518                     if match:
 519                         density = match.group(1)
 520                         path = match.group(2)
 521                         apk['icons_src'][density] = path
 522                 elif line.startswith("sdkVersion:"):
 523                     m = re.match(sdkversion_pat, line)
 524                     if m is None:
 525                         logging.error(line.replace('sdkVersion:', '')
 526                                       + ' is not a valid minSdkVersion!')
 527                     else:
 528                         apk['minSdkVersion'] = m.group(1)
 529                         # if target not set, default to min
 530                         if 'targetSdkVersion' not in apk:
 531                             apk['targetSdkVersion'] = m.group(1)
 532                 elif line.startswith("targetSdkVersion:"):
 533                     m = re.match(sdkversion_pat, line)
 534                     if m is None:
 535                         logging.error(line.replace('targetSdkVersion:', '')
 536                                       + ' is not a valid targetSdkVersion!')
 537                     else:
 538                         apk['targetSdkVersion'] = m.group(1)
 539                 elif line.startswith("maxSdkVersion:"):
 540                     apk['maxSdkVersion'] = re.match(sdkversion_pat, line).group(1)
 541                 elif line.startswith("native-code:"):
 542                     apk['nativecode'] = []
 543                     for arch in line[13:].split(' '):
 544                         apk['nativecode'].append(arch[1:-1])
 545                 elif line.startswith("uses-permission:"):
 546                     perm = re.match(string_pat, line).group(1)
 547                     if perm.startswith("android.permission."):
 548                         perm = perm[19:]
 549                     apk['permissions'].add(perm)
 550                 elif line.startswith("uses-feature:"):
 551                     perm = re.match(string_pat, line).group(1)
 552                     # Filter out this, it's only added with the latest SDK tools and
 553                     # causes problems for lots of apps.
 554                     if perm != "android.hardware.screen.portrait" \
 555                             and perm != "android.hardware.screen.landscape":
 556                         if perm.startswith("android.feature."):
 557                             perm = perm[16:]
 558                         apk['features'].add(perm)
 559
 560             if 'minSdkVersion' not in apk:
 561                 logging.warn("No SDK version information found in {0}".format(apkfile))
 562                 apk['minSdkVersion'] = 1
 563
 564             # Check for debuggable apks...
 565             if common.isApkDebuggable(apkfile, config):
 566                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 567
 568             # Get the signature (or md5 of, to be precise)...
 569             logging.debug('Getting signature of {0}'.format(apkfile))
 570             apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 571             if not apk['sig']:
 572                 logging.critical("Failed to get apk signature")
 573                 sys.exit(1)
 574
 575             apkzip = zipfile.ZipFile(apkfile, 'r')
 576
 577             # if an APK has files newer than the system time, suggest updating
 578             # the system clock.  This is useful for offline systems, used for
 579             # signing, which do not have another source of clock sync info. It
 580             # has to be more than 24 hours newer because ZIP/APK files do not
 581             # store timezone info
 582             manifest = apkzip.getinfo('AndroidManifest.xml')
 583             if manifest.date_time[1] == 0:  # month can't be zero
 584                 logging.debug('AndroidManifest.xml has no date')
 585             else:
 586                 dt_obj = datetime(*manifest.date_time)
 587                 checkdt = dt_obj - timedelta(1)
 588                 if datetime.today() < checkdt:
 589                     logging.warn('System clock is older than manifest in: '
 590                                  + apkfilename
 591                                  + '\nSet clock to that time using:\n'
 592                                  + 'sudo date -s "' + str(dt_obj) + '"')
 593
 594             iconfilename = "%s.%s.png" % (
 595                 apk['id'],
 596                 apk['versioncode'])
 597
 598             # Extract the icon file...
 599             empty_densities = []
 600             for density in screen_densities:
 601                 if density not in apk['icons_src']:
 602                     empty_densities.append(density)
 603                     continue
 604                 iconsrc = apk['icons_src'][density]
 605                 icon_dir = get_icon_dir(repodir, density)
 606                 icondest = os.path.join(icon_dir, iconfilename)
 607
 608                 try:
 609                     with open(icondest, 'wb') as f:
 610                         f.write(apkzip.read(iconsrc))
 611                     apk['icons'][density] = iconfilename
 612
 613                 except:
 614                     logging.warn("Error retrieving icon file")
 615                     del apk['icons'][density]
 616                     del apk['icons_src'][density]
 617                     empty_densities.append(density)
 618
 619             if '-1' in apk['icons_src']:
 620                 iconsrc = apk['icons_src']['-1']
 621                 iconpath = os.path.join(
 622                     get_icon_dir(repodir, '0'), iconfilename)
 623                 with open(iconpath, 'wb') as f:
 624                     f.write(apkzip.read(iconsrc))
 625                 try:
 626                     im = Image.open(iconpath)
 627                     dpi = px_to_dpi(im.size[0])
 628                     for density in screen_densities:
 629                         if density in apk['icons']:
 630                             break
 631                         if density == screen_densities[-1] or dpi >= int(density):
 632                             apk['icons'][density] = iconfilename
 633                             shutil.move(iconpath,
 634                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 635                             empty_densities.remove(density)
 636                             break
 637                 except Exception as e:
 638                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 639
 640             if apk['icons']:
 641                 apk['icon'] = iconfilename
 642
 643             apkzip.close()
 644
 645             # First try resizing down to not lose quality
 646             last_density = None
 647             for density in screen_densities:
 648                 if density not in empty_densities:
 649                     last_density = density
 650                     continue
 651                 if last_density is None:
 652                     continue
 653                 logging.debug("Density %s not available, resizing down from %s"
 654                               % (density, last_density))
 655
 656                 last_iconpath = os.path.join(
 657                     get_icon_dir(repodir, last_density), iconfilename)
 658                 iconpath = os.path.join(
 659                     get_icon_dir(repodir, density), iconfilename)
 660                 try:
 661                     im = Image.open(last_iconpath)
 662                 except:
 663                     logging.warn("Invalid image file at %s" % last_iconpath)
 664                     continue
 665
 666                 size = dpi_to_px(density)
 667
 668                 im.thumbnail((size, size), Image.ANTIALIAS)
 669                 im.save(iconpath, "PNG")
 670                 empty_densities.remove(density)
 671
 672             # Then just copy from the highest resolution available
 673             last_density = None
 674             for density in reversed(screen_densities):
 675                 if density not in empty_densities:
 676                     last_density = density
 677                     continue
 678                 if last_density is None:
 679                     continue
 680                 logging.debug("Density %s not available, copying from lower density %s"
 681                               % (density, last_density))
 682
 683                 shutil.copyfile(
 684                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 685                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 686
 687                 empty_densities.remove(density)
 688
 689             for density in screen_densities:
 690                 icon_dir = get_icon_dir(repodir, density)
 691                 icondest = os.path.join(icon_dir, iconfilename)
 692                 resize_icon(icondest, density)
 693
 694             # Copy from icons-mdpi to icons since mdpi is the baseline density
 695             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 696             if os.path.isfile(baseline):
 697                 apk['icons']['0'] = iconfilename
 698                 shutil.copyfile(baseline,
 699                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 700
 701             # Record in known apks, getting the added date at the same time..
 702             added = knownapks.recordapk(apk['apkname'], apk['id'])
 703             if added:
 704                 if use_date_from_apk and manifest.date_time[1] != 0:
 705                     added = datetime(*manifest.date_time).timetuple()
 706                     logging.debug("Using date from APK")
 707
 708                 apk['added'] = added
 709
 710             apkcache[apkfilename] = apk
 711             cachechanged = True
 712
 713         apks.append(apk)
 714
 715     return apks, cachechanged
 716
 717
 718 repo_pubkey_fingerprint = None
 719
 720
 721 # Generate a certificate fingerprint the same way keytool does it
 722 # (but with slightly different formatting)
 723 def cert_fingerprint(data):
 724     digest = hashlib.sha256(data).digest()
 725     ret = []
 726     ret.append(' '.join("%02X" % b for b in bytearray(digest)))
 727     return " ".join(ret)
 728
 729
 730 def extract_pubkey():
 731     global repo_pubkey_fingerprint
 732     if 'repo_pubkey' in config:
 733         pubkey = unhexlify(config['repo_pubkey'])
 734     else:
 735         p = FDroidPopenBytes([config['keytool'], '-exportcert',
 736                               '-alias', config['repo_keyalias'],
 737                               '-keystore', config['keystore'],
 738                               '-storepass:file', config['keystorepassfile']]
 739                              + config['smartcardoptions'],
 740                              output=False, stderr_to_stdout=False)
 741         if p.returncode != 0 or len(p.output) < 20:
 742             msg = "Failed to get repo pubkey!"
 743             if config['keystore'] == 'NONE':
 744                 msg += ' Is your crypto smartcard plugged in?'
 745             logging.critical(msg)
 746             sys.exit(1)
 747         pubkey = p.output
 748     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 749     return hexlify(pubkey)
 750
 751
 752 def make_index(apps, sortedids, apks, repodir, archive, categories):
 753     """Make a repo index.
 754
 755     :param apps: fully populated apps list
 756     :param apks: full populated apks list
 757     :param repodir: the repo directory
 758     :param archive: True if this is the archive repo, False if it's the
 759                     main one.
 760     :param categories: list of categories
 761     """
 762
 763     doc = Document()
 764
 765     def addElement(name, value, doc, parent):
 766         el = doc.createElement(name)
 767         el.appendChild(doc.createTextNode(value))
 768         parent.appendChild(el)
 769
 770     def addElementNonEmpty(name, value, doc, parent):
 771         if not value:
 772             return
 773         addElement(name, value, doc, parent)
 774
 775     def addElementCDATA(name, value, doc, parent):
 776         el = doc.createElement(name)
 777         el.appendChild(doc.createCDATASection(value))
 778         parent.appendChild(el)
 779
 780     root = doc.createElement("fdroid")
 781     doc.appendChild(root)
 782
 783     repoel = doc.createElement("repo")
 784
 785     mirrorcheckfailed = False
 786     for mirror in config.get('mirrors', []):
 787         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
 788         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
 789             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 790             mirrorcheckfailed = True
 791     if mirrorcheckfailed:
 792         sys.exit(1)
 793
 794     if archive:
 795         repoel.setAttribute("name", config['archive_name'])
 796         if config['repo_maxage'] != 0:
 797             repoel.setAttribute("maxage", str(config['repo_maxage']))
 798         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 799         repoel.setAttribute("url", config['archive_url'])
 800         addElement('description', config['archive_description'], doc, repoel)
 801         urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
 802         for mirror in config.get('mirrors', []):
 803             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 804
 805     else:
 806         repoel.setAttribute("name", config['repo_name'])
 807         if config['repo_maxage'] != 0:
 808             repoel.setAttribute("maxage", str(config['repo_maxage']))
 809         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 810         repoel.setAttribute("url", config['repo_url'])
 811         addElement('description', config['repo_description'], doc, repoel)
 812         urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
 813         for mirror in config.get('mirrors', []):
 814             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 815
 816     repoel.setAttribute("version", str(METADATA_VERSION))
 817     repoel.setAttribute("timestamp", str(int(time.time())))
 818
 819     nosigningkey = False
 820     if not options.nosign:
 821         if 'repo_keyalias' not in config:
 822             nosigningkey = True
 823             logging.critical("'repo_keyalias' not found in config.py!")
 824         if 'keystore' not in config:
 825             nosigningkey = True
 826             logging.critical("'keystore' not found in config.py!")
 827         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 828             nosigningkey = True
 829             logging.critical("'keystorepass' not found in config.py!")
 830         if 'keypass' not in config and 'keypassfile' not in config:
 831             nosigningkey = True
 832             logging.critical("'keypass' not found in config.py!")
 833         if not os.path.exists(config['keystore']):
 834             nosigningkey = True
 835             logging.critical("'" + config['keystore'] + "' does not exist!")
 836         if nosigningkey:
 837             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 838             logging.warning("\tfdroid update --create-key")
 839             sys.exit(1)
 840
 841     repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
 842     root.appendChild(repoel)
 843
 844     for appid in sortedids:
 845         app = apps[appid]
 846
 847         if app.Disabled is not None:
 848             continue
 849
 850         # Get a list of the apks for this app...
 851         apklist = []
 852         for apk in apks:
 853             if apk['id'] == appid:
 854                 apklist.append(apk)
 855
 856         if len(apklist) == 0:
 857             continue
 858
 859         apel = doc.createElement("application")
 860         apel.setAttribute("id", app.id)
 861         root.appendChild(apel)
 862
 863         addElement('id', app.id, doc, apel)
 864         if app.added:
 865             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
 866         if app.lastupdated:
 867             addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
 868         addElement('name', app.Name, doc, apel)
 869         addElement('summary', app.Summary, doc, apel)
 870         if app.icon:
 871             addElement('icon', app.icon, doc, apel)
 872
 873         def linkres(appid):
 874             if appid in apps:
 875                 return ("fdroid.app:" + appid, apps[appid].Name)
 876             raise MetaDataException("Cannot resolve app id " + appid)
 877
 878         addElement('desc',
 879                    metadata.description_html(app.Description, linkres),
 880                    doc, apel)
 881         addElement('license', app.License, doc, apel)
 882         if app.Categories:
 883             addElement('categories', ','.join(app.Categories), doc, apel)
 884             # We put the first (primary) category in LAST, which will have
 885             # the desired effect of making clients that only understand one
 886             # category see that one.
 887             addElement('category', app.Categories[0], doc, apel)
 888         addElement('web', app.WebSite, doc, apel)
 889         addElement('source', app.SourceCode, doc, apel)
 890         addElement('tracker', app.IssueTracker, doc, apel)
 891         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 892         addElementNonEmpty('author', app.AuthorName, doc, apel)
 893         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 894         addElementNonEmpty('donate', app.Donate, doc, apel)
 895         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 896         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 897         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 898
 899         # These elements actually refer to the current version (i.e. which
 900         # one is recommended. They are historically mis-named, and need
 901         # changing, but stay like this for now to support existing clients.
 902         addElement('marketversion', app.CurrentVersion, doc, apel)
 903         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 904
 905         if app.AntiFeatures:
 906             af = app.AntiFeatures
 907             if af:
 908                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 909         if app.Provides:
 910             pv = app.Provides.split(',')
 911             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 912         if app.RequiresRoot:
 913             addElement('requirements', 'root', doc, apel)
 914
 915         # Sort the apk list into version order, just so the web site
 916         # doesn't have to do any work by default...
 917         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 918
 919         # Check for duplicates - they will make the client unhappy...
 920         for i in range(len(apklist) - 1):
 921             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 922                 logging.critical("duplicate versions: '%s' - '%s'" % (
 923                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 924                 sys.exit(1)
 925
 926         current_version_code = 0
 927         current_version_file = None
 928         for apk in apklist:
 929             # find the APK for the "Current Version"
 930             if current_version_code < apk['versioncode']:
 931                 current_version_code = apk['versioncode']
 932             if current_version_code < int(app.CurrentVersionCode):
 933                 current_version_file = apk['apkname']
 934
 935             apkel = doc.createElement("package")
 936             apel.appendChild(apkel)
 937             addElement('version', apk['version'], doc, apkel)
 938             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 939             addElement('apkname', apk['apkname'], doc, apkel)
 940             if 'srcname' in apk:
 941                 addElement('srcname', apk['srcname'], doc, apkel)
 942             for hash_type in ['sha256']:
 943                 if hash_type not in apk:
 944                     continue
 945                 hashel = doc.createElement("hash")
 946                 hashel.setAttribute("type", hash_type)
 947                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 948                 apkel.appendChild(hashel)
 949             addElement('sig', apk['sig'], doc, apkel)
 950             addElement('size', str(apk['size']), doc, apkel)
 951             addElement('sdkver', str(apk['minSdkVersion']), doc, apkel)
 952             if 'targetSdkVersion' in apk:
 953                 addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
 954             if 'maxSdkVersion' in apk:
 955                 addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
 956             if 'added' in apk:
 957                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 958             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
 959             if 'nativecode' in apk:
 960                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 961             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
 962
 963         if current_version_file is not None \
 964                 and config['make_current_version_link'] \
 965                 and repodir == 'repo':  # only create these
 966             namefield = config['current_version_name_source']
 967             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
 968             apklinkname = sanitized_name + '.apk'
 969             current_version_path = os.path.join(repodir, current_version_file)
 970             if os.path.islink(apklinkname):
 971                 os.remove(apklinkname)
 972             os.symlink(current_version_path, apklinkname)
 973             # also symlink gpg signature, if it exists
 974             for extension in ('.asc', '.sig'):
 975                 sigfile_path = current_version_path + extension
 976                 if os.path.exists(sigfile_path):
 977                     siglinkname = apklinkname + extension
 978                     if os.path.islink(siglinkname):
 979                         os.remove(siglinkname)
 980                     os.symlink(sigfile_path, siglinkname)
 981
 982     if options.pretty:
 983         output = doc.toprettyxml(encoding='utf-8')
 984     else:
 985         output = doc.toxml(encoding='utf-8')
 986
 987     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 988         f.write(output)
 989
 990     if 'repo_keyalias' in config:
 991
 992         if options.nosign:
 993             logging.info("Creating unsigned index in preparation for signing")
 994         else:
 995             logging.info("Creating signed index with this key (SHA256):")
 996             logging.info("%s" % repo_pubkey_fingerprint)
 997
 998         # Create a jar of the index...
 999         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
1000         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
1001         if p.returncode != 0:
1002             logging.critical("Failed to create {0}".format(jar_output))
1003             sys.exit(1)
1004
1005         # Sign the index...
1006         signed = os.path.join(repodir, 'index.jar')
1007         if options.nosign:
1008             # Remove old signed index if not signing
1009             if os.path.exists(signed):
1010                 os.remove(signed)
1011         else:
1012             args = [config['jarsigner'], '-keystore', config['keystore'],
1013                     '-storepass:file', config['keystorepassfile'],
1014                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1015                     signed, config['repo_keyalias']]
1016             if config['keystore'] == 'NONE':
1017                 args += config['smartcardoptions']
1018             else:  # smardcards never use -keypass
1019                 args += ['-keypass:file', config['keypassfile']]
1020             p = FDroidPopen(args)
1021             if p.returncode != 0:
1022                 logging.critical("Failed to sign index")
1023                 sys.exit(1)
1024
1025     # Copy the repo icon into the repo directory...
1026     icon_dir = os.path.join(repodir, 'icons')
1027     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1028     shutil.copyfile(config['repo_icon'], iconfilename)
1029
1030     # Write a category list in the repo to allow quick access...
1031     catdata = ''
1032     for cat in categories:
1033         catdata += cat + '\n'
1034     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1035         f.write(catdata)
1036
1037
1038 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1039
1040     for appid, app in apps.items():
1041
1042         if app.ArchivePolicy:
1043             keepversions = int(app.ArchivePolicy[:-9])
1044         else:
1045             keepversions = defaultkeepversions
1046
1047         def filter_apk_list_sorted(apk_list):
1048             res = []
1049             for apk in apk_list:
1050                 if apk['id'] == appid:
1051                     res.append(apk)
1052
1053             # Sort the apk list by version code. First is highest/newest.
1054             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1055
1056         def move_file(from_dir, to_dir, filename, ignore_missing):
1057             from_path = os.path.join(from_dir, filename)
1058             if ignore_missing and not os.path.exists(from_path):
1059                 return
1060             to_path = os.path.join(to_dir, filename)
1061             shutil.move(from_path, to_path)
1062
1063         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1064                       .format(appid, len(apks), keepversions, len(archapks)))
1065
1066         if len(apks) > keepversions:
1067             apklist = filter_apk_list_sorted(apks)
1068             # Move back the ones we don't want.
1069             for apk in apklist[keepversions:]:
1070                 logging.info("Moving " + apk['apkname'] + " to archive")
1071                 move_file(repodir, archivedir, apk['apkname'], False)
1072                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1073                 for density in all_screen_densities:
1074                     repo_icon_dir = get_icon_dir(repodir, density)
1075                     archive_icon_dir = get_icon_dir(archivedir, density)
1076                     if density not in apk['icons']:
1077                         continue
1078                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1079                 if 'srcname' in apk:
1080                     move_file(repodir, archivedir, apk['srcname'], False)
1081                 archapks.append(apk)
1082                 apks.remove(apk)
1083         elif len(apks) < keepversions and len(archapks) > 0:
1084             required = keepversions - len(apks)
1085             archapklist = filter_apk_list_sorted(archapks)
1086             # Move forward the ones we want again.
1087             for apk in archapklist[:required]:
1088                 logging.info("Moving " + apk['apkname'] + " from archive")
1089                 move_file(archivedir, repodir, apk['apkname'], False)
1090                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1091                 for density in all_screen_densities:
1092                     repo_icon_dir = get_icon_dir(repodir, density)
1093                     archive_icon_dir = get_icon_dir(archivedir, density)
1094                     if density not in apk['icons']:
1095                         continue
1096                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1097                 if 'srcname' in apk:
1098                     move_file(archivedir, repodir, apk['srcname'], False)
1099                 archapks.remove(apk)
1100                 apks.append(apk)
1101
1102
1103 def add_apks_to_per_app_repos(repodir, apks):
1104     apks_per_app = dict()
1105     for apk in apks:
1106         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1107         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1108         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1109         apks_per_app[apk['id']] = apk
1110
1111         if not os.path.exists(apk['per_app_icons']):
1112             logging.info('Adding new repo for only ' + apk['id'])
1113             os.makedirs(apk['per_app_icons'])
1114
1115         apkpath = os.path.join(repodir, apk['apkname'])
1116         shutil.copy(apkpath, apk['per_app_repo'])
1117         apksigpath = apkpath + '.sig'
1118         if os.path.exists(apksigpath):
1119             shutil.copy(apksigpath, apk['per_app_repo'])
1120         apkascpath = apkpath + '.asc'
1121         if os.path.exists(apkascpath):
1122             shutil.copy(apkascpath, apk['per_app_repo'])
1123
1124
1125 config = None
1126 options = None
1127
1128
1129 def main():
1130
1131     global config, options
1132
1133     # Parse command line...
1134     parser = ArgumentParser()
1135     common.setup_global_opts(parser)
1136     parser.add_argument("--create-key", action="store_true", default=False,
1137                         help="Create a repo signing key in a keystore")
1138     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1139                         help="Create skeleton metadata files that are missing")
1140     parser.add_argument("--delete-unknown", action="store_true", default=False,
1141                         help="Delete APKs without metadata from the repo")
1142     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1143                         help="Report on build data status")
1144     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1145                         help="Interactively ask about things that need updating.")
1146     parser.add_argument("-I", "--icons", action="store_true", default=False,
1147                         help="Resize all the icons exceeding the max pixel size and exit")
1148     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1149                         help="Specify editor to use in interactive mode. Default " +
1150                         "is /etc/alternatives/editor")
1151     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1152                         help="Update the wiki")
1153     parser.add_argument("--pretty", action="store_true", default=False,
1154                         help="Produce human-readable index.xml")
1155     parser.add_argument("--clean", action="store_true", default=False,
1156                         help="Clean update - don't uses caches, reprocess all apks")
1157     parser.add_argument("--nosign", action="store_true", default=False,
1158                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1159     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1160                         help="Use date from apk instead of current time for newly added apks")
1161     options = parser.parse_args()
1162
1163     config = common.read_config(options)
1164
1165     if not ('jarsigner' in config and 'keytool' in config):
1166         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1167         sys.exit(1)
1168
1169     repodirs = ['repo']
1170     if config['archive_older'] != 0:
1171         repodirs.append('archive')
1172         if not os.path.exists('archive'):
1173             os.mkdir('archive')
1174
1175     if options.icons:
1176         resize_all_icons(repodirs)
1177         sys.exit(0)
1178
1179     # check that icons exist now, rather than fail at the end of `fdroid update`
1180     for k in ['repo_icon', 'archive_icon']:
1181         if k in config:
1182             if not os.path.exists(config[k]):
1183                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1184                 sys.exit(1)
1185
1186     # if the user asks to create a keystore, do it now, reusing whatever it can
1187     if options.create_key:
1188         if os.path.exists(config['keystore']):
1189             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1190             logging.critical("\t'" + config['keystore'] + "'")
1191             sys.exit(1)
1192
1193         if 'repo_keyalias' not in config:
1194             config['repo_keyalias'] = socket.getfqdn()
1195             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1196         if 'keydname' not in config:
1197             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1198             common.write_to_config(config, 'keydname', config['keydname'])
1199         if 'keystore' not in config:
1200             config['keystore'] = common.default_config.keystore
1201             common.write_to_config(config, 'keystore', config['keystore'])
1202
1203         password = common.genpassword()
1204         if 'keystorepass' not in config:
1205             config['keystorepass'] = password
1206             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1207         if 'keypass' not in config:
1208             config['keypass'] = password
1209             common.write_to_config(config, 'keypass', config['keypass'])
1210         common.genkeystore(config)
1211
1212     # Get all apps...
1213     apps = metadata.read_metadata()
1214
1215     # Generate a list of categories...
1216     categories = set()
1217     for app in apps.values():
1218         categories.update(app.Categories)
1219
1220     # Read known apks data (will be updated and written back when we've finished)
1221     knownapks = common.KnownApks()
1222
1223     # Gather information about all the apk files in the repo directory, using
1224     # cached data if possible.
1225     apkcachefile = os.path.join('tmp', 'apkcache')
1226     if not options.clean and os.path.exists(apkcachefile):
1227         with open(apkcachefile, 'rb') as cf:
1228             apkcache = pickle.load(cf, encoding='utf-8')
1229         if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
1230             apkcache = {}
1231     else:
1232         apkcache = {}
1233
1234     delete_disabled_builds(apps, apkcache, repodirs)
1235
1236     # Scan all apks in the main repo
1237     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1238
1239     # Generate warnings for apk's with no metadata (or create skeleton
1240     # metadata files, if requested on the command line)
1241     newmetadata = False
1242     for apk in apks:
1243         if apk['id'] not in apps:
1244             if options.create_metadata:
1245                 if 'name' not in apk:
1246                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1247                     continue
1248                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
1249                 f.write("License:Unknown\n")
1250                 f.write("Web Site:\n")
1251                 f.write("Source Code:\n")
1252                 f.write("Issue Tracker:\n")
1253                 f.write("Changelog:\n")
1254                 f.write("Summary:" + apk['name'] + "\n")
1255                 f.write("Description:\n")
1256                 f.write(apk['name'] + "\n")
1257                 f.write(".\n")
1258                 f.close()
1259                 logging.info("Generated skeleton metadata for " + apk['id'])
1260                 newmetadata = True
1261             else:
1262                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1263                 if options.delete_unknown:
1264                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1265                     rmf = os.path.join(repodirs[0], apk['apkname'])
1266                     if not os.path.exists(rmf):
1267                         logging.error("Could not find {0} to remove it".format(rmf))
1268                     else:
1269                         os.remove(rmf)
1270                 else:
1271                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1272
1273     # update the metadata with the newly created ones included
1274     if newmetadata:
1275         apps = metadata.read_metadata()
1276
1277     # Scan the archive repo for apks as well
1278     if len(repodirs) > 1:
1279         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1280         if cc:
1281             cachechanged = True
1282     else:
1283         archapks = []
1284
1285     # Some information from the apks needs to be applied up to the application
1286     # level. When doing this, we use the info from the most recent version's apk.
1287     # We deal with figuring out when the app was added and last updated at the
1288     # same time.
1289     for appid, app in apps.items():
1290         bestver = 0
1291         for apk in apks + archapks:
1292             if apk['id'] == appid:
1293                 if apk['versioncode'] > bestver:
1294                     bestver = apk['versioncode']
1295                     bestapk = apk
1296
1297                 if 'added' in apk:
1298                     if not app.added or apk['added'] < app.added:
1299                         app.added = apk['added']
1300                     if not app.lastupdated or apk['added'] > app.lastupdated:
1301                         app.lastupdated = apk['added']
1302
1303         if not app.added:
1304             logging.debug("Don't know when " + appid + " was added")
1305         if not app.lastupdated:
1306             logging.debug("Don't know when " + appid + " was last updated")
1307
1308         if bestver == 0:
1309             if app.Name is None:
1310                 app.Name = app.AutoName or appid
1311             app.icon = None
1312             logging.debug("Application " + appid + " has no packages")
1313         else:
1314             if app.Name is None:
1315                 app.Name = bestapk['name']
1316             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1317             if app.CurrentVersionCode is None:
1318                 app.CurrentVersionCode = str(bestver)
1319
1320     # Sort the app list by name, then the web site doesn't have to by default.
1321     # (we had to wait until we'd scanned the apks to do this, because mostly the
1322     # name comes from there!)
1323     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1324
1325     # APKs are placed into multiple repos based on the app package, providing
1326     # per-app subscription feeds for nightly builds and things like it
1327     if config['per_app_repos']:
1328         add_apks_to_per_app_repos(repodirs[0], apks)
1329         for appid, app in apps.items():
1330             repodir = os.path.join(appid, 'fdroid', 'repo')
1331             appdict = dict()
1332             appdict[appid] = app
1333             if os.path.isdir(repodir):
1334                 make_index(appdict, [appid], apks, repodir, False, categories)
1335             else:
1336                 logging.info('Skipping index generation for ' + appid)
1337         return
1338
1339     if len(repodirs) > 1:
1340         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1341
1342     # Make the index for the main repo...
1343     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1344
1345     # If there's an archive repo,  make the index for it. We already scanned it
1346     # earlier on.
1347     if len(repodirs) > 1:
1348         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1349
1350     if config['update_stats']:
1351
1352         # Update known apks info...
1353         knownapks.writeifchanged()
1354
1355         # Generate latest apps data for widget
1356         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1357             data = ''
1358             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1359                 for line in f:
1360                     appid = line.rstrip()
1361                     data += appid + "\t"
1362                     app = apps[appid]
1363                     data += app.Name + "\t"
1364                     if app.icon is not None:
1365                         data += app.icon + "\t"
1366                     data += app.License + "\n"
1367             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1368                 f.write(data)
1369
1370     if cachechanged:
1371         apkcache["METADATA_VERSION"] = METADATA_VERSION
1372         with open(apkcachefile, 'wb') as cf:
1373             pickle.dump(apkcache, cf)
1374
1375     # Update the wiki...
1376     if options.wiki:
1377         update_wiki(apps, sortedids, apks + archapks)
1378
1379     logging.info("Finished.")
1380
1381 if __name__ == "__main__":
1382     main()