fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   5 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU Affero General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Affero General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Affero General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 import sys
  21 import os
  22 import shutil
  23 import glob
  24 import re
  25 import socket
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 import urllib.parse
  30 from datetime import datetime, timedelta
  31 from xml.dom.minidom import Document
  32 from argparse import ArgumentParser
  33 import time
  34 from pyasn1.error import PyAsn1Error
  35 from pyasn1.codec.der import decoder, encoder
  36 from pyasn1_modules import rfc2315
  37 from binascii import hexlify, unhexlify
  38
  39 from PIL import Image
  40 import logging
  41
  42 from . import common
  43 from . import metadata
  44 from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
  45 from .metadata import MetaDataException
  46
  47 METADATA_VERSION = 16
  48
  49 screen_densities = ['640', '480', '320', '240', '160', '120']
  50
  51 all_screen_densities = ['0'] + screen_densities
  52
  53
  54 def dpi_to_px(density):
  55     return (int(density) * 48) / 160
  56
  57
  58 def px_to_dpi(px):
  59     return (int(px) * 160) / 48
  60
  61
  62 def get_icon_dir(repodir, density):
  63     if density == '0':
  64         return os.path.join(repodir, "icons")
  65     return os.path.join(repodir, "icons-%s" % density)
  66
  67
  68 def get_icon_dirs(repodir):
  69     for density in screen_densities:
  70         yield get_icon_dir(repodir, density)
  71
  72
  73 def get_all_icon_dirs(repodir):
  74     for density in all_screen_densities:
  75         yield get_icon_dir(repodir, density)
  76
  77
  78 def update_wiki(apps, sortedids, apks):
  79     """Update the wiki
  80
  81     :param apps: fully populated list of all applications
  82     :param apks: all apks, except...
  83     """
  84     logging.info("Updating wiki")
  85     wikicat = 'Apps'
  86     wikiredircat = 'App Redirects'
  87     import mwclient
  88     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  89                          path=config['wiki_path'])
  90     site.login(config['wiki_user'], config['wiki_password'])
  91     generated_pages = {}
  92     generated_redirects = {}
  93
  94     for appid in sortedids:
  95         app = apps[appid]
  96
  97         wikidata = ''
  98         if app.Disabled:
  99             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 100         if app.AntiFeatures:
 101             for af in app.AntiFeatures:
 102                 wikidata += '{{AntiFeature|' + af + '}}\n'
 103         if app.RequiresRoot:
 104             requiresroot = 'Yes'
 105         else:
 106             requiresroot = 'No'
 107         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 108             appid,
 109             app.Name,
 110             time.strftime('%Y-%m-%d', app.added) if app.added else '',
 111             time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
 112             app.SourceCode,
 113             app.IssueTracker,
 114             app.WebSite,
 115             app.Changelog,
 116             app.Donate,
 117             app.FlattrID,
 118             app.Bitcoin,
 119             app.Litecoin,
 120             app.License,
 121             requiresroot,
 122             app.AuthorName,
 123             app.AuthorEmail)
 124
 125         if app.Provides:
 126             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 127
 128         wikidata += app.Summary
 129         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 130
 131         wikidata += "=Description=\n"
 132         wikidata += metadata.description_wiki(app.Description) + "\n"
 133
 134         wikidata += "=Maintainer Notes=\n"
 135         if app.MaintainerNotes:
 136             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 137         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 138
 139         # Get a list of all packages for this application...
 140         apklist = []
 141         gotcurrentver = False
 142         cantupdate = False
 143         buildfails = False
 144         for apk in apks:
 145             if apk['id'] == appid:
 146                 if str(apk['versioncode']) == app.CurrentVersionCode:
 147                     gotcurrentver = True
 148                 apklist.append(apk)
 149         # Include ones we can't build, as a special case...
 150         for build in app.builds:
 151             if build.disable:
 152                 if build.vercode == app.CurrentVersionCode:
 153                     cantupdate = True
 154                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 155                 apklist.append({'versioncode': int(build.vercode),
 156                                 'version': build.version,
 157                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 158                                 })
 159             else:
 160                 builtit = False
 161                 for apk in apklist:
 162                     if apk['versioncode'] == int(build.vercode):
 163                         builtit = True
 164                         break
 165                 if not builtit:
 166                     buildfails = True
 167                     apklist.append({'versioncode': int(build.vercode),
 168                                     'version': build.version,
 169                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
 170                                     })
 171         if app.CurrentVersionCode == '0':
 172             cantupdate = True
 173         # Sort with most recent first...
 174         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 175
 176         wikidata += "=Versions=\n"
 177         if len(apklist) == 0:
 178             wikidata += "We currently have no versions of this app available."
 179         elif not gotcurrentver:
 180             wikidata += "We don't have the current version of this app."
 181         else:
 182             wikidata += "We have the current version of this app."
 183         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 184         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 185         if len(app.NoSourceSince) > 0:
 186             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 187         if len(app.CurrentVersion) > 0:
 188             wikidata += "The current (recommended) version is " + app.CurrentVersion
 189             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 190         validapks = 0
 191         for apk in apklist:
 192             wikidata += "==" + apk['version'] + "==\n"
 193
 194             if 'buildproblem' in apk:
 195                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 196             else:
 197                 validapks += 1
 198                 wikidata += "This version is built and signed by "
 199                 if 'srcname' in apk:
 200                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 201                 else:
 202                     wikidata += "the original developer.\n\n"
 203             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 204
 205         wikidata += '\n[[Category:' + wikicat + ']]\n'
 206         if len(app.NoSourceSince) > 0:
 207             wikidata += '\n[[Category:Apps missing source code]]\n'
 208         if validapks == 0 and not app.Disabled:
 209             wikidata += '\n[[Category:Apps with no packages]]\n'
 210         if cantupdate and not app.Disabled:
 211             wikidata += "\n[[Category:Apps we cannot update]]\n"
 212         if buildfails and not app.Disabled:
 213             wikidata += "\n[[Category:Apps with failing builds]]\n"
 214         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 215             wikidata += '\n[[Category:Apps to Update]]\n'
 216         if app.Disabled:
 217             wikidata += '\n[[Category:Apps that are disabled]]\n'
 218         if app.UpdateCheckMode == 'None' and not app.Disabled:
 219             wikidata += '\n[[Category:Apps with no update check]]\n'
 220         for appcat in app.Categories:
 221             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 222
 223         # We can't have underscores in the page name, even if they're in
 224         # the package ID, because MediaWiki messes with them...
 225         pagename = appid.replace('_', ' ')
 226
 227         # Drop a trailing newline, because mediawiki is going to drop it anyway
 228         # and it we don't we'll think the page has changed when it hasn't...
 229         if wikidata.endswith('\n'):
 230             wikidata = wikidata[:-1]
 231
 232         generated_pages[pagename] = wikidata
 233
 234         # Make a redirect from the name to the ID too, unless there's
 235         # already an existing page with the name and it isn't a redirect.
 236         noclobber = False
 237         apppagename = app.Name.replace('_', ' ')
 238         apppagename = apppagename.replace('{', '')
 239         apppagename = apppagename.replace('}', ' ')
 240         apppagename = apppagename.replace(':', ' ')
 241         # Drop double spaces caused mostly by replacing ':' above
 242         apppagename = apppagename.replace('  ', ' ')
 243         for expagename in site.allpages(prefix=apppagename,
 244                                         filterredir='nonredirects',
 245                                         generator=False):
 246             if expagename == apppagename:
 247                 noclobber = True
 248         # Another reason not to make the redirect page is if the app name
 249         # is the same as it's ID, because that will overwrite the real page
 250         # with an redirect to itself! (Although it seems like an odd
 251         # scenario this happens a lot, e.g. where there is metadata but no
 252         # builds or binaries to extract a name from.
 253         if apppagename == pagename:
 254             noclobber = True
 255         if not noclobber:
 256             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 257
 258     for tcat, genp in [(wikicat, generated_pages),
 259                        (wikiredircat, generated_redirects)]:
 260         catpages = site.Pages['Category:' + tcat]
 261         existingpages = []
 262         for page in catpages:
 263             existingpages.append(page.name)
 264             if page.name in genp:
 265                 pagetxt = page.edit()
 266                 if pagetxt != genp[page.name]:
 267                     logging.debug("Updating modified page " + page.name)
 268                     page.save(genp[page.name], summary='Auto-updated')
 269                 else:
 270                     logging.debug("Page " + page.name + " is unchanged")
 271             else:
 272                 logging.warn("Deleting page " + page.name)
 273                 page.delete('No longer published')
 274         for pagename, text in genp.items():
 275             logging.debug("Checking " + pagename)
 276             if pagename not in existingpages:
 277                 logging.debug("Creating page " + pagename)
 278                 try:
 279                     newpage = site.Pages[pagename]
 280                     newpage.save(text, summary='Auto-created')
 281                 except:
 282                     logging.error("...FAILED to create page '{0}'".format(pagename))
 283
 284     # Purge server cache to ensure counts are up to date
 285     site.pages['Repository Maintenance'].purge()
 286
 287
 288 def delete_disabled_builds(apps, apkcache, repodirs):
 289     """Delete disabled build outputs.
 290
 291     :param apps: list of all applications, as per metadata.read_metadata
 292     :param apkcache: current apk cache information
 293     :param repodirs: the repo directories to process
 294     """
 295     for appid, app in apps.items():
 296         for build in app.builds:
 297             if not build.disable:
 298                 continue
 299             apkfilename = appid + '_' + str(build.vercode) + '.apk'
 300             iconfilename = "%s.%s.png" % (
 301                 appid,
 302                 build.vercode)
 303             for repodir in repodirs:
 304                 files = [
 305                     os.path.join(repodir, apkfilename),
 306                     os.path.join(repodir, apkfilename + '.asc'),
 307                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 308                 ]
 309                 for density in all_screen_densities:
 310                     repo_dir = get_icon_dir(repodir, density)
 311                     files.append(os.path.join(repo_dir, iconfilename))
 312
 313                 for f in files:
 314                     if os.path.exists(f):
 315                         logging.info("Deleting disabled build output " + f)
 316                         os.remove(f)
 317             if apkfilename in apkcache:
 318                 del apkcache[apkfilename]
 319
 320
 321 def resize_icon(iconpath, density):
 322
 323     if not os.path.isfile(iconpath):
 324         return
 325
 326     try:
 327         im = Image.open(iconpath)
 328         size = dpi_to_px(density)
 329
 330         if any(length > size for length in im.size):
 331             oldsize = im.size
 332             im.thumbnail((size, size), Image.ANTIALIAS)
 333             logging.debug("%s was too large at %s - new size is %s" % (
 334                 iconpath, oldsize, im.size))
 335             im.save(iconpath, "PNG")
 336
 337     except Exception as e:
 338         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 339
 340
 341 def resize_all_icons(repodirs):
 342     """Resize all icons that exceed the max size
 343
 344     :param repodirs: the repo directories to process
 345     """
 346     for repodir in repodirs:
 347         for density in screen_densities:
 348             icon_dir = get_icon_dir(repodir, density)
 349             icon_glob = os.path.join(icon_dir, '*.png')
 350             for iconpath in glob.glob(icon_glob):
 351                 resize_icon(iconpath, density)
 352
 353
 354 # A signature block file with a .DSA, .RSA, or .EC extension
 355 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 356
 357
 358 def getsig(apkpath):
 359     """ Get the signing certificate of an apk. To get the same md5 has that
 360     Android gets, we encode the .RSA certificate in a specific format and pass
 361     it hex-encoded to the md5 digest algorithm.
 362
 363     :param apkpath: path to the apk
 364     :returns: A string containing the md5 of the signature of the apk or None
 365               if an error occurred.
 366     """
 367
 368     cert = None
 369
 370     # verify the jar signature is correct
 371     args = [config['jarsigner'], '-verify', apkpath]
 372     p = FDroidPopen(args)
 373     if p.returncode != 0:
 374         logging.critical(apkpath + " has a bad signature!")
 375         return None
 376
 377     with zipfile.ZipFile(apkpath, 'r') as apk:
 378
 379         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 380
 381         if len(certs) < 1:
 382             logging.error("Found no signing certificates on %s" % apkpath)
 383             return None
 384         if len(certs) > 1:
 385             logging.error("Found multiple signing certificates on %s" % apkpath)
 386             return None
 387
 388         cert = apk.read(certs[0])
 389
 390     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 391     if content.getComponentByName('contentType') != rfc2315.signedData:
 392         logging.error("Unexpected format.")
 393         return None
 394
 395     content = decoder.decode(content.getComponentByName('content'),
 396                              asn1Spec=rfc2315.SignedData())[0]
 397     try:
 398         certificates = content.getComponentByName('certificates')
 399     except PyAsn1Error:
 400         logging.error("Certificates not found.")
 401         return None
 402
 403     cert_encoded = encoder.encode(certificates)[4:]
 404
 405     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 406
 407
 408 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
 409     """Scan the apks in the given repo directory.
 410
 411     This also extracts the icons.
 412
 413     :param apps: list of all applications, as per metadata.read_metadata
 414     :param apkcache: current apk cache information
 415     :param repodir: repo directory to scan
 416     :param knownapks: known apks info
 417     :param use_date_from_apk: use date from APK (instead of current date)
 418                               for newly added APKs
 419     :returns: (apks, cachechanged) where apks is a list of apk information,
 420               and cachechanged is True if the apkcache got changed.
 421     """
 422
 423     cachechanged = False
 424
 425     for icon_dir in get_all_icon_dirs(repodir):
 426         if os.path.exists(icon_dir):
 427             if options.clean:
 428                 shutil.rmtree(icon_dir)
 429                 os.makedirs(icon_dir)
 430         else:
 431             os.makedirs(icon_dir)
 432
 433     apks = []
 434     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 435     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 436     vername_pat = re.compile(".*versionName='([^']*)'.*")
 437     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 438     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 439     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 440     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 441     string_pat = re.compile(".*'([^']*)'.*")
 442     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 443
 444         apkfilename = apkfile[len(repodir) + 1:]
 445         if ' ' in apkfilename:
 446             logging.critical("Spaces in filenames are not allowed.")
 447             sys.exit(1)
 448
 449         # Calculate the sha256...
 450         sha = hashlib.sha256()
 451         with open(apkfile, 'rb') as f:
 452             while True:
 453                 t = f.read(16384)
 454                 if len(t) == 0:
 455                     break
 456                 sha.update(t)
 457             shasum = sha.hexdigest()
 458
 459         usecache = False
 460         if apkfilename in apkcache:
 461             apk = apkcache[apkfilename]
 462             if apk['sha256'] == shasum:
 463                 logging.debug("Reading " + apkfilename + " from cache")
 464                 usecache = True
 465             else:
 466                 logging.debug("Ignoring stale cache data for " + apkfilename)
 467
 468         if not usecache:
 469             logging.debug("Processing " + apkfilename)
 470             apk = {}
 471             apk['apkname'] = apkfilename
 472             apk['sha256'] = shasum
 473             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 474             if os.path.exists(os.path.join(repodir, srcfilename)):
 475                 apk['srcname'] = srcfilename
 476             apk['size'] = os.path.getsize(apkfile)
 477             apk['permissions'] = set()
 478             apk['features'] = set()
 479             apk['icons_src'] = {}
 480             apk['icons'] = {}
 481             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 482             if p.returncode != 0:
 483                 if options.delete_unknown:
 484                     if os.path.exists(apkfile):
 485                         logging.error("Failed to get apk information, deleting " + apkfile)
 486                         os.remove(apkfile)
 487                     else:
 488                         logging.error("Could not find {0} to remove it".format(apkfile))
 489                 else:
 490                     logging.error("Failed to get apk information, skipping " + apkfile)
 491                 continue
 492             for line in p.output.splitlines():
 493                 if line.startswith("package:"):
 494                     try:
 495                         apk['id'] = re.match(name_pat, line).group(1)
 496                         apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
 497                         apk['version'] = re.match(vername_pat, line).group(1)
 498                     except Exception as e:
 499                         logging.error("Package matching failed: " + str(e))
 500                         logging.info("Line was: " + line)
 501                         sys.exit(1)
 502                 elif line.startswith("application:"):
 503                     apk['name'] = re.match(label_pat, line).group(1)
 504                     # Keep path to non-dpi icon in case we need it
 505                     match = re.match(icon_pat_nodpi, line)
 506                     if match:
 507                         apk['icons_src']['-1'] = match.group(1)
 508                 elif line.startswith("launchable-activity:"):
 509                     # Only use launchable-activity as fallback to application
 510                     if not apk['name']:
 511                         apk['name'] = re.match(label_pat, line).group(1)
 512                     if '-1' not in apk['icons_src']:
 513                         match = re.match(icon_pat_nodpi, line)
 514                         if match:
 515                             apk['icons_src']['-1'] = match.group(1)
 516                 elif line.startswith("application-icon-"):
 517                     match = re.match(icon_pat, line)
 518                     if match:
 519                         density = match.group(1)
 520                         path = match.group(2)
 521                         apk['icons_src'][density] = path
 522                 elif line.startswith("sdkVersion:"):
 523                     m = re.match(sdkversion_pat, line)
 524                     if m is None:
 525                         logging.error(line.replace('sdkVersion:', '')
 526                                       + ' is not a valid minSdkVersion!')
 527                     else:
 528                         apk['minSdkVersion'] = m.group(1)
 529                         # if target not set, default to min
 530                         if 'targetSdkVersion' not in apk:
 531                             apk['targetSdkVersion'] = m.group(1)
 532                 elif line.startswith("targetSdkVersion:"):
 533                     m = re.match(sdkversion_pat, line)
 534                     if m is None:
 535                         logging.error(line.replace('targetSdkVersion:', '')
 536                                       + ' is not a valid targetSdkVersion!')
 537                     else:
 538                         apk['targetSdkVersion'] = m.group(1)
 539                 elif line.startswith("maxSdkVersion:"):
 540                     apk['maxSdkVersion'] = re.match(sdkversion_pat, line).group(1)
 541                 elif line.startswith("native-code:"):
 542                     apk['nativecode'] = []
 543                     for arch in line[13:].split(' '):
 544                         apk['nativecode'].append(arch[1:-1])
 545                 elif line.startswith("uses-permission:"):
 546                     perm = re.match(string_pat, line).group(1)
 547                     if perm.startswith("android.permission."):
 548                         perm = perm[19:]
 549                     apk['permissions'].add(perm)
 550                 elif line.startswith("uses-feature:"):
 551                     perm = re.match(string_pat, line).group(1)
 552                     # Filter out this, it's only added with the latest SDK tools and
 553                     # causes problems for lots of apps.
 554                     if perm != "android.hardware.screen.portrait" \
 555                             and perm != "android.hardware.screen.landscape":
 556                         if perm.startswith("android.feature."):
 557                             perm = perm[16:]
 558                         apk['features'].add(perm)
 559
 560             if 'minSdkVersion' not in apk:
 561                 logging.warn("No SDK version information found in {0}".format(apkfile))
 562                 apk['minSdkVersion'] = 1
 563
 564             # Check for debuggable apks...
 565             if common.isApkDebuggable(apkfile, config):
 566                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 567
 568             # Get the signature (or md5 of, to be precise)...
 569             logging.debug('Getting signature of {0}'.format(apkfile))
 570             apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 571             if not apk['sig']:
 572                 logging.critical("Failed to get apk signature")
 573                 sys.exit(1)
 574
 575             apkzip = zipfile.ZipFile(apkfile, 'r')
 576
 577             # if an APK has files newer than the system time, suggest updating
 578             # the system clock.  This is useful for offline systems, used for
 579             # signing, which do not have another source of clock sync info. It
 580             # has to be more than 24 hours newer because ZIP/APK files do not
 581             # store timezone info
 582             manifest = apkzip.getinfo('AndroidManifest.xml')
 583             if manifest.date_time[1] == 0:  # month can't be zero
 584                 logging.debug('AndroidManifest.xml has no date')
 585             else:
 586                 dt_obj = datetime(*manifest.date_time)
 587                 checkdt = dt_obj - timedelta(1)
 588                 if datetime.today() < checkdt:
 589                     logging.warn('System clock is older than manifest in: '
 590                                  + apkfilename
 591                                  + '\nSet clock to that time using:\n'
 592                                  + 'sudo date -s "' + str(dt_obj) + '"')
 593
 594             iconfilename = "%s.%s.png" % (
 595                 apk['id'],
 596                 apk['versioncode'])
 597
 598             # Extract the icon file...
 599             empty_densities = []
 600             for density in screen_densities:
 601                 if density not in apk['icons_src']:
 602                     empty_densities.append(density)
 603                     continue
 604                 iconsrc = apk['icons_src'][density]
 605                 icon_dir = get_icon_dir(repodir, density)
 606                 icondest = os.path.join(icon_dir, iconfilename)
 607
 608                 try:
 609                     with open(icondest, 'wb') as f:
 610                         f.write(apkzip.read(iconsrc))
 611                     apk['icons'][density] = iconfilename
 612
 613                 except:
 614                     logging.warn("Error retrieving icon file")
 615                     del apk['icons'][density]
 616                     del apk['icons_src'][density]
 617                     empty_densities.append(density)
 618
 619             if '-1' in apk['icons_src']:
 620                 iconsrc = apk['icons_src']['-1']
 621                 iconpath = os.path.join(
 622                     get_icon_dir(repodir, '0'), iconfilename)
 623                 with open(iconpath, 'wb') as f:
 624                     f.write(apkzip.read(iconsrc))
 625                 try:
 626                     im = Image.open(iconpath)
 627                     dpi = px_to_dpi(im.size[0])
 628                     for density in screen_densities:
 629                         if density in apk['icons']:
 630                             break
 631                         if density == screen_densities[-1] or dpi >= int(density):
 632                             apk['icons'][density] = iconfilename
 633                             shutil.move(iconpath,
 634                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 635                             empty_densities.remove(density)
 636                             break
 637                 except Exception as e:
 638                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 639
 640             if apk['icons']:
 641                 apk['icon'] = iconfilename
 642
 643             apkzip.close()
 644
 645             # First try resizing down to not lose quality
 646             last_density = None
 647             for density in screen_densities:
 648                 if density not in empty_densities:
 649                     last_density = density
 650                     continue
 651                 if last_density is None:
 652                     continue
 653                 logging.debug("Density %s not available, resizing down from %s"
 654                               % (density, last_density))
 655
 656                 last_iconpath = os.path.join(
 657                     get_icon_dir(repodir, last_density), iconfilename)
 658                 iconpath = os.path.join(
 659                     get_icon_dir(repodir, density), iconfilename)
 660                 try:
 661                     im = Image.open(last_iconpath)
 662                 except:
 663                     logging.warn("Invalid image file at %s" % last_iconpath)
 664                     continue
 665
 666                 size = dpi_to_px(density)
 667
 668                 im.thumbnail((size, size), Image.ANTIALIAS)
 669                 im.save(iconpath, "PNG")
 670                 empty_densities.remove(density)
 671
 672             # Then just copy from the highest resolution available
 673             last_density = None
 674             for density in reversed(screen_densities):
 675                 if density not in empty_densities:
 676                     last_density = density
 677                     continue
 678                 if last_density is None:
 679                     continue
 680                 logging.debug("Density %s not available, copying from lower density %s"
 681                               % (density, last_density))
 682
 683                 shutil.copyfile(
 684                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 685                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 686
 687                 empty_densities.remove(density)
 688
 689             for density in screen_densities:
 690                 icon_dir = get_icon_dir(repodir, density)
 691                 icondest = os.path.join(icon_dir, iconfilename)
 692                 resize_icon(icondest, density)
 693
 694             # Copy from icons-mdpi to icons since mdpi is the baseline density
 695             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 696             if os.path.isfile(baseline):
 697                 apk['icons']['0'] = iconfilename
 698                 shutil.copyfile(baseline,
 699                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 700
 701             if use_date_from_apk and manifest.date_time[1] != 0:
 702                 default_date_param = datetime(*manifest.date_time).utctimetuple()
 703             else:
 704                 default_date_param = None
 705
 706             # Record in known apks, getting the added date at the same time..
 707             added = knownapks.recordapk(apk['apkname'], apk['id'], default_date=default_date_param)
 708             if added:
 709                 apk['added'] = added
 710
 711             apkcache[apkfilename] = apk
 712             cachechanged = True
 713
 714         apks.append(apk)
 715
 716     return apks, cachechanged
 717
 718
 719 repo_pubkey_fingerprint = None
 720
 721
 722 # Generate a certificate fingerprint the same way keytool does it
 723 # (but with slightly different formatting)
 724 def cert_fingerprint(data):
 725     digest = hashlib.sha256(data).digest()
 726     ret = []
 727     ret.append(' '.join("%02X" % b for b in bytearray(digest)))
 728     return " ".join(ret)
 729
 730
 731 def extract_pubkey():
 732     global repo_pubkey_fingerprint
 733     if 'repo_pubkey' in config:
 734         pubkey = unhexlify(config['repo_pubkey'])
 735     else:
 736         p = FDroidPopenBytes([config['keytool'], '-exportcert',
 737                               '-alias', config['repo_keyalias'],
 738                               '-keystore', config['keystore'],
 739                               '-storepass:file', config['keystorepassfile']]
 740                              + config['smartcardoptions'],
 741                              output=False, stderr_to_stdout=False)
 742         if p.returncode != 0 or len(p.output) < 20:
 743             msg = "Failed to get repo pubkey!"
 744             if config['keystore'] == 'NONE':
 745                 msg += ' Is your crypto smartcard plugged in?'
 746             logging.critical(msg)
 747             sys.exit(1)
 748         pubkey = p.output
 749     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 750     return hexlify(pubkey)
 751
 752
 753 def make_index(apps, sortedids, apks, repodir, archive, categories):
 754     """Make a repo index.
 755
 756     :param apps: fully populated apps list
 757     :param apks: full populated apks list
 758     :param repodir: the repo directory
 759     :param archive: True if this is the archive repo, False if it's the
 760                     main one.
 761     :param categories: list of categories
 762     """
 763
 764     doc = Document()
 765
 766     def addElement(name, value, doc, parent):
 767         el = doc.createElement(name)
 768         el.appendChild(doc.createTextNode(value))
 769         parent.appendChild(el)
 770
 771     def addElementNonEmpty(name, value, doc, parent):
 772         if not value:
 773             return
 774         addElement(name, value, doc, parent)
 775
 776     def addElementCDATA(name, value, doc, parent):
 777         el = doc.createElement(name)
 778         el.appendChild(doc.createCDATASection(value))
 779         parent.appendChild(el)
 780
 781     root = doc.createElement("fdroid")
 782     doc.appendChild(root)
 783
 784     repoel = doc.createElement("repo")
 785
 786     mirrorcheckfailed = False
 787     for mirror in config.get('mirrors', []):
 788         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
 789         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
 790             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 791             mirrorcheckfailed = True
 792     if mirrorcheckfailed:
 793         sys.exit(1)
 794
 795     if archive:
 796         repoel.setAttribute("name", config['archive_name'])
 797         if config['repo_maxage'] != 0:
 798             repoel.setAttribute("maxage", str(config['repo_maxage']))
 799         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 800         repoel.setAttribute("url", config['archive_url'])
 801         addElement('description', config['archive_description'], doc, repoel)
 802         urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
 803         for mirror in config.get('mirrors', []):
 804             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 805
 806     else:
 807         repoel.setAttribute("name", config['repo_name'])
 808         if config['repo_maxage'] != 0:
 809             repoel.setAttribute("maxage", str(config['repo_maxage']))
 810         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 811         repoel.setAttribute("url", config['repo_url'])
 812         addElement('description', config['repo_description'], doc, repoel)
 813         urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
 814         for mirror in config.get('mirrors', []):
 815             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 816
 817     repoel.setAttribute("version", str(METADATA_VERSION))
 818     repoel.setAttribute("timestamp", str(int(time.time())))
 819
 820     nosigningkey = False
 821     if not options.nosign:
 822         if 'repo_keyalias' not in config:
 823             nosigningkey = True
 824             logging.critical("'repo_keyalias' not found in config.py!")
 825         if 'keystore' not in config:
 826             nosigningkey = True
 827             logging.critical("'keystore' not found in config.py!")
 828         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 829             nosigningkey = True
 830             logging.critical("'keystorepass' not found in config.py!")
 831         if 'keypass' not in config and 'keypassfile' not in config:
 832             nosigningkey = True
 833             logging.critical("'keypass' not found in config.py!")
 834         if not os.path.exists(config['keystore']):
 835             nosigningkey = True
 836             logging.critical("'" + config['keystore'] + "' does not exist!")
 837         if nosigningkey:
 838             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 839             logging.warning("\tfdroid update --create-key")
 840             sys.exit(1)
 841
 842     repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
 843     root.appendChild(repoel)
 844
 845     for appid in sortedids:
 846         app = apps[appid]
 847
 848         if app.Disabled is not None:
 849             continue
 850
 851         # Get a list of the apks for this app...
 852         apklist = []
 853         for apk in apks:
 854             if apk['id'] == appid:
 855                 apklist.append(apk)
 856
 857         if len(apklist) == 0:
 858             continue
 859
 860         apel = doc.createElement("application")
 861         apel.setAttribute("id", app.id)
 862         root.appendChild(apel)
 863
 864         addElement('id', app.id, doc, apel)
 865         if app.added:
 866             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
 867         if app.lastupdated:
 868             addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
 869         addElement('name', app.Name, doc, apel)
 870         addElement('summary', app.Summary, doc, apel)
 871         if app.icon:
 872             addElement('icon', app.icon, doc, apel)
 873
 874         def linkres(appid):
 875             if appid in apps:
 876                 return ("fdroid.app:" + appid, apps[appid].Name)
 877             raise MetaDataException("Cannot resolve app id " + appid)
 878
 879         addElement('desc',
 880                    metadata.description_html(app.Description, linkres),
 881                    doc, apel)
 882         addElement('license', app.License, doc, apel)
 883         if app.Categories:
 884             addElement('categories', ','.join(app.Categories), doc, apel)
 885             # We put the first (primary) category in LAST, which will have
 886             # the desired effect of making clients that only understand one
 887             # category see that one.
 888             addElement('category', app.Categories[0], doc, apel)
 889         addElement('web', app.WebSite, doc, apel)
 890         addElement('source', app.SourceCode, doc, apel)
 891         addElement('tracker', app.IssueTracker, doc, apel)
 892         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 893         addElementNonEmpty('author', app.AuthorName, doc, apel)
 894         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 895         addElementNonEmpty('donate', app.Donate, doc, apel)
 896         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 897         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 898         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 899
 900         # These elements actually refer to the current version (i.e. which
 901         # one is recommended. They are historically mis-named, and need
 902         # changing, but stay like this for now to support existing clients.
 903         addElement('marketversion', app.CurrentVersion, doc, apel)
 904         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 905
 906         if app.AntiFeatures:
 907             af = app.AntiFeatures
 908             if af:
 909                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 910         if app.Provides:
 911             pv = app.Provides.split(',')
 912             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 913         if app.RequiresRoot:
 914             addElement('requirements', 'root', doc, apel)
 915
 916         # Sort the apk list into version order, just so the web site
 917         # doesn't have to do any work by default...
 918         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 919
 920         # Check for duplicates - they will make the client unhappy...
 921         for i in range(len(apklist) - 1):
 922             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 923                 logging.critical("duplicate versions: '%s' - '%s'" % (
 924                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 925                 sys.exit(1)
 926
 927         current_version_code = 0
 928         current_version_file = None
 929         for apk in apklist:
 930             # find the APK for the "Current Version"
 931             if current_version_code < apk['versioncode']:
 932                 current_version_code = apk['versioncode']
 933             if current_version_code < int(app.CurrentVersionCode):
 934                 current_version_file = apk['apkname']
 935
 936             apkel = doc.createElement("package")
 937             apel.appendChild(apkel)
 938             addElement('version', apk['version'], doc, apkel)
 939             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 940             addElement('apkname', apk['apkname'], doc, apkel)
 941             if 'srcname' in apk:
 942                 addElement('srcname', apk['srcname'], doc, apkel)
 943             for hash_type in ['sha256']:
 944                 if hash_type not in apk:
 945                     continue
 946                 hashel = doc.createElement("hash")
 947                 hashel.setAttribute("type", hash_type)
 948                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 949                 apkel.appendChild(hashel)
 950             addElement('sig', apk['sig'], doc, apkel)
 951             addElement('size', str(apk['size']), doc, apkel)
 952             addElement('sdkver', str(apk['minSdkVersion']), doc, apkel)
 953             if 'targetSdkVersion' in apk:
 954                 addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
 955             if 'maxSdkVersion' in apk:
 956                 addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
 957             if 'added' in apk:
 958                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 959             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
 960             if 'nativecode' in apk:
 961                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 962             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
 963
 964         if current_version_file is not None \
 965                 and config['make_current_version_link'] \
 966                 and repodir == 'repo':  # only create these
 967             namefield = config['current_version_name_source']
 968             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
 969             apklinkname = sanitized_name + '.apk'
 970             current_version_path = os.path.join(repodir, current_version_file)
 971             if os.path.islink(apklinkname):
 972                 os.remove(apklinkname)
 973             os.symlink(current_version_path, apklinkname)
 974             # also symlink gpg signature, if it exists
 975             for extension in ('.asc', '.sig'):
 976                 sigfile_path = current_version_path + extension
 977                 if os.path.exists(sigfile_path):
 978                     siglinkname = apklinkname + extension
 979                     if os.path.islink(siglinkname):
 980                         os.remove(siglinkname)
 981                     os.symlink(sigfile_path, siglinkname)
 982
 983     if options.pretty:
 984         output = doc.toprettyxml(encoding='utf-8')
 985     else:
 986         output = doc.toxml(encoding='utf-8')
 987
 988     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 989         f.write(output)
 990
 991     if 'repo_keyalias' in config:
 992
 993         if options.nosign:
 994             logging.info("Creating unsigned index in preparation for signing")
 995         else:
 996             logging.info("Creating signed index with this key (SHA256):")
 997             logging.info("%s" % repo_pubkey_fingerprint)
 998
 999         # Create a jar of the index...
1000         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
1001         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
1002         if p.returncode != 0:
1003             logging.critical("Failed to create {0}".format(jar_output))
1004             sys.exit(1)
1005
1006         # Sign the index...
1007         signed = os.path.join(repodir, 'index.jar')
1008         if options.nosign:
1009             # Remove old signed index if not signing
1010             if os.path.exists(signed):
1011                 os.remove(signed)
1012         else:
1013             args = [config['jarsigner'], '-keystore', config['keystore'],
1014                     '-storepass:file', config['keystorepassfile'],
1015                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1016                     signed, config['repo_keyalias']]
1017             if config['keystore'] == 'NONE':
1018                 args += config['smartcardoptions']
1019             else:  # smardcards never use -keypass
1020                 args += ['-keypass:file', config['keypassfile']]
1021             p = FDroidPopen(args)
1022             if p.returncode != 0:
1023                 logging.critical("Failed to sign index")
1024                 sys.exit(1)
1025
1026     # Copy the repo icon into the repo directory...
1027     icon_dir = os.path.join(repodir, 'icons')
1028     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1029     shutil.copyfile(config['repo_icon'], iconfilename)
1030
1031     # Write a category list in the repo to allow quick access...
1032     catdata = ''
1033     for cat in categories:
1034         catdata += cat + '\n'
1035     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1036         f.write(catdata)
1037
1038
1039 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1040
1041     for appid, app in apps.items():
1042
1043         if app.ArchivePolicy:
1044             keepversions = int(app.ArchivePolicy[:-9])
1045         else:
1046             keepversions = defaultkeepversions
1047
1048         def filter_apk_list_sorted(apk_list):
1049             res = []
1050             for apk in apk_list:
1051                 if apk['id'] == appid:
1052                     res.append(apk)
1053
1054             # Sort the apk list by version code. First is highest/newest.
1055             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1056
1057         def move_file(from_dir, to_dir, filename, ignore_missing):
1058             from_path = os.path.join(from_dir, filename)
1059             if ignore_missing and not os.path.exists(from_path):
1060                 return
1061             to_path = os.path.join(to_dir, filename)
1062             shutil.move(from_path, to_path)
1063
1064         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1065                       .format(appid, len(apks), keepversions, len(archapks)))
1066
1067         if len(apks) > keepversions:
1068             apklist = filter_apk_list_sorted(apks)
1069             # Move back the ones we don't want.
1070             for apk in apklist[keepversions:]:
1071                 logging.info("Moving " + apk['apkname'] + " to archive")
1072                 move_file(repodir, archivedir, apk['apkname'], False)
1073                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1074                 for density in all_screen_densities:
1075                     repo_icon_dir = get_icon_dir(repodir, density)
1076                     archive_icon_dir = get_icon_dir(archivedir, density)
1077                     if density not in apk['icons']:
1078                         continue
1079                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1080                 if 'srcname' in apk:
1081                     move_file(repodir, archivedir, apk['srcname'], False)
1082                 archapks.append(apk)
1083                 apks.remove(apk)
1084         elif len(apks) < keepversions and len(archapks) > 0:
1085             required = keepversions - len(apks)
1086             archapklist = filter_apk_list_sorted(archapks)
1087             # Move forward the ones we want again.
1088             for apk in archapklist[:required]:
1089                 logging.info("Moving " + apk['apkname'] + " from archive")
1090                 move_file(archivedir, repodir, apk['apkname'], False)
1091                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1092                 for density in all_screen_densities:
1093                     repo_icon_dir = get_icon_dir(repodir, density)
1094                     archive_icon_dir = get_icon_dir(archivedir, density)
1095                     if density not in apk['icons']:
1096                         continue
1097                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1098                 if 'srcname' in apk:
1099                     move_file(archivedir, repodir, apk['srcname'], False)
1100                 archapks.remove(apk)
1101                 apks.append(apk)
1102
1103
1104 def add_apks_to_per_app_repos(repodir, apks):
1105     apks_per_app = dict()
1106     for apk in apks:
1107         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1108         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1109         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1110         apks_per_app[apk['id']] = apk
1111
1112         if not os.path.exists(apk['per_app_icons']):
1113             logging.info('Adding new repo for only ' + apk['id'])
1114             os.makedirs(apk['per_app_icons'])
1115
1116         apkpath = os.path.join(repodir, apk['apkname'])
1117         shutil.copy(apkpath, apk['per_app_repo'])
1118         apksigpath = apkpath + '.sig'
1119         if os.path.exists(apksigpath):
1120             shutil.copy(apksigpath, apk['per_app_repo'])
1121         apkascpath = apkpath + '.asc'
1122         if os.path.exists(apkascpath):
1123             shutil.copy(apkascpath, apk['per_app_repo'])
1124
1125
1126 config = None
1127 options = None
1128
1129
1130 def main():
1131
1132     global config, options
1133
1134     # Parse command line...
1135     parser = ArgumentParser()
1136     common.setup_global_opts(parser)
1137     parser.add_argument("--create-key", action="store_true", default=False,
1138                         help="Create a repo signing key in a keystore")
1139     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1140                         help="Create skeleton metadata files that are missing")
1141     parser.add_argument("--delete-unknown", action="store_true", default=False,
1142                         help="Delete APKs without metadata from the repo")
1143     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1144                         help="Report on build data status")
1145     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1146                         help="Interactively ask about things that need updating.")
1147     parser.add_argument("-I", "--icons", action="store_true", default=False,
1148                         help="Resize all the icons exceeding the max pixel size and exit")
1149     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1150                         help="Specify editor to use in interactive mode. Default " +
1151                         "is /etc/alternatives/editor")
1152     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1153                         help="Update the wiki")
1154     parser.add_argument("--pretty", action="store_true", default=False,
1155                         help="Produce human-readable index.xml")
1156     parser.add_argument("--clean", action="store_true", default=False,
1157                         help="Clean update - don't uses caches, reprocess all apks")
1158     parser.add_argument("--nosign", action="store_true", default=False,
1159                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1160     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1161                         help="Use date from apk instead of current time for newly added apks")
1162     options = parser.parse_args()
1163
1164     config = common.read_config(options)
1165
1166     if not ('jarsigner' in config and 'keytool' in config):
1167         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1168         sys.exit(1)
1169
1170     repodirs = ['repo']
1171     if config['archive_older'] != 0:
1172         repodirs.append('archive')
1173         if not os.path.exists('archive'):
1174             os.mkdir('archive')
1175
1176     if options.icons:
1177         resize_all_icons(repodirs)
1178         sys.exit(0)
1179
1180     # check that icons exist now, rather than fail at the end of `fdroid update`
1181     for k in ['repo_icon', 'archive_icon']:
1182         if k in config:
1183             if not os.path.exists(config[k]):
1184                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1185                 sys.exit(1)
1186
1187     # if the user asks to create a keystore, do it now, reusing whatever it can
1188     if options.create_key:
1189         if os.path.exists(config['keystore']):
1190             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1191             logging.critical("\t'" + config['keystore'] + "'")
1192             sys.exit(1)
1193
1194         if 'repo_keyalias' not in config:
1195             config['repo_keyalias'] = socket.getfqdn()
1196             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1197         if 'keydname' not in config:
1198             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1199             common.write_to_config(config, 'keydname', config['keydname'])
1200         if 'keystore' not in config:
1201             config['keystore'] = common.default_config.keystore
1202             common.write_to_config(config, 'keystore', config['keystore'])
1203
1204         password = common.genpassword()
1205         if 'keystorepass' not in config:
1206             config['keystorepass'] = password
1207             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1208         if 'keypass' not in config:
1209             config['keypass'] = password
1210             common.write_to_config(config, 'keypass', config['keypass'])
1211         common.genkeystore(config)
1212
1213     # Get all apps...
1214     apps = metadata.read_metadata()
1215
1216     # Generate a list of categories...
1217     categories = set()
1218     for app in apps.values():
1219         categories.update(app.Categories)
1220
1221     # Read known apks data (will be updated and written back when we've finished)
1222     knownapks = common.KnownApks()
1223
1224     # Gather information about all the apk files in the repo directory, using
1225     # cached data if possible.
1226     apkcachefile = os.path.join('tmp', 'apkcache')
1227     if not options.clean and os.path.exists(apkcachefile):
1228         with open(apkcachefile, 'rb') as cf:
1229             apkcache = pickle.load(cf, encoding='utf-8')
1230         if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
1231             apkcache = {}
1232     else:
1233         apkcache = {}
1234
1235     delete_disabled_builds(apps, apkcache, repodirs)
1236
1237     # Scan all apks in the main repo
1238     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1239
1240     # Generate warnings for apk's with no metadata (or create skeleton
1241     # metadata files, if requested on the command line)
1242     newmetadata = False
1243     for apk in apks:
1244         if apk['id'] not in apps:
1245             if options.create_metadata:
1246                 if 'name' not in apk:
1247                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1248                     continue
1249                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
1250                 f.write("License:Unknown\n")
1251                 f.write("Web Site:\n")
1252                 f.write("Source Code:\n")
1253                 f.write("Issue Tracker:\n")
1254                 f.write("Changelog:\n")
1255                 f.write("Summary:" + apk['name'] + "\n")
1256                 f.write("Description:\n")
1257                 f.write(apk['name'] + "\n")
1258                 f.write(".\n")
1259                 f.close()
1260                 logging.info("Generated skeleton metadata for " + apk['id'])
1261                 newmetadata = True
1262             else:
1263                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1264                 if options.delete_unknown:
1265                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1266                     rmf = os.path.join(repodirs[0], apk['apkname'])
1267                     if not os.path.exists(rmf):
1268                         logging.error("Could not find {0} to remove it".format(rmf))
1269                     else:
1270                         os.remove(rmf)
1271                 else:
1272                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1273
1274     # update the metadata with the newly created ones included
1275     if newmetadata:
1276         apps = metadata.read_metadata()
1277
1278     # Scan the archive repo for apks as well
1279     if len(repodirs) > 1:
1280         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1281         if cc:
1282             cachechanged = True
1283     else:
1284         archapks = []
1285
1286     # Some information from the apks needs to be applied up to the application
1287     # level. When doing this, we use the info from the most recent version's apk.
1288     # We deal with figuring out when the app was added and last updated at the
1289     # same time.
1290     for appid, app in apps.items():
1291         bestver = 0
1292         for apk in apks + archapks:
1293             if apk['id'] == appid:
1294                 if apk['versioncode'] > bestver:
1295                     bestver = apk['versioncode']
1296                     bestapk = apk
1297
1298                 if 'added' in apk:
1299                     if not app.added or apk['added'] < app.added:
1300                         app.added = apk['added']
1301                     if not app.lastupdated or apk['added'] > app.lastupdated:
1302                         app.lastupdated = apk['added']
1303
1304         if not app.added:
1305             logging.debug("Don't know when " + appid + " was added")
1306         if not app.lastupdated:
1307             logging.debug("Don't know when " + appid + " was last updated")
1308
1309         if bestver == 0:
1310             if app.Name is None:
1311                 app.Name = app.AutoName or appid
1312             app.icon = None
1313             logging.debug("Application " + appid + " has no packages")
1314         else:
1315             if app.Name is None:
1316                 app.Name = bestapk['name']
1317             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1318             if app.CurrentVersionCode is None:
1319                 app.CurrentVersionCode = str(bestver)
1320
1321     # Sort the app list by name, then the web site doesn't have to by default.
1322     # (we had to wait until we'd scanned the apks to do this, because mostly the
1323     # name comes from there!)
1324     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1325
1326     # APKs are placed into multiple repos based on the app package, providing
1327     # per-app subscription feeds for nightly builds and things like it
1328     if config['per_app_repos']:
1329         add_apks_to_per_app_repos(repodirs[0], apks)
1330         for appid, app in apps.items():
1331             repodir = os.path.join(appid, 'fdroid', 'repo')
1332             appdict = dict()
1333             appdict[appid] = app
1334             if os.path.isdir(repodir):
1335                 make_index(appdict, [appid], apks, repodir, False, categories)
1336             else:
1337                 logging.info('Skipping index generation for ' + appid)
1338         return
1339
1340     if len(repodirs) > 1:
1341         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1342
1343     # Make the index for the main repo...
1344     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1345
1346     # If there's an archive repo,  make the index for it. We already scanned it
1347     # earlier on.
1348     if len(repodirs) > 1:
1349         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1350
1351     if config['update_stats']:
1352
1353         # Update known apks info...
1354         knownapks.writeifchanged()
1355
1356         # Generate latest apps data for widget
1357         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1358             data = ''
1359             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1360                 for line in f:
1361                     appid = line.rstrip()
1362                     data += appid + "\t"
1363                     app = apps[appid]
1364                     data += app.Name + "\t"
1365                     if app.icon is not None:
1366                         data += app.icon + "\t"
1367                     data += app.License + "\n"
1368             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1369                 f.write(data)
1370
1371     if cachechanged:
1372         apkcache["METADATA_VERSION"] = METADATA_VERSION
1373         with open(apkcachefile, 'wb') as cf:
1374             pickle.dump(apkcache, cf)
1375
1376     # Update the wiki...
1377     if options.wiki:
1378         update_wiki(apps, sortedids, apks + archapks)
1379
1380     logging.info("Finished.")
1381
1382 if __name__ == "__main__":
1383     main()