fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   5 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU Affero General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Affero General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Affero General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 import sys
  21 import os
  22 import shutil
  23 import glob
  24 import re
  25 import socket
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 import urllib.parse
  30 from datetime import datetime, timedelta
  31 from xml.dom.minidom import Document
  32 from argparse import ArgumentParser
  33 import time
  34 from pyasn1.error import PyAsn1Error
  35 from pyasn1.codec.der import decoder, encoder
  36 from pyasn1_modules import rfc2315
  37 from binascii import hexlify, unhexlify
  38
  39 from PIL import Image
  40 import logging
  41
  42 from . import common
  43 from . import metadata
  44 from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
  45 from .metadata import MetaDataException
  46
  47 METADATA_VERSION = 16
  48
  49 screen_densities = ['640', '480', '320', '240', '160', '120']
  50
  51 all_screen_densities = ['0'] + screen_densities
  52
  53
  54 def dpi_to_px(density):
  55     return (int(density) * 48) / 160
  56
  57
  58 def px_to_dpi(px):
  59     return (int(px) * 160) / 48
  60
  61
  62 def get_icon_dir(repodir, density):
  63     if density == '0':
  64         return os.path.join(repodir, "icons")
  65     return os.path.join(repodir, "icons-%s" % density)
  66
  67
  68 def get_icon_dirs(repodir):
  69     for density in screen_densities:
  70         yield get_icon_dir(repodir, density)
  71
  72
  73 def get_all_icon_dirs(repodir):
  74     for density in all_screen_densities:
  75         yield get_icon_dir(repodir, density)
  76
  77
  78 def update_wiki(apps, sortedids, apks):
  79     """Update the wiki
  80
  81     :param apps: fully populated list of all applications
  82     :param apks: all apks, except...
  83     """
  84     logging.info("Updating wiki")
  85     wikicat = 'Apps'
  86     wikiredircat = 'App Redirects'
  87     import mwclient
  88     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  89                          path=config['wiki_path'])
  90     site.login(config['wiki_user'], config['wiki_password'])
  91     generated_pages = {}
  92     generated_redirects = {}
  93
  94     for appid in sortedids:
  95         app = apps[appid]
  96
  97         wikidata = ''
  98         if app.Disabled:
  99             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 100         if app.AntiFeatures:
 101             for af in app.AntiFeatures:
 102                 wikidata += '{{AntiFeature|' + af + '}}\n'
 103         if app.RequiresRoot:
 104             requiresroot = 'Yes'
 105         else:
 106             requiresroot = 'No'
 107         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 108             appid,
 109             app.Name,
 110             time.strftime('%Y-%m-%d', app.added) if app.added else '',
 111             time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
 112             app.SourceCode,
 113             app.IssueTracker,
 114             app.WebSite,
 115             app.Changelog,
 116             app.Donate,
 117             app.FlattrID,
 118             app.Bitcoin,
 119             app.Litecoin,
 120             app.License,
 121             requiresroot,
 122             app.AuthorName,
 123             app.AuthorEmail)
 124
 125         if app.Provides:
 126             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 127
 128         wikidata += app.Summary
 129         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 130
 131         wikidata += "=Description=\n"
 132         wikidata += metadata.description_wiki(app.Description) + "\n"
 133
 134         wikidata += "=Maintainer Notes=\n"
 135         if app.MaintainerNotes:
 136             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 137         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 138
 139         # Get a list of all packages for this application...
 140         apklist = []
 141         gotcurrentver = False
 142         cantupdate = False
 143         buildfails = False
 144         for apk in apks:
 145             if apk['id'] == appid:
 146                 if str(apk['versioncode']) == app.CurrentVersionCode:
 147                     gotcurrentver = True
 148                 apklist.append(apk)
 149         # Include ones we can't build, as a special case...
 150         for build in app.builds:
 151             if build.disable:
 152                 if build.vercode == app.CurrentVersionCode:
 153                     cantupdate = True
 154                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 155                 apklist.append({'versioncode': int(build.vercode),
 156                                 'version': build.version,
 157                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 158                                 })
 159             else:
 160                 builtit = False
 161                 for apk in apklist:
 162                     if apk['versioncode'] == int(build.vercode):
 163                         builtit = True
 164                         break
 165                 if not builtit:
 166                     buildfails = True
 167                     apklist.append({'versioncode': int(build.vercode),
 168                                     'version': build.version,
 169                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
 170                                     })
 171         if app.CurrentVersionCode == '0':
 172             cantupdate = True
 173         # Sort with most recent first...
 174         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 175
 176         wikidata += "=Versions=\n"
 177         if len(apklist) == 0:
 178             wikidata += "We currently have no versions of this app available."
 179         elif not gotcurrentver:
 180             wikidata += "We don't have the current version of this app."
 181         else:
 182             wikidata += "We have the current version of this app."
 183         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 184         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 185         if len(app.NoSourceSince) > 0:
 186             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 187         if len(app.CurrentVersion) > 0:
 188             wikidata += "The current (recommended) version is " + app.CurrentVersion
 189             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 190         validapks = 0
 191         for apk in apklist:
 192             wikidata += "==" + apk['version'] + "==\n"
 193
 194             if 'buildproblem' in apk:
 195                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 196             else:
 197                 validapks += 1
 198                 wikidata += "This version is built and signed by "
 199                 if 'srcname' in apk:
 200                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 201                 else:
 202                     wikidata += "the original developer.\n\n"
 203             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 204
 205         wikidata += '\n[[Category:' + wikicat + ']]\n'
 206         if len(app.NoSourceSince) > 0:
 207             wikidata += '\n[[Category:Apps missing source code]]\n'
 208         if validapks == 0 and not app.Disabled:
 209             wikidata += '\n[[Category:Apps with no packages]]\n'
 210         if cantupdate and not app.Disabled:
 211             wikidata += "\n[[Category:Apps we cannot update]]\n"
 212         if buildfails and not app.Disabled:
 213             wikidata += "\n[[Category:Apps with failing builds]]\n"
 214         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 215             wikidata += '\n[[Category:Apps to Update]]\n'
 216         if app.Disabled:
 217             wikidata += '\n[[Category:Apps that are disabled]]\n'
 218         if app.UpdateCheckMode == 'None' and not app.Disabled:
 219             wikidata += '\n[[Category:Apps with no update check]]\n'
 220         for appcat in app.Categories:
 221             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 222
 223         # We can't have underscores in the page name, even if they're in
 224         # the package ID, because MediaWiki messes with them...
 225         pagename = appid.replace('_', ' ')
 226
 227         # Drop a trailing newline, because mediawiki is going to drop it anyway
 228         # and it we don't we'll think the page has changed when it hasn't...
 229         if wikidata.endswith('\n'):
 230             wikidata = wikidata[:-1]
 231
 232         generated_pages[pagename] = wikidata
 233
 234         # Make a redirect from the name to the ID too, unless there's
 235         # already an existing page with the name and it isn't a redirect.
 236         noclobber = False
 237         apppagename = app.Name.replace('_', ' ')
 238         apppagename = apppagename.replace('{', '')
 239         apppagename = apppagename.replace('}', ' ')
 240         apppagename = apppagename.replace(':', ' ')
 241         # Drop double spaces caused mostly by replacing ':' above
 242         apppagename = apppagename.replace('  ', ' ')
 243         for expagename in site.allpages(prefix=apppagename,
 244                                         filterredir='nonredirects',
 245                                         generator=False):
 246             if expagename == apppagename:
 247                 noclobber = True
 248         # Another reason not to make the redirect page is if the app name
 249         # is the same as it's ID, because that will overwrite the real page
 250         # with an redirect to itself! (Although it seems like an odd
 251         # scenario this happens a lot, e.g. where there is metadata but no
 252         # builds or binaries to extract a name from.
 253         if apppagename == pagename:
 254             noclobber = True
 255         if not noclobber:
 256             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 257
 258     for tcat, genp in [(wikicat, generated_pages),
 259                        (wikiredircat, generated_redirects)]:
 260         catpages = site.Pages['Category:' + tcat]
 261         existingpages = []
 262         for page in catpages:
 263             existingpages.append(page.name)
 264             if page.name in genp:
 265                 pagetxt = page.edit()
 266                 if pagetxt != genp[page.name]:
 267                     logging.debug("Updating modified page " + page.name)
 268                     page.save(genp[page.name], summary='Auto-updated')
 269                 else:
 270                     logging.debug("Page " + page.name + " is unchanged")
 271             else:
 272                 logging.warn("Deleting page " + page.name)
 273                 page.delete('No longer published')
 274         for pagename, text in genp.items():
 275             logging.debug("Checking " + pagename)
 276             if pagename not in existingpages:
 277                 logging.debug("Creating page " + pagename)
 278                 try:
 279                     newpage = site.Pages[pagename]
 280                     newpage.save(text, summary='Auto-created')
 281                 except:
 282                     logging.error("...FAILED to create page '{0}'".format(pagename))
 283
 284     # Purge server cache to ensure counts are up to date
 285     site.pages['Repository Maintenance'].purge()
 286
 287
 288 def delete_disabled_builds(apps, apkcache, repodirs):
 289     """Delete disabled build outputs.
 290
 291     :param apps: list of all applications, as per metadata.read_metadata
 292     :param apkcache: current apk cache information
 293     :param repodirs: the repo directories to process
 294     """
 295     for appid, app in apps.items():
 296         for build in app.builds:
 297             if not build.disable:
 298                 continue
 299             apkfilename = appid + '_' + str(build.vercode) + '.apk'
 300             iconfilename = "%s.%s.png" % (
 301                 appid,
 302                 build.vercode)
 303             for repodir in repodirs:
 304                 files = [
 305                     os.path.join(repodir, apkfilename),
 306                     os.path.join(repodir, apkfilename + '.asc'),
 307                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 308                 ]
 309                 for density in all_screen_densities:
 310                     repo_dir = get_icon_dir(repodir, density)
 311                     files.append(os.path.join(repo_dir, iconfilename))
 312
 313                 for f in files:
 314                     if os.path.exists(f):
 315                         logging.info("Deleting disabled build output " + f)
 316                         os.remove(f)
 317             if apkfilename in apkcache:
 318                 del apkcache[apkfilename]
 319
 320
 321 def resize_icon(iconpath, density):
 322
 323     if not os.path.isfile(iconpath):
 324         return
 325
 326     fp = None
 327     try:
 328         fp = open(iconpath, 'rb')
 329         im = Image.open(fp)
 330         size = dpi_to_px(density)
 331
 332         if any(length > size for length in im.size):
 333             oldsize = im.size
 334             im.thumbnail((size, size), Image.ANTIALIAS)
 335             logging.debug("%s was too large at %s - new size is %s" % (
 336                 iconpath, oldsize, im.size))
 337             im.save(iconpath, "PNG")
 338
 339     except Exception as e:
 340         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 341
 342     finally:
 343         if fp:
 344             fp.close()
 345
 346
 347 def resize_all_icons(repodirs):
 348     """Resize all icons that exceed the max size
 349
 350     :param repodirs: the repo directories to process
 351     """
 352     for repodir in repodirs:
 353         for density in screen_densities:
 354             icon_dir = get_icon_dir(repodir, density)
 355             icon_glob = os.path.join(icon_dir, '*.png')
 356             for iconpath in glob.glob(icon_glob):
 357                 resize_icon(iconpath, density)
 358
 359
 360 # A signature block file with a .DSA, .RSA, or .EC extension
 361 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 362
 363
 364 def getsig(apkpath):
 365     """ Get the signing certificate of an apk. To get the same md5 has that
 366     Android gets, we encode the .RSA certificate in a specific format and pass
 367     it hex-encoded to the md5 digest algorithm.
 368
 369     :param apkpath: path to the apk
 370     :returns: A string containing the md5 of the signature of the apk or None
 371               if an error occurred.
 372     """
 373
 374     cert = None
 375
 376     # verify the jar signature is correct
 377     args = [config['jarsigner'], '-verify', apkpath]
 378     p = FDroidPopen(args)
 379     if p.returncode != 0:
 380         logging.critical(apkpath + " has a bad signature!")
 381         return None
 382
 383     with zipfile.ZipFile(apkpath, 'r') as apk:
 384
 385         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 386
 387         if len(certs) < 1:
 388             logging.error("Found no signing certificates on %s" % apkpath)
 389             return None
 390         if len(certs) > 1:
 391             logging.error("Found multiple signing certificates on %s" % apkpath)
 392             return None
 393
 394         cert = apk.read(certs[0])
 395
 396     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 397     if content.getComponentByName('contentType') != rfc2315.signedData:
 398         logging.error("Unexpected format.")
 399         return None
 400
 401     content = decoder.decode(content.getComponentByName('content'),
 402                              asn1Spec=rfc2315.SignedData())[0]
 403     try:
 404         certificates = content.getComponentByName('certificates')
 405     except PyAsn1Error:
 406         logging.error("Certificates not found.")
 407         return None
 408
 409     cert_encoded = encoder.encode(certificates)[4:]
 410
 411     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 412
 413
 414 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
 415     """Scan the apks in the given repo directory.
 416
 417     This also extracts the icons.
 418
 419     :param apps: list of all applications, as per metadata.read_metadata
 420     :param apkcache: current apk cache information
 421     :param repodir: repo directory to scan
 422     :param knownapks: known apks info
 423     :param use_date_from_apk: use date from APK (instead of current date)
 424                               for newly added APKs
 425     :returns: (apks, cachechanged) where apks is a list of apk information,
 426               and cachechanged is True if the apkcache got changed.
 427     """
 428
 429     cachechanged = False
 430
 431     for icon_dir in get_all_icon_dirs(repodir):
 432         if os.path.exists(icon_dir):
 433             if options.clean:
 434                 shutil.rmtree(icon_dir)
 435                 os.makedirs(icon_dir)
 436         else:
 437             os.makedirs(icon_dir)
 438
 439     apks = []
 440     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 441     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 442     vername_pat = re.compile(".*versionName='([^']*)'.*")
 443     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 444     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 445     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 446     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 447     string_pat = re.compile(".* name='([^']*)'.*")
 448     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 449
 450         apkfilename = apkfile[len(repodir) + 1:]
 451         if ' ' in apkfilename:
 452             logging.critical("Spaces in filenames are not allowed.")
 453             sys.exit(1)
 454
 455         # Calculate the sha256...
 456         sha = hashlib.sha256()
 457         with open(apkfile, 'rb') as f:
 458             while True:
 459                 t = f.read(16384)
 460                 if len(t) == 0:
 461                     break
 462                 sha.update(t)
 463             shasum = sha.hexdigest()
 464
 465         usecache = False
 466         if apkfilename in apkcache:
 467             apk = apkcache[apkfilename]
 468             if apk['sha256'] == shasum:
 469                 logging.debug("Reading " + apkfilename + " from cache")
 470                 usecache = True
 471             else:
 472                 logging.debug("Ignoring stale cache data for " + apkfilename)
 473
 474         if not usecache:
 475             logging.debug("Processing " + apkfilename)
 476             apk = {}
 477             apk['apkname'] = apkfilename
 478             apk['sha256'] = shasum
 479             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 480             if os.path.exists(os.path.join(repodir, srcfilename)):
 481                 apk['srcname'] = srcfilename
 482             apk['size'] = os.path.getsize(apkfile)
 483             apk['permissions'] = set()
 484             apk['features'] = set()
 485             apk['icons_src'] = {}
 486             apk['icons'] = {}
 487             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 488             if p.returncode != 0:
 489                 if options.delete_unknown:
 490                     if os.path.exists(apkfile):
 491                         logging.error("Failed to get apk information, deleting " + apkfile)
 492                         os.remove(apkfile)
 493                     else:
 494                         logging.error("Could not find {0} to remove it".format(apkfile))
 495                 else:
 496                     logging.error("Failed to get apk information, skipping " + apkfile)
 497                 continue
 498             for line in p.output.splitlines():
 499                 if line.startswith("package:"):
 500                     try:
 501                         apk['id'] = re.match(name_pat, line).group(1)
 502                         apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
 503                         apk['version'] = re.match(vername_pat, line).group(1)
 504                     except Exception as e:
 505                         logging.error("Package matching failed: " + str(e))
 506                         logging.info("Line was: " + line)
 507                         sys.exit(1)
 508                 elif line.startswith("application:"):
 509                     apk['name'] = re.match(label_pat, line).group(1)
 510                     # Keep path to non-dpi icon in case we need it
 511                     match = re.match(icon_pat_nodpi, line)
 512                     if match:
 513                         apk['icons_src']['-1'] = match.group(1)
 514                 elif line.startswith("launchable-activity:"):
 515                     # Only use launchable-activity as fallback to application
 516                     if not apk['name']:
 517                         apk['name'] = re.match(label_pat, line).group(1)
 518                     if '-1' not in apk['icons_src']:
 519                         match = re.match(icon_pat_nodpi, line)
 520                         if match:
 521                             apk['icons_src']['-1'] = match.group(1)
 522                 elif line.startswith("application-icon-"):
 523                     match = re.match(icon_pat, line)
 524                     if match:
 525                         density = match.group(1)
 526                         path = match.group(2)
 527                         apk['icons_src'][density] = path
 528                 elif line.startswith("sdkVersion:"):
 529                     m = re.match(sdkversion_pat, line)
 530                     if m is None:
 531                         logging.error(line.replace('sdkVersion:', '')
 532                                       + ' is not a valid minSdkVersion!')
 533                     else:
 534                         apk['minSdkVersion'] = m.group(1)
 535                         # if target not set, default to min
 536                         if 'targetSdkVersion' not in apk:
 537                             apk['targetSdkVersion'] = m.group(1)
 538                 elif line.startswith("targetSdkVersion:"):
 539                     m = re.match(sdkversion_pat, line)
 540                     if m is None:
 541                         logging.error(line.replace('targetSdkVersion:', '')
 542                                       + ' is not a valid targetSdkVersion!')
 543                     else:
 544                         apk['targetSdkVersion'] = m.group(1)
 545                 elif line.startswith("maxSdkVersion:"):
 546                     apk['maxSdkVersion'] = re.match(sdkversion_pat, line).group(1)
 547                 elif line.startswith("native-code:"):
 548                     apk['nativecode'] = []
 549                     for arch in line[13:].split(' '):
 550                         apk['nativecode'].append(arch[1:-1])
 551                 elif line.startswith("uses-permission:"):
 552                     perm = re.match(string_pat, line).group(1)
 553                     if perm.startswith("android.permission."):
 554                         perm = perm[19:]
 555                     apk['permissions'].add(perm)
 556                 elif line.startswith("uses-feature:"):
 557                     perm = re.match(string_pat, line).group(1)
 558                     # Filter out this, it's only added with the latest SDK tools and
 559                     # causes problems for lots of apps.
 560                     if perm != "android.hardware.screen.portrait" \
 561                             and perm != "android.hardware.screen.landscape":
 562                         if perm.startswith("android.feature."):
 563                             perm = perm[16:]
 564                         apk['features'].add(perm)
 565
 566             if 'minSdkVersion' not in apk:
 567                 logging.warn("No SDK version information found in {0}".format(apkfile))
 568                 apk['minSdkVersion'] = 1
 569
 570             # Check for debuggable apks...
 571             if common.isApkDebuggable(apkfile, config):
 572                 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
 573
 574             # Get the signature (or md5 of, to be precise)...
 575             logging.debug('Getting signature of {0}'.format(apkfile))
 576             apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 577             if not apk['sig']:
 578                 logging.critical("Failed to get apk signature")
 579                 sys.exit(1)
 580
 581             apkzip = zipfile.ZipFile(apkfile, 'r')
 582
 583             # if an APK has files newer than the system time, suggest updating
 584             # the system clock.  This is useful for offline systems, used for
 585             # signing, which do not have another source of clock sync info. It
 586             # has to be more than 24 hours newer because ZIP/APK files do not
 587             # store timezone info
 588             manifest = apkzip.getinfo('AndroidManifest.xml')
 589             if manifest.date_time[1] == 0:  # month can't be zero
 590                 logging.debug('AndroidManifest.xml has no date')
 591             else:
 592                 dt_obj = datetime(*manifest.date_time)
 593                 checkdt = dt_obj - timedelta(1)
 594                 if datetime.today() < checkdt:
 595                     logging.warn('System clock is older than manifest in: '
 596                                  + apkfilename
 597                                  + '\nSet clock to that time using:\n'
 598                                  + 'sudo date -s "' + str(dt_obj) + '"')
 599
 600             iconfilename = "%s.%s.png" % (
 601                 apk['id'],
 602                 apk['versioncode'])
 603
 604             # Extract the icon file...
 605             empty_densities = []
 606             for density in screen_densities:
 607                 if density not in apk['icons_src']:
 608                     empty_densities.append(density)
 609                     continue
 610                 iconsrc = apk['icons_src'][density]
 611                 icon_dir = get_icon_dir(repodir, density)
 612                 icondest = os.path.join(icon_dir, iconfilename)
 613
 614                 try:
 615                     with open(icondest, 'wb') as f:
 616                         f.write(apkzip.read(iconsrc))
 617                     apk['icons'][density] = iconfilename
 618
 619                 except:
 620                     logging.warn("Error retrieving icon file")
 621                     del apk['icons'][density]
 622                     del apk['icons_src'][density]
 623                     empty_densities.append(density)
 624
 625             if '-1' in apk['icons_src']:
 626                 iconsrc = apk['icons_src']['-1']
 627                 iconpath = os.path.join(
 628                     get_icon_dir(repodir, '0'), iconfilename)
 629                 with open(iconpath, 'wb') as f:
 630                     f.write(apkzip.read(iconsrc))
 631                 try:
 632                     im = Image.open(iconpath)
 633                     dpi = px_to_dpi(im.size[0])
 634                     for density in screen_densities:
 635                         if density in apk['icons']:
 636                             break
 637                         if density == screen_densities[-1] or dpi >= int(density):
 638                             apk['icons'][density] = iconfilename
 639                             shutil.move(iconpath,
 640                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 641                             empty_densities.remove(density)
 642                             break
 643                 except Exception as e:
 644                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 645
 646             if apk['icons']:
 647                 apk['icon'] = iconfilename
 648
 649             apkzip.close()
 650
 651             # First try resizing down to not lose quality
 652             last_density = None
 653             for density in screen_densities:
 654                 if density not in empty_densities:
 655                     last_density = density
 656                     continue
 657                 if last_density is None:
 658                     continue
 659                 logging.debug("Density %s not available, resizing down from %s"
 660                               % (density, last_density))
 661
 662                 last_iconpath = os.path.join(
 663                     get_icon_dir(repodir, last_density), iconfilename)
 664                 iconpath = os.path.join(
 665                     get_icon_dir(repodir, density), iconfilename)
 666                 fp = None
 667                 try:
 668                     fp = open(last_iconpath, 'rb')
 669                     im = Image.open(fp)
 670
 671                     size = dpi_to_px(density)
 672
 673                     im.thumbnail((size, size), Image.ANTIALIAS)
 674                     im.save(iconpath, "PNG")
 675                     empty_densities.remove(density)
 676                 except:
 677                     logging.warning("Invalid image file at %s" % last_iconpath)
 678                 finally:
 679                     if fp:
 680                         fp.close()
 681
 682             # Then just copy from the highest resolution available
 683             last_density = None
 684             for density in reversed(screen_densities):
 685                 if density not in empty_densities:
 686                     last_density = density
 687                     continue
 688                 if last_density is None:
 689                     continue
 690                 logging.debug("Density %s not available, copying from lower density %s"
 691                               % (density, last_density))
 692
 693                 shutil.copyfile(
 694                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 695                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 696
 697                 empty_densities.remove(density)
 698
 699             for density in screen_densities:
 700                 icon_dir = get_icon_dir(repodir, density)
 701                 icondest = os.path.join(icon_dir, iconfilename)
 702                 resize_icon(icondest, density)
 703
 704             # Copy from icons-mdpi to icons since mdpi is the baseline density
 705             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 706             if os.path.isfile(baseline):
 707                 apk['icons']['0'] = iconfilename
 708                 shutil.copyfile(baseline,
 709                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 710
 711             # Record in known apks, getting the added date at the same time..
 712             added = knownapks.recordapk(apk['apkname'], apk['id'])
 713             if added:
 714                 if use_date_from_apk and manifest.date_time[1] != 0:
 715                     added = datetime(*manifest.date_time).timetuple()
 716                     logging.debug("Using date from APK")
 717
 718                 apk['added'] = added
 719
 720             apkcache[apkfilename] = apk
 721             cachechanged = True
 722
 723         apks.append(apk)
 724
 725     return apks, cachechanged
 726
 727
 728 repo_pubkey_fingerprint = None
 729
 730
 731 # Generate a certificate fingerprint the same way keytool does it
 732 # (but with slightly different formatting)
 733 def cert_fingerprint(data):
 734     digest = hashlib.sha256(data).digest()
 735     ret = []
 736     ret.append(' '.join("%02X" % b for b in bytearray(digest)))
 737     return " ".join(ret)
 738
 739
 740 def extract_pubkey():
 741     global repo_pubkey_fingerprint
 742     if 'repo_pubkey' in config:
 743         pubkey = unhexlify(config['repo_pubkey'])
 744     else:
 745         p = FDroidPopenBytes([config['keytool'], '-exportcert',
 746                               '-alias', config['repo_keyalias'],
 747                               '-keystore', config['keystore'],
 748                               '-storepass:file', config['keystorepassfile']]
 749                              + config['smartcardoptions'],
 750                              output=False, stderr_to_stdout=False)
 751         if p.returncode != 0 or len(p.output) < 20:
 752             msg = "Failed to get repo pubkey!"
 753             if config['keystore'] == 'NONE':
 754                 msg += ' Is your crypto smartcard plugged in?'
 755             logging.critical(msg)
 756             sys.exit(1)
 757         pubkey = p.output
 758     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 759     return hexlify(pubkey)
 760
 761
 762 def make_index(apps, sortedids, apks, repodir, archive, categories):
 763     """Make a repo index.
 764
 765     :param apps: fully populated apps list
 766     :param apks: full populated apks list
 767     :param repodir: the repo directory
 768     :param archive: True if this is the archive repo, False if it's the
 769                     main one.
 770     :param categories: list of categories
 771     """
 772
 773     doc = Document()
 774
 775     def addElement(name, value, doc, parent):
 776         el = doc.createElement(name)
 777         el.appendChild(doc.createTextNode(value))
 778         parent.appendChild(el)
 779
 780     def addElementNonEmpty(name, value, doc, parent):
 781         if not value:
 782             return
 783         addElement(name, value, doc, parent)
 784
 785     def addElementCDATA(name, value, doc, parent):
 786         el = doc.createElement(name)
 787         el.appendChild(doc.createCDATASection(value))
 788         parent.appendChild(el)
 789
 790     root = doc.createElement("fdroid")
 791     doc.appendChild(root)
 792
 793     repoel = doc.createElement("repo")
 794
 795     mirrorcheckfailed = False
 796     for mirror in config.get('mirrors', []):
 797         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
 798         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
 799             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 800             mirrorcheckfailed = True
 801     if mirrorcheckfailed:
 802         sys.exit(1)
 803
 804     if archive:
 805         repoel.setAttribute("name", config['archive_name'])
 806         if config['repo_maxage'] != 0:
 807             repoel.setAttribute("maxage", str(config['repo_maxage']))
 808         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 809         repoel.setAttribute("url", config['archive_url'])
 810         addElement('description', config['archive_description'], doc, repoel)
 811         urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
 812         for mirror in config.get('mirrors', []):
 813             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 814
 815     else:
 816         repoel.setAttribute("name", config['repo_name'])
 817         if config['repo_maxage'] != 0:
 818             repoel.setAttribute("maxage", str(config['repo_maxage']))
 819         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 820         repoel.setAttribute("url", config['repo_url'])
 821         addElement('description', config['repo_description'], doc, repoel)
 822         urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
 823         for mirror in config.get('mirrors', []):
 824             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 825
 826     repoel.setAttribute("version", str(METADATA_VERSION))
 827     repoel.setAttribute("timestamp", str(int(time.time())))
 828
 829     nosigningkey = False
 830     if not options.nosign:
 831         if 'repo_keyalias' not in config:
 832             nosigningkey = True
 833             logging.critical("'repo_keyalias' not found in config.py!")
 834         if 'keystore' not in config:
 835             nosigningkey = True
 836             logging.critical("'keystore' not found in config.py!")
 837         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 838             nosigningkey = True
 839             logging.critical("'keystorepass' not found in config.py!")
 840         if 'keypass' not in config and 'keypassfile' not in config:
 841             nosigningkey = True
 842             logging.critical("'keypass' not found in config.py!")
 843         if not os.path.exists(config['keystore']):
 844             nosigningkey = True
 845             logging.critical("'" + config['keystore'] + "' does not exist!")
 846         if nosigningkey:
 847             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 848             logging.warning("\tfdroid update --create-key")
 849             sys.exit(1)
 850
 851     repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
 852     root.appendChild(repoel)
 853
 854     for appid in sortedids:
 855         app = apps[appid]
 856
 857         if app.Disabled is not None:
 858             continue
 859
 860         # Get a list of the apks for this app...
 861         apklist = []
 862         for apk in apks:
 863             if apk['id'] == appid:
 864                 apklist.append(apk)
 865
 866         if len(apklist) == 0:
 867             continue
 868
 869         apel = doc.createElement("application")
 870         apel.setAttribute("id", app.id)
 871         root.appendChild(apel)
 872
 873         addElement('id', app.id, doc, apel)
 874         if app.added:
 875             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
 876         if app.lastupdated:
 877             addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
 878         addElement('name', app.Name, doc, apel)
 879         addElement('summary', app.Summary, doc, apel)
 880         if app.icon:
 881             addElement('icon', app.icon, doc, apel)
 882
 883         def linkres(appid):
 884             if appid in apps:
 885                 return ("fdroid.app:" + appid, apps[appid].Name)
 886             raise MetaDataException("Cannot resolve app id " + appid)
 887
 888         addElement('desc',
 889                    metadata.description_html(app.Description, linkres),
 890                    doc, apel)
 891         addElement('license', app.License, doc, apel)
 892         if app.Categories:
 893             addElement('categories', ','.join(app.Categories), doc, apel)
 894             # We put the first (primary) category in LAST, which will have
 895             # the desired effect of making clients that only understand one
 896             # category see that one.
 897             addElement('category', app.Categories[0], doc, apel)
 898         addElement('web', app.WebSite, doc, apel)
 899         addElement('source', app.SourceCode, doc, apel)
 900         addElement('tracker', app.IssueTracker, doc, apel)
 901         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 902         addElementNonEmpty('author', app.AuthorName, doc, apel)
 903         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 904         addElementNonEmpty('donate', app.Donate, doc, apel)
 905         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 906         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 907         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 908
 909         # These elements actually refer to the current version (i.e. which
 910         # one is recommended. They are historically mis-named, and need
 911         # changing, but stay like this for now to support existing clients.
 912         addElement('marketversion', app.CurrentVersion, doc, apel)
 913         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 914
 915         if app.AntiFeatures:
 916             af = app.AntiFeatures
 917             if af:
 918                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 919         if app.Provides:
 920             pv = app.Provides.split(',')
 921             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 922         if app.RequiresRoot:
 923             addElement('requirements', 'root', doc, apel)
 924
 925         # Sort the apk list into version order, just so the web site
 926         # doesn't have to do any work by default...
 927         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 928
 929         # Check for duplicates - they will make the client unhappy...
 930         for i in range(len(apklist) - 1):
 931             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 932                 logging.critical("duplicate versions: '%s' - '%s'" % (
 933                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 934                 sys.exit(1)
 935
 936         current_version_code = 0
 937         current_version_file = None
 938         for apk in apklist:
 939             # find the APK for the "Current Version"
 940             if current_version_code < apk['versioncode']:
 941                 current_version_code = apk['versioncode']
 942             if current_version_code < int(app.CurrentVersionCode):
 943                 current_version_file = apk['apkname']
 944
 945             apkel = doc.createElement("package")
 946             apel.appendChild(apkel)
 947             addElement('version', apk['version'], doc, apkel)
 948             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 949             addElement('apkname', apk['apkname'], doc, apkel)
 950             if 'srcname' in apk:
 951                 addElement('srcname', apk['srcname'], doc, apkel)
 952             for hash_type in ['sha256']:
 953                 if hash_type not in apk:
 954                     continue
 955                 hashel = doc.createElement("hash")
 956                 hashel.setAttribute("type", hash_type)
 957                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 958                 apkel.appendChild(hashel)
 959             addElement('sig', apk['sig'], doc, apkel)
 960             addElement('size', str(apk['size']), doc, apkel)
 961             addElement('sdkver', str(apk['minSdkVersion']), doc, apkel)
 962             if 'targetSdkVersion' in apk:
 963                 addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
 964             if 'maxSdkVersion' in apk:
 965                 addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
 966             if 'added' in apk:
 967                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 968             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
 969             if 'nativecode' in apk:
 970                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 971             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
 972
 973         if current_version_file is not None \
 974                 and config['make_current_version_link'] \
 975                 and repodir == 'repo':  # only create these
 976             namefield = config['current_version_name_source']
 977             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
 978             apklinkname = sanitized_name + '.apk'
 979             current_version_path = os.path.join(repodir, current_version_file)
 980             if os.path.islink(apklinkname):
 981                 os.remove(apklinkname)
 982             os.symlink(current_version_path, apklinkname)
 983             # also symlink gpg signature, if it exists
 984             for extension in ('.asc', '.sig'):
 985                 sigfile_path = current_version_path + extension
 986                 if os.path.exists(sigfile_path):
 987                     siglinkname = apklinkname + extension
 988                     if os.path.islink(siglinkname):
 989                         os.remove(siglinkname)
 990                     os.symlink(sigfile_path, siglinkname)
 991
 992     if options.pretty:
 993         output = doc.toprettyxml(encoding='utf-8')
 994     else:
 995         output = doc.toxml(encoding='utf-8')
 996
 997     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
 998         f.write(output)
 999
1000     if 'repo_keyalias' in config:
1001
1002         if options.nosign:
1003             logging.info("Creating unsigned index in preparation for signing")
1004         else:
1005             logging.info("Creating signed index with this key (SHA256):")
1006             logging.info("%s" % repo_pubkey_fingerprint)
1007
1008         # Create a jar of the index...
1009         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
1010         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
1011         if p.returncode != 0:
1012             logging.critical("Failed to create {0}".format(jar_output))
1013             sys.exit(1)
1014
1015         # Sign the index...
1016         signed = os.path.join(repodir, 'index.jar')
1017         if options.nosign:
1018             # Remove old signed index if not signing
1019             if os.path.exists(signed):
1020                 os.remove(signed)
1021         else:
1022             args = [config['jarsigner'], '-keystore', config['keystore'],
1023                     '-storepass:file', config['keystorepassfile'],
1024                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1025                     signed, config['repo_keyalias']]
1026             if config['keystore'] == 'NONE':
1027                 args += config['smartcardoptions']
1028             else:  # smardcards never use -keypass
1029                 args += ['-keypass:file', config['keypassfile']]
1030             p = FDroidPopen(args)
1031             if p.returncode != 0:
1032                 logging.critical("Failed to sign index")
1033                 sys.exit(1)
1034
1035     # Copy the repo icon into the repo directory...
1036     icon_dir = os.path.join(repodir, 'icons')
1037     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1038     shutil.copyfile(config['repo_icon'], iconfilename)
1039
1040     # Write a category list in the repo to allow quick access...
1041     catdata = ''
1042     for cat in categories:
1043         catdata += cat + '\n'
1044     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1045         f.write(catdata)
1046
1047
1048 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1049
1050     for appid, app in apps.items():
1051
1052         if app.ArchivePolicy:
1053             keepversions = int(app.ArchivePolicy[:-9])
1054         else:
1055             keepversions = defaultkeepversions
1056
1057         def filter_apk_list_sorted(apk_list):
1058             res = []
1059             for apk in apk_list:
1060                 if apk['id'] == appid:
1061                     res.append(apk)
1062
1063             # Sort the apk list by version code. First is highest/newest.
1064             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1065
1066         def move_file(from_dir, to_dir, filename, ignore_missing):
1067             from_path = os.path.join(from_dir, filename)
1068             if ignore_missing and not os.path.exists(from_path):
1069                 return
1070             to_path = os.path.join(to_dir, filename)
1071             shutil.move(from_path, to_path)
1072
1073         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1074                       .format(appid, len(apks), keepversions, len(archapks)))
1075
1076         if len(apks) > keepversions:
1077             apklist = filter_apk_list_sorted(apks)
1078             # Move back the ones we don't want.
1079             for apk in apklist[keepversions:]:
1080                 logging.info("Moving " + apk['apkname'] + " to archive")
1081                 move_file(repodir, archivedir, apk['apkname'], False)
1082                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1083                 for density in all_screen_densities:
1084                     repo_icon_dir = get_icon_dir(repodir, density)
1085                     archive_icon_dir = get_icon_dir(archivedir, density)
1086                     if density not in apk['icons']:
1087                         continue
1088                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1089                 if 'srcname' in apk:
1090                     move_file(repodir, archivedir, apk['srcname'], False)
1091                 archapks.append(apk)
1092                 apks.remove(apk)
1093         elif len(apks) < keepversions and len(archapks) > 0:
1094             required = keepversions - len(apks)
1095             archapklist = filter_apk_list_sorted(archapks)
1096             # Move forward the ones we want again.
1097             for apk in archapklist[:required]:
1098                 logging.info("Moving " + apk['apkname'] + " from archive")
1099                 move_file(archivedir, repodir, apk['apkname'], False)
1100                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1101                 for density in all_screen_densities:
1102                     repo_icon_dir = get_icon_dir(repodir, density)
1103                     archive_icon_dir = get_icon_dir(archivedir, density)
1104                     if density not in apk['icons']:
1105                         continue
1106                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1107                 if 'srcname' in apk:
1108                     move_file(archivedir, repodir, apk['srcname'], False)
1109                 archapks.remove(apk)
1110                 apks.append(apk)
1111
1112
1113 def add_apks_to_per_app_repos(repodir, apks):
1114     apks_per_app = dict()
1115     for apk in apks:
1116         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1117         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1118         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1119         apks_per_app[apk['id']] = apk
1120
1121         if not os.path.exists(apk['per_app_icons']):
1122             logging.info('Adding new repo for only ' + apk['id'])
1123             os.makedirs(apk['per_app_icons'])
1124
1125         apkpath = os.path.join(repodir, apk['apkname'])
1126         shutil.copy(apkpath, apk['per_app_repo'])
1127         apksigpath = apkpath + '.sig'
1128         if os.path.exists(apksigpath):
1129             shutil.copy(apksigpath, apk['per_app_repo'])
1130         apkascpath = apkpath + '.asc'
1131         if os.path.exists(apkascpath):
1132             shutil.copy(apkascpath, apk['per_app_repo'])
1133
1134
1135 config = None
1136 options = None
1137
1138
1139 def main():
1140
1141     global config, options
1142
1143     # Parse command line...
1144     parser = ArgumentParser()
1145     common.setup_global_opts(parser)
1146     parser.add_argument("--create-key", action="store_true", default=False,
1147                         help="Create a repo signing key in a keystore")
1148     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1149                         help="Create skeleton metadata files that are missing")
1150     parser.add_argument("--delete-unknown", action="store_true", default=False,
1151                         help="Delete APKs without metadata from the repo")
1152     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1153                         help="Report on build data status")
1154     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1155                         help="Interactively ask about things that need updating.")
1156     parser.add_argument("-I", "--icons", action="store_true", default=False,
1157                         help="Resize all the icons exceeding the max pixel size and exit")
1158     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1159                         help="Specify editor to use in interactive mode. Default " +
1160                         "is /etc/alternatives/editor")
1161     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1162                         help="Update the wiki")
1163     parser.add_argument("--pretty", action="store_true", default=False,
1164                         help="Produce human-readable index.xml")
1165     parser.add_argument("--clean", action="store_true", default=False,
1166                         help="Clean update - don't uses caches, reprocess all apks")
1167     parser.add_argument("--nosign", action="store_true", default=False,
1168                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1169     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1170                         help="Use date from apk instead of current time for newly added apks")
1171     options = parser.parse_args()
1172
1173     config = common.read_config(options)
1174
1175     if not ('jarsigner' in config and 'keytool' in config):
1176         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1177         sys.exit(1)
1178
1179     repodirs = ['repo']
1180     if config['archive_older'] != 0:
1181         repodirs.append('archive')
1182         if not os.path.exists('archive'):
1183             os.mkdir('archive')
1184
1185     if options.icons:
1186         resize_all_icons(repodirs)
1187         sys.exit(0)
1188
1189     # check that icons exist now, rather than fail at the end of `fdroid update`
1190     for k in ['repo_icon', 'archive_icon']:
1191         if k in config:
1192             if not os.path.exists(config[k]):
1193                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1194                 sys.exit(1)
1195
1196     # if the user asks to create a keystore, do it now, reusing whatever it can
1197     if options.create_key:
1198         if os.path.exists(config['keystore']):
1199             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1200             logging.critical("\t'" + config['keystore'] + "'")
1201             sys.exit(1)
1202
1203         if 'repo_keyalias' not in config:
1204             config['repo_keyalias'] = socket.getfqdn()
1205             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1206         if 'keydname' not in config:
1207             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1208             common.write_to_config(config, 'keydname', config['keydname'])
1209         if 'keystore' not in config:
1210             config['keystore'] = common.default_config.keystore
1211             common.write_to_config(config, 'keystore', config['keystore'])
1212
1213         password = common.genpassword()
1214         if 'keystorepass' not in config:
1215             config['keystorepass'] = password
1216             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1217         if 'keypass' not in config:
1218             config['keypass'] = password
1219             common.write_to_config(config, 'keypass', config['keypass'])
1220         common.genkeystore(config)
1221
1222     # Get all apps...
1223     apps = metadata.read_metadata()
1224
1225     # Generate a list of categories...
1226     categories = set()
1227     for app in apps.values():
1228         categories.update(app.Categories)
1229
1230     # Read known apks data (will be updated and written back when we've finished)
1231     knownapks = common.KnownApks()
1232
1233     # Gather information about all the apk files in the repo directory, using
1234     # cached data if possible.
1235     apkcachefile = os.path.join('tmp', 'apkcache')
1236     if not options.clean and os.path.exists(apkcachefile):
1237         with open(apkcachefile, 'rb') as cf:
1238             apkcache = pickle.load(cf, encoding='utf-8')
1239         if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
1240             apkcache = {}
1241     else:
1242         apkcache = {}
1243
1244     delete_disabled_builds(apps, apkcache, repodirs)
1245
1246     # Scan all apks in the main repo
1247     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1248
1249     # Generate warnings for apk's with no metadata (or create skeleton
1250     # metadata files, if requested on the command line)
1251     newmetadata = False
1252     for apk in apks:
1253         if apk['id'] not in apps:
1254             if options.create_metadata:
1255                 if 'name' not in apk:
1256                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1257                     continue
1258                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
1259                 f.write("License:Unknown\n")
1260                 f.write("Web Site:\n")
1261                 f.write("Source Code:\n")
1262                 f.write("Issue Tracker:\n")
1263                 f.write("Changelog:\n")
1264                 f.write("Summary:" + apk['name'] + "\n")
1265                 f.write("Description:\n")
1266                 f.write(apk['name'] + "\n")
1267                 f.write(".\n")
1268                 f.close()
1269                 logging.info("Generated skeleton metadata for " + apk['id'])
1270                 newmetadata = True
1271             else:
1272                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1273                 if options.delete_unknown:
1274                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1275                     rmf = os.path.join(repodirs[0], apk['apkname'])
1276                     if not os.path.exists(rmf):
1277                         logging.error("Could not find {0} to remove it".format(rmf))
1278                     else:
1279                         os.remove(rmf)
1280                 else:
1281                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1282
1283     # update the metadata with the newly created ones included
1284     if newmetadata:
1285         apps = metadata.read_metadata()
1286
1287     # Scan the archive repo for apks as well
1288     if len(repodirs) > 1:
1289         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1290         if cc:
1291             cachechanged = True
1292     else:
1293         archapks = []
1294
1295     # Some information from the apks needs to be applied up to the application
1296     # level. When doing this, we use the info from the most recent version's apk.
1297     # We deal with figuring out when the app was added and last updated at the
1298     # same time.
1299     for appid, app in apps.items():
1300         bestver = 0
1301         for apk in apks + archapks:
1302             if apk['id'] == appid:
1303                 if apk['versioncode'] > bestver:
1304                     bestver = apk['versioncode']
1305                     bestapk = apk
1306
1307                 if 'added' in apk:
1308                     if not app.added or apk['added'] < app.added:
1309                         app.added = apk['added']
1310                     if not app.lastupdated or apk['added'] > app.lastupdated:
1311                         app.lastupdated = apk['added']
1312
1313         if not app.added:
1314             logging.debug("Don't know when " + appid + " was added")
1315         if not app.lastupdated:
1316             logging.debug("Don't know when " + appid + " was last updated")
1317
1318         if bestver == 0:
1319             if app.Name is None:
1320                 app.Name = app.AutoName or appid
1321             app.icon = None
1322             logging.debug("Application " + appid + " has no packages")
1323         else:
1324             if app.Name is None:
1325                 app.Name = bestapk['name']
1326             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1327             if app.CurrentVersionCode is None:
1328                 app.CurrentVersionCode = str(bestver)
1329
1330     # Sort the app list by name, then the web site doesn't have to by default.
1331     # (we had to wait until we'd scanned the apks to do this, because mostly the
1332     # name comes from there!)
1333     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1334
1335     # APKs are placed into multiple repos based on the app package, providing
1336     # per-app subscription feeds for nightly builds and things like it
1337     if config['per_app_repos']:
1338         add_apks_to_per_app_repos(repodirs[0], apks)
1339         for appid, app in apps.items():
1340             repodir = os.path.join(appid, 'fdroid', 'repo')
1341             appdict = dict()
1342             appdict[appid] = app
1343             if os.path.isdir(repodir):
1344                 make_index(appdict, [appid], apks, repodir, False, categories)
1345             else:
1346                 logging.info('Skipping index generation for ' + appid)
1347         return
1348
1349     if len(repodirs) > 1:
1350         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1351
1352     # Make the index for the main repo...
1353     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1354
1355     # If there's an archive repo,  make the index for it. We already scanned it
1356     # earlier on.
1357     if len(repodirs) > 1:
1358         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1359
1360     if config['update_stats']:
1361
1362         # Update known apks info...
1363         knownapks.writeifchanged()
1364
1365         # Generate latest apps data for widget
1366         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1367             data = ''
1368             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1369                 for line in f:
1370                     appid = line.rstrip()
1371                     data += appid + "\t"
1372                     app = apps[appid]
1373                     data += app.Name + "\t"
1374                     if app.icon is not None:
1375                         data += app.icon + "\t"
1376                     data += app.License + "\n"
1377             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1378                 f.write(data)
1379
1380     if cachechanged:
1381         apkcache["METADATA_VERSION"] = METADATA_VERSION
1382         with open(apkcachefile, 'wb') as cf:
1383             pickle.dump(apkcache, cf)
1384
1385     # Update the wiki...
1386     if options.wiki:
1387         update_wiki(apps, sortedids, apks + archapks)
1388
1389     logging.info("Finished.")
1390
1391 if __name__ == "__main__":
1392     main()