fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   5 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU Affero General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Affero General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Affero General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 import sys
  21 import os
  22 import shutil
  23 import glob
  24 import re
  25 import socket
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 import urllib.parse
  30 from datetime import datetime, timedelta
  31 from xml.dom.minidom import Document
  32 from argparse import ArgumentParser
  33 import time
  34
  35 import collections
  36 from pyasn1.error import PyAsn1Error
  37 from pyasn1.codec.der import decoder, encoder
  38 from pyasn1_modules import rfc2315
  39 from binascii import hexlify, unhexlify
  40
  41 from PIL import Image
  42 import logging
  43
  44 from . import common
  45 from . import metadata
  46 from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
  47 from .metadata import MetaDataException
  48
  49 METADATA_VERSION = 17
  50
  51 screen_densities = ['640', '480', '320', '240', '160', '120']
  52
  53 all_screen_densities = ['0'] + screen_densities
  54
  55 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
  56 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
  57
  58
  59 def dpi_to_px(density):
  60     return (int(density) * 48) / 160
  61
  62
  63 def px_to_dpi(px):
  64     return (int(px) * 160) / 48
  65
  66
  67 def get_icon_dir(repodir, density):
  68     if density == '0':
  69         return os.path.join(repodir, "icons")
  70     return os.path.join(repodir, "icons-%s" % density)
  71
  72
  73 def get_icon_dirs(repodir):
  74     for density in screen_densities:
  75         yield get_icon_dir(repodir, density)
  76
  77
  78 def get_all_icon_dirs(repodir):
  79     for density in all_screen_densities:
  80         yield get_icon_dir(repodir, density)
  81
  82
  83 def update_wiki(apps, sortedids, apks):
  84     """Update the wiki
  85
  86     :param apps: fully populated list of all applications
  87     :param apks: all apks, except...
  88     """
  89     logging.info("Updating wiki")
  90     wikicat = 'Apps'
  91     wikiredircat = 'App Redirects'
  92     import mwclient
  93     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  94                          path=config['wiki_path'])
  95     site.login(config['wiki_user'], config['wiki_password'])
  96     generated_pages = {}
  97     generated_redirects = {}
  98
  99     for appid in sortedids:
 100         app = apps[appid]
 101
 102         wikidata = ''
 103         if app.Disabled:
 104             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 105         if app.AntiFeatures:
 106             for af in app.AntiFeatures:
 107                 wikidata += '{{AntiFeature|' + af + '}}\n'
 108         if app.RequiresRoot:
 109             requiresroot = 'Yes'
 110         else:
 111             requiresroot = 'No'
 112         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 113             appid,
 114             app.Name,
 115             time.strftime('%Y-%m-%d', app.added) if app.added else '',
 116             time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
 117             app.SourceCode,
 118             app.IssueTracker,
 119             app.WebSite,
 120             app.Changelog,
 121             app.Donate,
 122             app.FlattrID,
 123             app.Bitcoin,
 124             app.Litecoin,
 125             app.License,
 126             requiresroot,
 127             app.AuthorName,
 128             app.AuthorEmail)
 129
 130         if app.Provides:
 131             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 132
 133         wikidata += app.Summary
 134         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 135
 136         wikidata += "=Description=\n"
 137         wikidata += metadata.description_wiki(app.Description) + "\n"
 138
 139         wikidata += "=Maintainer Notes=\n"
 140         if app.MaintainerNotes:
 141             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 142         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 143
 144         # Get a list of all packages for this application...
 145         apklist = []
 146         gotcurrentver = False
 147         cantupdate = False
 148         buildfails = False
 149         for apk in apks:
 150             if apk['id'] == appid:
 151                 if str(apk['versioncode']) == app.CurrentVersionCode:
 152                     gotcurrentver = True
 153                 apklist.append(apk)
 154         # Include ones we can't build, as a special case...
 155         for build in app.builds:
 156             if build.disable:
 157                 if build.vercode == app.CurrentVersionCode:
 158                     cantupdate = True
 159                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 160                 apklist.append({'versioncode': int(build.vercode),
 161                                 'version': build.version,
 162                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 163                                 })
 164             else:
 165                 builtit = False
 166                 for apk in apklist:
 167                     if apk['versioncode'] == int(build.vercode):
 168                         builtit = True
 169                         break
 170                 if not builtit:
 171                     buildfails = True
 172                     apklist.append({'versioncode': int(build.vercode),
 173                                     'version': build.version,
 174                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
 175                                     })
 176         if app.CurrentVersionCode == '0':
 177             cantupdate = True
 178         # Sort with most recent first...
 179         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 180
 181         wikidata += "=Versions=\n"
 182         if len(apklist) == 0:
 183             wikidata += "We currently have no versions of this app available."
 184         elif not gotcurrentver:
 185             wikidata += "We don't have the current version of this app."
 186         else:
 187             wikidata += "We have the current version of this app."
 188         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 189         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 190         if len(app.NoSourceSince) > 0:
 191             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 192         if len(app.CurrentVersion) > 0:
 193             wikidata += "The current (recommended) version is " + app.CurrentVersion
 194             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 195         validapks = 0
 196         for apk in apklist:
 197             wikidata += "==" + apk['version'] + "==\n"
 198
 199             if 'buildproblem' in apk:
 200                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 201             else:
 202                 validapks += 1
 203                 wikidata += "This version is built and signed by "
 204                 if 'srcname' in apk:
 205                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 206                 else:
 207                     wikidata += "the original developer.\n\n"
 208             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 209
 210         wikidata += '\n[[Category:' + wikicat + ']]\n'
 211         if len(app.NoSourceSince) > 0:
 212             wikidata += '\n[[Category:Apps missing source code]]\n'
 213         if validapks == 0 and not app.Disabled:
 214             wikidata += '\n[[Category:Apps with no packages]]\n'
 215         if cantupdate and not app.Disabled:
 216             wikidata += "\n[[Category:Apps we cannot update]]\n"
 217         if buildfails and not app.Disabled:
 218             wikidata += "\n[[Category:Apps with failing builds]]\n"
 219         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 220             wikidata += '\n[[Category:Apps to Update]]\n'
 221         if app.Disabled:
 222             wikidata += '\n[[Category:Apps that are disabled]]\n'
 223         if app.UpdateCheckMode == 'None' and not app.Disabled:
 224             wikidata += '\n[[Category:Apps with no update check]]\n'
 225         for appcat in app.Categories:
 226             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 227
 228         # We can't have underscores in the page name, even if they're in
 229         # the package ID, because MediaWiki messes with them...
 230         pagename = appid.replace('_', ' ')
 231
 232         # Drop a trailing newline, because mediawiki is going to drop it anyway
 233         # and it we don't we'll think the page has changed when it hasn't...
 234         if wikidata.endswith('\n'):
 235             wikidata = wikidata[:-1]
 236
 237         generated_pages[pagename] = wikidata
 238
 239         # Make a redirect from the name to the ID too, unless there's
 240         # already an existing page with the name and it isn't a redirect.
 241         noclobber = False
 242         apppagename = app.Name.replace('_', ' ')
 243         apppagename = apppagename.replace('{', '')
 244         apppagename = apppagename.replace('}', ' ')
 245         apppagename = apppagename.replace(':', ' ')
 246         apppagename = apppagename.replace('[', ' ')
 247         apppagename = apppagename.replace(']', ' ')
 248         # Drop double spaces caused mostly by replacing ':' above
 249         apppagename = apppagename.replace('  ', ' ')
 250         for expagename in site.allpages(prefix=apppagename,
 251                                         filterredir='nonredirects',
 252                                         generator=False):
 253             if expagename == apppagename:
 254                 noclobber = True
 255         # Another reason not to make the redirect page is if the app name
 256         # is the same as it's ID, because that will overwrite the real page
 257         # with an redirect to itself! (Although it seems like an odd
 258         # scenario this happens a lot, e.g. where there is metadata but no
 259         # builds or binaries to extract a name from.
 260         if apppagename == pagename:
 261             noclobber = True
 262         if not noclobber:
 263             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 264
 265     for tcat, genp in [(wikicat, generated_pages),
 266                        (wikiredircat, generated_redirects)]:
 267         catpages = site.Pages['Category:' + tcat]
 268         existingpages = []
 269         for page in catpages:
 270             existingpages.append(page.name)
 271             if page.name in genp:
 272                 pagetxt = page.edit()
 273                 if pagetxt != genp[page.name]:
 274                     logging.debug("Updating modified page " + page.name)
 275                     page.save(genp[page.name], summary='Auto-updated')
 276                 else:
 277                     logging.debug("Page " + page.name + " is unchanged")
 278             else:
 279                 logging.warn("Deleting page " + page.name)
 280                 page.delete('No longer published')
 281         for pagename, text in genp.items():
 282             logging.debug("Checking " + pagename)
 283             if pagename not in existingpages:
 284                 logging.debug("Creating page " + pagename)
 285                 try:
 286                     newpage = site.Pages[pagename]
 287                     newpage.save(text, summary='Auto-created')
 288                 except:
 289                     logging.error("...FAILED to create page '{0}'".format(pagename))
 290
 291     # Purge server cache to ensure counts are up to date
 292     site.pages['Repository Maintenance'].purge()
 293
 294
 295 def delete_disabled_builds(apps, apkcache, repodirs):
 296     """Delete disabled build outputs.
 297
 298     :param apps: list of all applications, as per metadata.read_metadata
 299     :param apkcache: current apk cache information
 300     :param repodirs: the repo directories to process
 301     """
 302     for appid, app in apps.items():
 303         for build in app.builds:
 304             if not build.disable:
 305                 continue
 306             apkfilename = appid + '_' + str(build.vercode) + '.apk'
 307             iconfilename = "%s.%s.png" % (
 308                 appid,
 309                 build.vercode)
 310             for repodir in repodirs:
 311                 files = [
 312                     os.path.join(repodir, apkfilename),
 313                     os.path.join(repodir, apkfilename + '.asc'),
 314                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 315                 ]
 316                 for density in all_screen_densities:
 317                     repo_dir = get_icon_dir(repodir, density)
 318                     files.append(os.path.join(repo_dir, iconfilename))
 319
 320                 for f in files:
 321                     if os.path.exists(f):
 322                         logging.info("Deleting disabled build output " + f)
 323                         os.remove(f)
 324             if apkfilename in apkcache:
 325                 del apkcache[apkfilename]
 326
 327
 328 def resize_icon(iconpath, density):
 329
 330     if not os.path.isfile(iconpath):
 331         return
 332
 333     fp = None
 334     try:
 335         fp = open(iconpath, 'rb')
 336         im = Image.open(fp)
 337         size = dpi_to_px(density)
 338
 339         if any(length > size for length in im.size):
 340             oldsize = im.size
 341             im.thumbnail((size, size), Image.ANTIALIAS)
 342             logging.debug("%s was too large at %s - new size is %s" % (
 343                 iconpath, oldsize, im.size))
 344             im.save(iconpath, "PNG")
 345
 346     except Exception as e:
 347         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 348
 349     finally:
 350         if fp:
 351             fp.close()
 352
 353
 354 def resize_all_icons(repodirs):
 355     """Resize all icons that exceed the max size
 356
 357     :param repodirs: the repo directories to process
 358     """
 359     for repodir in repodirs:
 360         for density in screen_densities:
 361             icon_dir = get_icon_dir(repodir, density)
 362             icon_glob = os.path.join(icon_dir, '*.png')
 363             for iconpath in glob.glob(icon_glob):
 364                 resize_icon(iconpath, density)
 365
 366
 367 # A signature block file with a .DSA, .RSA, or .EC extension
 368 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 369
 370
 371 def getsig(apkpath):
 372     """ Get the signing certificate of an apk. To get the same md5 has that
 373     Android gets, we encode the .RSA certificate in a specific format and pass
 374     it hex-encoded to the md5 digest algorithm.
 375
 376     :param apkpath: path to the apk
 377     :returns: A string containing the md5 of the signature of the apk or None
 378               if an error occurred.
 379     """
 380
 381     cert = None
 382
 383     # verify the jar signature is correct
 384     args = [config['jarsigner'], '-verify', apkpath]
 385     p = FDroidPopen(args)
 386     if p.returncode != 0:
 387         logging.critical(apkpath + " has a bad signature!")
 388         return None
 389
 390     with zipfile.ZipFile(apkpath, 'r') as apk:
 391
 392         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 393
 394         if len(certs) < 1:
 395             logging.error("Found no signing certificates on %s" % apkpath)
 396             return None
 397         if len(certs) > 1:
 398             logging.error("Found multiple signing certificates on %s" % apkpath)
 399             return None
 400
 401         cert = apk.read(certs[0])
 402
 403     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 404     if content.getComponentByName('contentType') != rfc2315.signedData:
 405         logging.error("Unexpected format.")
 406         return None
 407
 408     content = decoder.decode(content.getComponentByName('content'),
 409                              asn1Spec=rfc2315.SignedData())[0]
 410     try:
 411         certificates = content.getComponentByName('certificates')
 412     except PyAsn1Error:
 413         logging.error("Certificates not found.")
 414         return None
 415
 416     cert_encoded = encoder.encode(certificates)[4:]
 417
 418     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 419
 420
 421 def get_icon_bytes(apkzip, iconsrc):
 422     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 423     try:
 424         return apkzip.read(iconsrc)
 425     except KeyError:
 426         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 427
 428
 429 def sha256sum(filename):
 430     '''Calculate the sha256 of the given file'''
 431     sha = hashlib.sha256()
 432     with open(filename, 'rb') as f:
 433         while True:
 434             t = f.read(16384)
 435             if len(t) == 0:
 436                 break
 437             sha.update(t)
 438     return sha.hexdigest()
 439
 440
 441 def insert_obbs(repodir, apps, apks):
 442     """Scans the .obb files in a given repo directory and adds them to the
 443     relevant APK instances.  OBB files have versionCodes like APK
 444     files, and they are loosely associated.  If there is an OBB file
 445     present, then any APK with the same or higher versionCode will use
 446     that OBB file.  There are two OBB types: main and patch, each APK
 447     can only have only have one of each.
 448
 449     https://developer.android.com/google/play/expansion-files.html
 450
 451     :param repodir: repo directory to scan
 452     :param apps: list of current, valid apps
 453     :param apks: current information on all APKs
 454
 455     """
 456
 457     def obbWarnDelete(f, msg):
 458         logging.warning(msg + f)
 459         if options.delete_unknown:
 460             logging.error("Deleting unknown file: " + f)
 461             os.remove(f)
 462
 463     obbs = []
 464     java_Integer_MIN_VALUE = -pow(2, 31)
 465     for f in glob.glob(os.path.join(repodir, '*.obb')):
 466         obbfile = os.path.basename(f)
 467         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 468         chunks = obbfile.split('.')
 469         if chunks[0] != 'main' and chunks[0] != 'patch':
 470             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 471             continue
 472         if not re.match(r'^-?[0-9]+$', chunks[1]):
 473             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 474             continue
 475         versioncode = int(chunks[1])
 476         packagename = ".".join(chunks[2:-1])
 477
 478         highestVersionCode = java_Integer_MIN_VALUE
 479         if packagename not in apps.keys():
 480             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 481             continue
 482         for apk in apks:
 483             if packagename == apk['id'] and apk['versioncode'] > highestVersionCode:
 484                 highestVersionCode = apk['versioncode']
 485         if versioncode > highestVersionCode:
 486             obbWarnDelete(f, 'OBB file has newer versioncode(' + str(versioncode)
 487                           + ') than any APK: ')
 488             continue
 489         obbsha256 = sha256sum(f)
 490         obbs.append((packagename, versioncode, obbfile, obbsha256))
 491
 492     for apk in apks:
 493         for (packagename, versioncode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 494             if versioncode <= apk['versioncode'] and packagename == apk['id']:
 495                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 496                     apk['obbMainFile'] = obbfile
 497                     apk['obbMainFileSha256'] = obbsha256
 498                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 499                     apk['obbPatchFile'] = obbfile
 500                     apk['obbPatchFileSha256'] = obbsha256
 501             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 502                 break
 503
 504
 505 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
 506     """Scan the apks in the given repo directory.
 507
 508     This also extracts the icons.
 509
 510     :param apps: list of all applications, as per metadata.read_metadata
 511     :param apkcache: current apk cache information
 512     :param repodir: repo directory to scan
 513     :param knownapks: known apks info
 514     :param use_date_from_apk: use date from APK (instead of current date)
 515                               for newly added APKs
 516     :returns: (apks, cachechanged) where apks is a list of apk information,
 517               and cachechanged is True if the apkcache got changed.
 518     """
 519
 520     cachechanged = False
 521
 522     for icon_dir in get_all_icon_dirs(repodir):
 523         if os.path.exists(icon_dir):
 524             if options.clean:
 525                 shutil.rmtree(icon_dir)
 526                 os.makedirs(icon_dir)
 527         else:
 528             os.makedirs(icon_dir)
 529
 530     apks = []
 531     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 532     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 533     vername_pat = re.compile(".*versionName='([^']*)'.*")
 534     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 535     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 536     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 537     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 538     permission_pat = re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
 539     feature_pat = re.compile(".*name='([^']*)'.*")
 540     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 541
 542         apkfilename = apkfile[len(repodir) + 1:]
 543         if ' ' in apkfilename:
 544             logging.critical("Spaces in filenames are not allowed.")
 545             sys.exit(1)
 546
 547         shasum = sha256sum(apkfile)
 548
 549         usecache = False
 550         if apkfilename in apkcache:
 551             apk = apkcache[apkfilename]
 552             if apk['sha256'] == shasum:
 553                 logging.debug("Reading " + apkfilename + " from cache")
 554                 usecache = True
 555             else:
 556                 logging.debug("Ignoring stale cache data for " + apkfilename)
 557
 558         if not usecache:
 559             logging.debug("Processing " + apkfilename)
 560             apk = {}
 561             apk['apkname'] = apkfilename
 562             apk['sha256'] = shasum
 563             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 564             if os.path.exists(os.path.join(repodir, srcfilename)):
 565                 apk['srcname'] = srcfilename
 566             apk['size'] = os.path.getsize(apkfile)
 567             apk['uses-permission'] = set()
 568             apk['uses-permission-sdk-23'] = set()
 569             apk['features'] = set()
 570             apk['icons_src'] = {}
 571             apk['icons'] = {}
 572             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 573             if p.returncode != 0:
 574                 if options.delete_unknown:
 575                     if os.path.exists(apkfile):
 576                         logging.error("Failed to get apk information, deleting " + apkfile)
 577                         os.remove(apkfile)
 578                     else:
 579                         logging.error("Could not find {0} to remove it".format(apkfile))
 580                 else:
 581                     logging.error("Failed to get apk information, skipping " + apkfile)
 582                 continue
 583             for line in p.output.splitlines():
 584                 if line.startswith("package:"):
 585                     try:
 586                         apk['id'] = re.match(name_pat, line).group(1)
 587                         apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
 588                         apk['version'] = re.match(vername_pat, line).group(1)
 589                     except Exception as e:
 590                         logging.error("Package matching failed: " + str(e))
 591                         logging.info("Line was: " + line)
 592                         sys.exit(1)
 593                 elif line.startswith("application:"):
 594                     apk['name'] = re.match(label_pat, line).group(1)
 595                     # Keep path to non-dpi icon in case we need it
 596                     match = re.match(icon_pat_nodpi, line)
 597                     if match:
 598                         apk['icons_src']['-1'] = match.group(1)
 599                 elif line.startswith("launchable-activity:"):
 600                     # Only use launchable-activity as fallback to application
 601                     if not apk['name']:
 602                         apk['name'] = re.match(label_pat, line).group(1)
 603                     if '-1' not in apk['icons_src']:
 604                         match = re.match(icon_pat_nodpi, line)
 605                         if match:
 606                             apk['icons_src']['-1'] = match.group(1)
 607                 elif line.startswith("application-icon-"):
 608                     match = re.match(icon_pat, line)
 609                     if match:
 610                         density = match.group(1)
 611                         path = match.group(2)
 612                         apk['icons_src'][density] = path
 613                 elif line.startswith("sdkVersion:"):
 614                     m = re.match(sdkversion_pat, line)
 615                     if m is None:
 616                         logging.error(line.replace('sdkVersion:', '')
 617                                       + ' is not a valid minSdkVersion!')
 618                     else:
 619                         apk['minSdkVersion'] = m.group(1)
 620                         # if target not set, default to min
 621                         if 'targetSdkVersion' not in apk:
 622                             apk['targetSdkVersion'] = m.group(1)
 623                 elif line.startswith("targetSdkVersion:"):
 624                     m = re.match(sdkversion_pat, line)
 625                     if m is None:
 626                         logging.error(line.replace('targetSdkVersion:', '')
 627                                       + ' is not a valid targetSdkVersion!')
 628                     else:
 629                         apk['targetSdkVersion'] = m.group(1)
 630                 elif line.startswith("maxSdkVersion:"):
 631                     apk['maxSdkVersion'] = re.match(sdkversion_pat, line).group(1)
 632                 elif line.startswith("native-code:"):
 633                     apk['nativecode'] = []
 634                     for arch in line[13:].split(' '):
 635                         apk['nativecode'].append(arch[1:-1])
 636                 elif line.startswith('uses-permission:'):
 637                     perm_match = re.match(permission_pat, line).groupdict()
 638
 639                     permission = UsesPermission(
 640                         perm_match['name'],
 641                         perm_match['maxSdkVersion']
 642                     )
 643
 644                     apk['uses-permission'].add(permission)
 645                 elif line.startswith('uses-permission-sdk-23:'):
 646                     perm_match = re.match(permission_pat, line).groupdict()
 647
 648                     permission_sdk_23 = UsesPermissionSdk23(
 649                         perm_match['name'],
 650                         perm_match['maxSdkVersion']
 651                     )
 652
 653                     apk['uses-permission-sdk-23'].add(permission_sdk_23)
 654
 655                 elif line.startswith('uses-feature:'):
 656                     feature = re.match(feature_pat, line).group(1)
 657                     # Filter out this, it's only added with the latest SDK tools and
 658                     # causes problems for lots of apps.
 659                     if feature != "android.hardware.screen.portrait" \
 660                             and feature != "android.hardware.screen.landscape":
 661                         if feature.startswith("android.feature."):
 662                             feature = feature[16:]
 663                         apk['features'].add(feature)
 664
 665             if 'minSdkVersion' not in apk:
 666                 logging.warn("No SDK version information found in {0}".format(apkfile))
 667                 apk['minSdkVersion'] = 1
 668
 669             # Check for debuggable apks...
 670             if common.isApkDebuggable(apkfile, config):
 671                 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
 672
 673             # Get the signature (or md5 of, to be precise)...
 674             logging.debug('Getting signature of {0}'.format(apkfile))
 675             apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 676             if not apk['sig']:
 677                 logging.critical("Failed to get apk signature")
 678                 sys.exit(1)
 679
 680             apkzip = zipfile.ZipFile(apkfile, 'r')
 681
 682             # if an APK has files newer than the system time, suggest updating
 683             # the system clock.  This is useful for offline systems, used for
 684             # signing, which do not have another source of clock sync info. It
 685             # has to be more than 24 hours newer because ZIP/APK files do not
 686             # store timezone info
 687             manifest = apkzip.getinfo('AndroidManifest.xml')
 688             if manifest.date_time[1] == 0:  # month can't be zero
 689                 logging.debug('AndroidManifest.xml has no date')
 690             else:
 691                 dt_obj = datetime(*manifest.date_time)
 692                 checkdt = dt_obj - timedelta(1)
 693                 if datetime.today() < checkdt:
 694                     logging.warn('System clock is older than manifest in: '
 695                                  + apkfilename
 696                                  + '\nSet clock to that time using:\n'
 697                                  + 'sudo date -s "' + str(dt_obj) + '"')
 698
 699             iconfilename = "%s.%s.png" % (
 700                 apk['id'],
 701                 apk['versioncode'])
 702
 703             # Extract the icon file...
 704             empty_densities = []
 705             for density in screen_densities:
 706                 if density not in apk['icons_src']:
 707                     empty_densities.append(density)
 708                     continue
 709                 iconsrc = apk['icons_src'][density]
 710                 icon_dir = get_icon_dir(repodir, density)
 711                 icondest = os.path.join(icon_dir, iconfilename)
 712
 713                 try:
 714                     with open(icondest, 'wb') as f:
 715                         f.write(get_icon_bytes(apkzip, iconsrc))
 716                     apk['icons'][density] = iconfilename
 717
 718                 except:
 719                     logging.warn("Error retrieving icon file")
 720                     del apk['icons'][density]
 721                     del apk['icons_src'][density]
 722                     empty_densities.append(density)
 723
 724             if '-1' in apk['icons_src']:
 725                 iconsrc = apk['icons_src']['-1']
 726                 iconpath = os.path.join(
 727                     get_icon_dir(repodir, '0'), iconfilename)
 728                 with open(iconpath, 'wb') as f:
 729                     f.write(get_icon_bytes(apkzip, iconsrc))
 730                 try:
 731                     im = Image.open(iconpath)
 732                     dpi = px_to_dpi(im.size[0])
 733                     for density in screen_densities:
 734                         if density in apk['icons']:
 735                             break
 736                         if density == screen_densities[-1] or dpi >= int(density):
 737                             apk['icons'][density] = iconfilename
 738                             shutil.move(iconpath,
 739                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 740                             empty_densities.remove(density)
 741                             break
 742                 except Exception as e:
 743                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 744
 745             if apk['icons']:
 746                 apk['icon'] = iconfilename
 747
 748             apkzip.close()
 749
 750             # First try resizing down to not lose quality
 751             last_density = None
 752             for density in screen_densities:
 753                 if density not in empty_densities:
 754                     last_density = density
 755                     continue
 756                 if last_density is None:
 757                     continue
 758                 logging.debug("Density %s not available, resizing down from %s"
 759                               % (density, last_density))
 760
 761                 last_iconpath = os.path.join(
 762                     get_icon_dir(repodir, last_density), iconfilename)
 763                 iconpath = os.path.join(
 764                     get_icon_dir(repodir, density), iconfilename)
 765                 fp = None
 766                 try:
 767                     fp = open(last_iconpath, 'rb')
 768                     im = Image.open(fp)
 769
 770                     size = dpi_to_px(density)
 771
 772                     im.thumbnail((size, size), Image.ANTIALIAS)
 773                     im.save(iconpath, "PNG")
 774                     empty_densities.remove(density)
 775                 except:
 776                     logging.warning("Invalid image file at %s" % last_iconpath)
 777                 finally:
 778                     if fp:
 779                         fp.close()
 780
 781             # Then just copy from the highest resolution available
 782             last_density = None
 783             for density in reversed(screen_densities):
 784                 if density not in empty_densities:
 785                     last_density = density
 786                     continue
 787                 if last_density is None:
 788                     continue
 789                 logging.debug("Density %s not available, copying from lower density %s"
 790                               % (density, last_density))
 791
 792                 shutil.copyfile(
 793                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 794                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 795
 796                 empty_densities.remove(density)
 797
 798             for density in screen_densities:
 799                 icon_dir = get_icon_dir(repodir, density)
 800                 icondest = os.path.join(icon_dir, iconfilename)
 801                 resize_icon(icondest, density)
 802
 803             # Copy from icons-mdpi to icons since mdpi is the baseline density
 804             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 805             if os.path.isfile(baseline):
 806                 apk['icons']['0'] = iconfilename
 807                 shutil.copyfile(baseline,
 808                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 809
 810             if use_date_from_apk and manifest.date_time[1] != 0:
 811                 default_date_param = datetime(*manifest.date_time).utctimetuple()
 812             else:
 813                 default_date_param = None
 814
 815             # Record in known apks, getting the added date at the same time..
 816             added = knownapks.recordapk(apk['apkname'], apk['id'], default_date=default_date_param)
 817             if added:
 818                 apk['added'] = added
 819
 820             apkcache[apkfilename] = apk
 821             cachechanged = True
 822
 823         apks.append(apk)
 824
 825     return apks, cachechanged
 826
 827
 828 repo_pubkey_fingerprint = None
 829
 830
 831 # Generate a certificate fingerprint the same way keytool does it
 832 # (but with slightly different formatting)
 833 def cert_fingerprint(data):
 834     digest = hashlib.sha256(data).digest()
 835     ret = []
 836     ret.append(' '.join("%02X" % b for b in bytearray(digest)))
 837     return " ".join(ret)
 838
 839
 840 def extract_pubkey():
 841     global repo_pubkey_fingerprint
 842     if 'repo_pubkey' in config:
 843         pubkey = unhexlify(config['repo_pubkey'])
 844     else:
 845         p = FDroidPopenBytes([config['keytool'], '-exportcert',
 846                               '-alias', config['repo_keyalias'],
 847                               '-keystore', config['keystore'],
 848                               '-storepass:file', config['keystorepassfile']]
 849                              + config['smartcardoptions'],
 850                              output=False, stderr_to_stdout=False)
 851         if p.returncode != 0 or len(p.output) < 20:
 852             msg = "Failed to get repo pubkey!"
 853             if config['keystore'] == 'NONE':
 854                 msg += ' Is your crypto smartcard plugged in?'
 855             logging.critical(msg)
 856             sys.exit(1)
 857         pubkey = p.output
 858     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 859     return hexlify(pubkey)
 860
 861
 862 def make_index(apps, sortedids, apks, repodir, archive, categories):
 863     """Make a repo index.
 864
 865     :param apps: fully populated apps list
 866     :param apks: full populated apks list
 867     :param repodir: the repo directory
 868     :param archive: True if this is the archive repo, False if it's the
 869                     main one.
 870     :param categories: list of categories
 871     """
 872
 873     doc = Document()
 874
 875     def addElement(name, value, doc, parent):
 876         el = doc.createElement(name)
 877         el.appendChild(doc.createTextNode(value))
 878         parent.appendChild(el)
 879
 880     def addElementNonEmpty(name, value, doc, parent):
 881         if not value:
 882             return
 883         addElement(name, value, doc, parent)
 884
 885     def addElementCDATA(name, value, doc, parent):
 886         el = doc.createElement(name)
 887         el.appendChild(doc.createCDATASection(value))
 888         parent.appendChild(el)
 889
 890     root = doc.createElement("fdroid")
 891     doc.appendChild(root)
 892
 893     repoel = doc.createElement("repo")
 894
 895     mirrorcheckfailed = False
 896     for mirror in config.get('mirrors', []):
 897         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
 898         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
 899             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 900             mirrorcheckfailed = True
 901     if mirrorcheckfailed:
 902         sys.exit(1)
 903
 904     if archive:
 905         repoel.setAttribute("name", config['archive_name'])
 906         if config['repo_maxage'] != 0:
 907             repoel.setAttribute("maxage", str(config['repo_maxage']))
 908         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 909         repoel.setAttribute("url", config['archive_url'])
 910         addElement('description', config['archive_description'], doc, repoel)
 911         urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
 912         for mirror in config.get('mirrors', []):
 913             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 914
 915     else:
 916         repoel.setAttribute("name", config['repo_name'])
 917         if config['repo_maxage'] != 0:
 918             repoel.setAttribute("maxage", str(config['repo_maxage']))
 919         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 920         repoel.setAttribute("url", config['repo_url'])
 921         addElement('description', config['repo_description'], doc, repoel)
 922         urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
 923         for mirror in config.get('mirrors', []):
 924             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 925
 926     repoel.setAttribute("version", str(METADATA_VERSION))
 927     repoel.setAttribute("timestamp", str(int(time.time())))
 928
 929     nosigningkey = False
 930     if not options.nosign:
 931         if 'repo_keyalias' not in config:
 932             nosigningkey = True
 933             logging.critical("'repo_keyalias' not found in config.py!")
 934         if 'keystore' not in config:
 935             nosigningkey = True
 936             logging.critical("'keystore' not found in config.py!")
 937         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 938             nosigningkey = True
 939             logging.critical("'keystorepass' not found in config.py!")
 940         if 'keypass' not in config and 'keypassfile' not in config:
 941             nosigningkey = True
 942             logging.critical("'keypass' not found in config.py!")
 943         if not os.path.exists(config['keystore']):
 944             nosigningkey = True
 945             logging.critical("'" + config['keystore'] + "' does not exist!")
 946         if nosigningkey:
 947             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 948             logging.warning("\tfdroid update --create-key")
 949             sys.exit(1)
 950
 951     repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
 952     root.appendChild(repoel)
 953
 954     for appid in sortedids:
 955         app = apps[appid]
 956
 957         if app.Disabled is not None:
 958             continue
 959
 960         # Get a list of the apks for this app...
 961         apklist = []
 962         for apk in apks:
 963             if apk['id'] == appid:
 964                 apklist.append(apk)
 965
 966         if len(apklist) == 0:
 967             continue
 968
 969         apel = doc.createElement("application")
 970         apel.setAttribute("id", app.id)
 971         root.appendChild(apel)
 972
 973         addElement('id', app.id, doc, apel)
 974         if app.added:
 975             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
 976         if app.lastupdated:
 977             addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
 978         addElement('name', app.Name, doc, apel)
 979         addElement('summary', app.Summary, doc, apel)
 980         if app.icon:
 981             addElement('icon', app.icon, doc, apel)
 982
 983         def linkres(appid):
 984             if appid in apps:
 985                 return ("fdroid.app:" + appid, apps[appid].Name)
 986             raise MetaDataException("Cannot resolve app id " + appid)
 987
 988         addElement('desc',
 989                    metadata.description_html(app.Description, linkres),
 990                    doc, apel)
 991         addElement('license', app.License, doc, apel)
 992         if app.Categories:
 993             addElement('categories', ','.join(app.Categories), doc, apel)
 994             # We put the first (primary) category in LAST, which will have
 995             # the desired effect of making clients that only understand one
 996             # category see that one.
 997             addElement('category', app.Categories[0], doc, apel)
 998         addElement('web', app.WebSite, doc, apel)
 999         addElement('source', app.SourceCode, doc, apel)
1000         addElement('tracker', app.IssueTracker, doc, apel)
1001         addElementNonEmpty('changelog', app.Changelog, doc, apel)
1002         addElementNonEmpty('author', app.AuthorName, doc, apel)
1003         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
1004         addElementNonEmpty('donate', app.Donate, doc, apel)
1005         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
1006         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
1007         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
1008
1009         # These elements actually refer to the current version (i.e. which
1010         # one is recommended. They are historically mis-named, and need
1011         # changing, but stay like this for now to support existing clients.
1012         addElement('marketversion', app.CurrentVersion, doc, apel)
1013         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
1014
1015         if app.AntiFeatures:
1016             af = app.AntiFeatures
1017             if af:
1018                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
1019         if app.Provides:
1020             pv = app.Provides.split(',')
1021             addElementNonEmpty('provides', ','.join(pv), doc, apel)
1022         if app.RequiresRoot:
1023             addElement('requirements', 'root', doc, apel)
1024
1025         # Sort the apk list into version order, just so the web site
1026         # doesn't have to do any work by default...
1027         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
1028
1029         # Check for duplicates - they will make the client unhappy...
1030         for i in range(len(apklist) - 1):
1031             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
1032                 logging.critical("duplicate versions: '%s' - '%s'" % (
1033                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
1034                 sys.exit(1)
1035
1036         current_version_code = 0
1037         current_version_file = None
1038         for apk in apklist:
1039             # find the APK for the "Current Version"
1040             if current_version_code < apk['versioncode']:
1041                 current_version_code = apk['versioncode']
1042             if current_version_code < int(app.CurrentVersionCode):
1043                 current_version_file = apk['apkname']
1044
1045             apkel = doc.createElement("package")
1046             apel.appendChild(apkel)
1047             addElement('version', apk['version'], doc, apkel)
1048             addElement('versioncode', str(apk['versioncode']), doc, apkel)
1049             addElement('apkname', apk['apkname'], doc, apkel)
1050             if 'srcname' in apk:
1051                 addElement('srcname', apk['srcname'], doc, apkel)
1052             for hash_type in ['sha256']:
1053                 if hash_type not in apk:
1054                     continue
1055                 hashel = doc.createElement("hash")
1056                 hashel.setAttribute("type", hash_type)
1057                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
1058                 apkel.appendChild(hashel)
1059             addElement('sig', apk['sig'], doc, apkel)
1060             addElement('size', str(apk['size']), doc, apkel)
1061             addElement('sdkver', str(apk['minSdkVersion']), doc, apkel)
1062             if 'targetSdkVersion' in apk:
1063                 addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
1064             if 'maxSdkVersion' in apk:
1065                 addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
1066             addElementNonEmpty('obbMainFile', apk.get('obbMainFile'), doc, apkel)
1067             addElementNonEmpty('obbMainFileSha256', apk.get('obbMainFileSha256'), doc, apkel)
1068             addElementNonEmpty('obbPatchFile', apk.get('obbPatchFile'), doc, apkel)
1069             addElementNonEmpty('obbPatchFileSha256', apk.get('obbPatchFileSha256'), doc, apkel)
1070             if 'added' in apk:
1071                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
1072
1073             # TODO: remove old permission format
1074             old_permissions = set()
1075             for perm in apk['uses-permission']:
1076                 perm_name = perm.name
1077                 if perm_name.startswith("android.permission."):
1078                     perm_name = perm_name[19:]
1079                 old_permissions.add(perm_name)
1080             addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel)
1081
1082             for permission in apk['uses-permission']:
1083                 permel = doc.createElement('uses-permission')
1084                 permel.setAttribute('name', permission.name)
1085                 if permission.maxSdkVersion is not None:
1086                     permel.setAttribute('maxSdkVersion', permission.maxSdkVersion)
1087                     apkel.appendChild(permel)
1088             for permission_sdk_23 in apk['uses-permission-sdk-23']:
1089                 permel = doc.createElement('uses-permission-sdk-23')
1090                 permel.setAttribute('name', permission_sdk_23.name)
1091                 if permission_sdk_23.maxSdkVersion is not None:
1092                     permel.setAttribute('maxSdkVersion', permission_sdk_23.maxSdkVersion)
1093                     apkel.appendChild(permel)
1094             if 'nativecode' in apk:
1095                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
1096             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
1097
1098         if current_version_file is not None \
1099                 and config['make_current_version_link'] \
1100                 and repodir == 'repo':  # only create these
1101             namefield = config['current_version_name_source']
1102             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
1103             apklinkname = sanitized_name + '.apk'
1104             current_version_path = os.path.join(repodir, current_version_file)
1105             if os.path.islink(apklinkname):
1106                 os.remove(apklinkname)
1107             os.symlink(current_version_path, apklinkname)
1108             # also symlink gpg signature, if it exists
1109             for extension in ('.asc', '.sig'):
1110                 sigfile_path = current_version_path + extension
1111                 if os.path.exists(sigfile_path):
1112                     siglinkname = apklinkname + extension
1113                     if os.path.islink(siglinkname):
1114                         os.remove(siglinkname)
1115                     os.symlink(sigfile_path, siglinkname)
1116
1117     if options.pretty:
1118         output = doc.toprettyxml(encoding='utf-8')
1119     else:
1120         output = doc.toxml(encoding='utf-8')
1121
1122     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
1123         f.write(output)
1124
1125     if 'repo_keyalias' in config:
1126
1127         if options.nosign:
1128             logging.info("Creating unsigned index in preparation for signing")
1129         else:
1130             logging.info("Creating signed index with this key (SHA256):")
1131             logging.info("%s" % repo_pubkey_fingerprint)
1132
1133         # Create a jar of the index...
1134         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
1135         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
1136         if p.returncode != 0:
1137             logging.critical("Failed to create {0}".format(jar_output))
1138             sys.exit(1)
1139
1140         # Sign the index...
1141         signed = os.path.join(repodir, 'index.jar')
1142         if options.nosign:
1143             # Remove old signed index if not signing
1144             if os.path.exists(signed):
1145                 os.remove(signed)
1146         else:
1147             args = [config['jarsigner'], '-keystore', config['keystore'],
1148                     '-storepass:file', config['keystorepassfile'],
1149                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1150                     signed, config['repo_keyalias']]
1151             if config['keystore'] == 'NONE':
1152                 args += config['smartcardoptions']
1153             else:  # smardcards never use -keypass
1154                 args += ['-keypass:file', config['keypassfile']]
1155             p = FDroidPopen(args)
1156             if p.returncode != 0:
1157                 logging.critical("Failed to sign index")
1158                 sys.exit(1)
1159
1160     # Copy the repo icon into the repo directory...
1161     icon_dir = os.path.join(repodir, 'icons')
1162     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1163     shutil.copyfile(config['repo_icon'], iconfilename)
1164
1165     # Write a category list in the repo to allow quick access...
1166     catdata = ''
1167     for cat in categories:
1168         catdata += cat + '\n'
1169     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1170         f.write(catdata)
1171
1172
1173 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1174
1175     for appid, app in apps.items():
1176
1177         if app.ArchivePolicy:
1178             keepversions = int(app.ArchivePolicy[:-9])
1179         else:
1180             keepversions = defaultkeepversions
1181
1182         def filter_apk_list_sorted(apk_list):
1183             res = []
1184             for apk in apk_list:
1185                 if apk['id'] == appid:
1186                     res.append(apk)
1187
1188             # Sort the apk list by version code. First is highest/newest.
1189             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1190
1191         def move_file(from_dir, to_dir, filename, ignore_missing):
1192             from_path = os.path.join(from_dir, filename)
1193             if ignore_missing and not os.path.exists(from_path):
1194                 return
1195             to_path = os.path.join(to_dir, filename)
1196             shutil.move(from_path, to_path)
1197
1198         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1199                       .format(appid, len(apks), keepversions, len(archapks)))
1200
1201         if len(apks) > keepversions:
1202             apklist = filter_apk_list_sorted(apks)
1203             # Move back the ones we don't want.
1204             for apk in apklist[keepversions:]:
1205                 logging.info("Moving " + apk['apkname'] + " to archive")
1206                 move_file(repodir, archivedir, apk['apkname'], False)
1207                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1208                 for density in all_screen_densities:
1209                     repo_icon_dir = get_icon_dir(repodir, density)
1210                     archive_icon_dir = get_icon_dir(archivedir, density)
1211                     if density not in apk['icons']:
1212                         continue
1213                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1214                 if 'srcname' in apk:
1215                     move_file(repodir, archivedir, apk['srcname'], False)
1216                 archapks.append(apk)
1217                 apks.remove(apk)
1218         elif len(apks) < keepversions and len(archapks) > 0:
1219             required = keepversions - len(apks)
1220             archapklist = filter_apk_list_sorted(archapks)
1221             # Move forward the ones we want again.
1222             for apk in archapklist[:required]:
1223                 logging.info("Moving " + apk['apkname'] + " from archive")
1224                 move_file(archivedir, repodir, apk['apkname'], False)
1225                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1226                 for density in all_screen_densities:
1227                     repo_icon_dir = get_icon_dir(repodir, density)
1228                     archive_icon_dir = get_icon_dir(archivedir, density)
1229                     if density not in apk['icons']:
1230                         continue
1231                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1232                 if 'srcname' in apk:
1233                     move_file(archivedir, repodir, apk['srcname'], False)
1234                 archapks.remove(apk)
1235                 apks.append(apk)
1236
1237
1238 def add_apks_to_per_app_repos(repodir, apks):
1239     apks_per_app = dict()
1240     for apk in apks:
1241         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1242         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1243         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1244         apks_per_app[apk['id']] = apk
1245
1246         if not os.path.exists(apk['per_app_icons']):
1247             logging.info('Adding new repo for only ' + apk['id'])
1248             os.makedirs(apk['per_app_icons'])
1249
1250         apkpath = os.path.join(repodir, apk['apkname'])
1251         shutil.copy(apkpath, apk['per_app_repo'])
1252         apksigpath = apkpath + '.sig'
1253         if os.path.exists(apksigpath):
1254             shutil.copy(apksigpath, apk['per_app_repo'])
1255         apkascpath = apkpath + '.asc'
1256         if os.path.exists(apkascpath):
1257             shutil.copy(apkascpath, apk['per_app_repo'])
1258
1259
1260 config = None
1261 options = None
1262
1263
1264 def main():
1265
1266     global config, options
1267
1268     # Parse command line...
1269     parser = ArgumentParser()
1270     common.setup_global_opts(parser)
1271     parser.add_argument("--create-key", action="store_true", default=False,
1272                         help="Create a repo signing key in a keystore")
1273     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1274                         help="Create skeleton metadata files that are missing")
1275     parser.add_argument("--delete-unknown", action="store_true", default=False,
1276                         help="Delete APKs and/or OBBs without metadata from the repo")
1277     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1278                         help="Report on build data status")
1279     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1280                         help="Interactively ask about things that need updating.")
1281     parser.add_argument("-I", "--icons", action="store_true", default=False,
1282                         help="Resize all the icons exceeding the max pixel size and exit")
1283     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1284                         help="Specify editor to use in interactive mode. Default " +
1285                         "is /etc/alternatives/editor")
1286     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1287                         help="Update the wiki")
1288     parser.add_argument("--pretty", action="store_true", default=False,
1289                         help="Produce human-readable index.xml")
1290     parser.add_argument("--clean", action="store_true", default=False,
1291                         help="Clean update - don't uses caches, reprocess all apks")
1292     parser.add_argument("--nosign", action="store_true", default=False,
1293                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1294     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1295                         help="Use date from apk instead of current time for newly added apks")
1296     options = parser.parse_args()
1297
1298     config = common.read_config(options)
1299
1300     if not ('jarsigner' in config and 'keytool' in config):
1301         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1302         sys.exit(1)
1303
1304     repodirs = ['repo']
1305     if config['archive_older'] != 0:
1306         repodirs.append('archive')
1307         if not os.path.exists('archive'):
1308             os.mkdir('archive')
1309
1310     if options.icons:
1311         resize_all_icons(repodirs)
1312         sys.exit(0)
1313
1314     # check that icons exist now, rather than fail at the end of `fdroid update`
1315     for k in ['repo_icon', 'archive_icon']:
1316         if k in config:
1317             if not os.path.exists(config[k]):
1318                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1319                 sys.exit(1)
1320
1321     # if the user asks to create a keystore, do it now, reusing whatever it can
1322     if options.create_key:
1323         if os.path.exists(config['keystore']):
1324             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1325             logging.critical("\t'" + config['keystore'] + "'")
1326             sys.exit(1)
1327
1328         if 'repo_keyalias' not in config:
1329             config['repo_keyalias'] = socket.getfqdn()
1330             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1331         if 'keydname' not in config:
1332             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1333             common.write_to_config(config, 'keydname', config['keydname'])
1334         if 'keystore' not in config:
1335             config['keystore'] = common.default_config.keystore
1336             common.write_to_config(config, 'keystore', config['keystore'])
1337
1338         password = common.genpassword()
1339         if 'keystorepass' not in config:
1340             config['keystorepass'] = password
1341             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1342         if 'keypass' not in config:
1343             config['keypass'] = password
1344             common.write_to_config(config, 'keypass', config['keypass'])
1345         common.genkeystore(config)
1346
1347     # Get all apps...
1348     apps = metadata.read_metadata()
1349
1350     # Generate a list of categories...
1351     categories = set()
1352     for app in apps.values():
1353         categories.update(app.Categories)
1354
1355     # Read known apks data (will be updated and written back when we've finished)
1356     knownapks = common.KnownApks()
1357
1358     # Gather information about all the apk files in the repo directory, using
1359     # cached data if possible.
1360     apkcachefile = os.path.join('tmp', 'apkcache')
1361     if not options.clean and os.path.exists(apkcachefile):
1362         with open(apkcachefile, 'rb') as cf:
1363             apkcache = pickle.load(cf, encoding='utf-8')
1364         if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
1365             apkcache = {}
1366     else:
1367         apkcache = {}
1368
1369     delete_disabled_builds(apps, apkcache, repodirs)
1370
1371     # Scan all apks in the main repo
1372     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1373
1374     # Generate warnings for apk's with no metadata (or create skeleton
1375     # metadata files, if requested on the command line)
1376     newmetadata = False
1377     for apk in apks:
1378         if apk['id'] not in apps:
1379             if options.create_metadata:
1380                 if 'name' not in apk:
1381                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1382                     continue
1383                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
1384                 f.write("License:Unknown\n")
1385                 f.write("Web Site:\n")
1386                 f.write("Source Code:\n")
1387                 f.write("Issue Tracker:\n")
1388                 f.write("Changelog:\n")
1389                 f.write("Summary:" + apk['name'] + "\n")
1390                 f.write("Description:\n")
1391                 f.write(apk['name'] + "\n")
1392                 f.write(".\n")
1393                 f.close()
1394                 logging.info("Generated skeleton metadata for " + apk['id'])
1395                 newmetadata = True
1396             else:
1397                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1398                 if options.delete_unknown:
1399                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1400                     rmf = os.path.join(repodirs[0], apk['apkname'])
1401                     if not os.path.exists(rmf):
1402                         logging.error("Could not find {0} to remove it".format(rmf))
1403                     else:
1404                         os.remove(rmf)
1405                 else:
1406                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1407
1408     # update the metadata with the newly created ones included
1409     if newmetadata:
1410         apps = metadata.read_metadata()
1411
1412     insert_obbs(repodirs[0], apps, apks)
1413
1414     # Scan the archive repo for apks as well
1415     if len(repodirs) > 1:
1416         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1417         if cc:
1418             cachechanged = True
1419     else:
1420         archapks = []
1421
1422     # Some information from the apks needs to be applied up to the application
1423     # level. When doing this, we use the info from the most recent version's apk.
1424     # We deal with figuring out when the app was added and last updated at the
1425     # same time.
1426     for appid, app in apps.items():
1427         bestver = 0
1428         for apk in apks + archapks:
1429             if apk['id'] == appid:
1430                 if apk['versioncode'] > bestver:
1431                     bestver = apk['versioncode']
1432                     bestapk = apk
1433
1434                 if 'added' in apk:
1435                     if not app.added or apk['added'] < app.added:
1436                         app.added = apk['added']
1437                     if not app.lastupdated or apk['added'] > app.lastupdated:
1438                         app.lastupdated = apk['added']
1439
1440         if not app.added:
1441             logging.debug("Don't know when " + appid + " was added")
1442         if not app.lastupdated:
1443             logging.debug("Don't know when " + appid + " was last updated")
1444
1445         if bestver == 0:
1446             if app.Name is None:
1447                 app.Name = app.AutoName or appid
1448             app.icon = None
1449             logging.debug("Application " + appid + " has no packages")
1450         else:
1451             if app.Name is None:
1452                 app.Name = bestapk['name']
1453             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1454             if app.CurrentVersionCode is None:
1455                 app.CurrentVersionCode = str(bestver)
1456
1457     # Sort the app list by name, then the web site doesn't have to by default.
1458     # (we had to wait until we'd scanned the apks to do this, because mostly the
1459     # name comes from there!)
1460     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1461
1462     # APKs are placed into multiple repos based on the app package, providing
1463     # per-app subscription feeds for nightly builds and things like it
1464     if config['per_app_repos']:
1465         add_apks_to_per_app_repos(repodirs[0], apks)
1466         for appid, app in apps.items():
1467             repodir = os.path.join(appid, 'fdroid', 'repo')
1468             appdict = dict()
1469             appdict[appid] = app
1470             if os.path.isdir(repodir):
1471                 make_index(appdict, [appid], apks, repodir, False, categories)
1472             else:
1473                 logging.info('Skipping index generation for ' + appid)
1474         return
1475
1476     if len(repodirs) > 1:
1477         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1478
1479     # Make the index for the main repo...
1480     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1481
1482     # If there's an archive repo,  make the index for it. We already scanned it
1483     # earlier on.
1484     if len(repodirs) > 1:
1485         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1486
1487     if config['update_stats']:
1488
1489         # Update known apks info...
1490         knownapks.writeifchanged()
1491
1492         # Generate latest apps data for widget
1493         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1494             data = ''
1495             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1496                 for line in f:
1497                     appid = line.rstrip()
1498                     data += appid + "\t"
1499                     app = apps[appid]
1500                     data += app.Name + "\t"
1501                     if app.icon is not None:
1502                         data += app.icon + "\t"
1503                     data += app.License + "\n"
1504             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1505                 f.write(data)
1506
1507     if cachechanged:
1508         apkcache["METADATA_VERSION"] = METADATA_VERSION
1509         with open(apkcachefile, 'wb') as cf:
1510             pickle.dump(apkcache, cf)
1511
1512     # Update the wiki...
1513     if options.wiki:
1514         update_wiki(apps, sortedids, apks + archapks)
1515
1516     logging.info("Finished.")
1517
1518 if __name__ == "__main__":
1519     main()