fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   5 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU Affero General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Affero General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Affero General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 import sys
  21 import os
  22 import shutil
  23 import glob
  24 import re
  25 import socket
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 import urllib.parse
  30 from datetime import datetime, timedelta
  31 from xml.dom.minidom import Document
  32 from argparse import ArgumentParser
  33 import time
  34 from pyasn1.error import PyAsn1Error
  35 from pyasn1.codec.der import decoder, encoder
  36 from pyasn1_modules import rfc2315
  37 from binascii import hexlify, unhexlify
  38
  39 from PIL import Image
  40 import logging
  41
  42 from . import common
  43 from . import metadata
  44 from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
  45 from .metadata import MetaDataException
  46
  47 METADATA_VERSION = 16
  48
  49 screen_densities = ['640', '480', '320', '240', '160', '120']
  50
  51 all_screen_densities = ['0'] + screen_densities
  52
  53
  54 def dpi_to_px(density):
  55     return (int(density) * 48) / 160
  56
  57
  58 def px_to_dpi(px):
  59     return (int(px) * 160) / 48
  60
  61
  62 def get_icon_dir(repodir, density):
  63     if density == '0':
  64         return os.path.join(repodir, "icons")
  65     return os.path.join(repodir, "icons-%s" % density)
  66
  67
  68 def get_icon_dirs(repodir):
  69     for density in screen_densities:
  70         yield get_icon_dir(repodir, density)
  71
  72
  73 def get_all_icon_dirs(repodir):
  74     for density in all_screen_densities:
  75         yield get_icon_dir(repodir, density)
  76
  77
  78 def update_wiki(apps, sortedids, apks):
  79     """Update the wiki
  80
  81     :param apps: fully populated list of all applications
  82     :param apks: all apks, except...
  83     """
  84     logging.info("Updating wiki")
  85     wikicat = 'Apps'
  86     wikiredircat = 'App Redirects'
  87     import mwclient
  88     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  89                          path=config['wiki_path'])
  90     site.login(config['wiki_user'], config['wiki_password'])
  91     generated_pages = {}
  92     generated_redirects = {}
  93
  94     for appid in sortedids:
  95         app = apps[appid]
  96
  97         wikidata = ''
  98         if app.Disabled:
  99             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 100         if app.AntiFeatures:
 101             for af in app.AntiFeatures:
 102                 wikidata += '{{AntiFeature|' + af + '}}\n'
 103         if app.RequiresRoot:
 104             requiresroot = 'Yes'
 105         else:
 106             requiresroot = 'No'
 107         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 108             appid,
 109             app.Name,
 110             time.strftime('%Y-%m-%d', app.added) if app.added else '',
 111             time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
 112             app.SourceCode,
 113             app.IssueTracker,
 114             app.WebSite,
 115             app.Changelog,
 116             app.Donate,
 117             app.FlattrID,
 118             app.Bitcoin,
 119             app.Litecoin,
 120             app.License,
 121             requiresroot,
 122             app.AuthorName,
 123             app.AuthorEmail)
 124
 125         if app.Provides:
 126             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 127
 128         wikidata += app.Summary
 129         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 130
 131         wikidata += "=Description=\n"
 132         wikidata += metadata.description_wiki(app.Description) + "\n"
 133
 134         wikidata += "=Maintainer Notes=\n"
 135         if app.MaintainerNotes:
 136             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 137         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 138
 139         # Get a list of all packages for this application...
 140         apklist = []
 141         gotcurrentver = False
 142         cantupdate = False
 143         buildfails = False
 144         for apk in apks:
 145             if apk['id'] == appid:
 146                 if str(apk['versioncode']) == app.CurrentVersionCode:
 147                     gotcurrentver = True
 148                 apklist.append(apk)
 149         # Include ones we can't build, as a special case...
 150         for build in app.builds:
 151             if build.disable:
 152                 if build.vercode == app.CurrentVersionCode:
 153                     cantupdate = True
 154                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 155                 apklist.append({'versioncode': int(build.vercode),
 156                                 'version': build.version,
 157                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 158                                 })
 159             else:
 160                 builtit = False
 161                 for apk in apklist:
 162                     if apk['versioncode'] == int(build.vercode):
 163                         builtit = True
 164                         break
 165                 if not builtit:
 166                     buildfails = True
 167                     apklist.append({'versioncode': int(build.vercode),
 168                                     'version': build.version,
 169                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
 170                                     })
 171         if app.CurrentVersionCode == '0':
 172             cantupdate = True
 173         # Sort with most recent first...
 174         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 175
 176         wikidata += "=Versions=\n"
 177         if len(apklist) == 0:
 178             wikidata += "We currently have no versions of this app available."
 179         elif not gotcurrentver:
 180             wikidata += "We don't have the current version of this app."
 181         else:
 182             wikidata += "We have the current version of this app."
 183         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 184         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 185         if len(app.NoSourceSince) > 0:
 186             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 187         if len(app.CurrentVersion) > 0:
 188             wikidata += "The current (recommended) version is " + app.CurrentVersion
 189             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 190         validapks = 0
 191         for apk in apklist:
 192             wikidata += "==" + apk['version'] + "==\n"
 193
 194             if 'buildproblem' in apk:
 195                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 196             else:
 197                 validapks += 1
 198                 wikidata += "This version is built and signed by "
 199                 if 'srcname' in apk:
 200                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 201                 else:
 202                     wikidata += "the original developer.\n\n"
 203             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 204
 205         wikidata += '\n[[Category:' + wikicat + ']]\n'
 206         if len(app.NoSourceSince) > 0:
 207             wikidata += '\n[[Category:Apps missing source code]]\n'
 208         if validapks == 0 and not app.Disabled:
 209             wikidata += '\n[[Category:Apps with no packages]]\n'
 210         if cantupdate and not app.Disabled:
 211             wikidata += "\n[[Category:Apps we cannot update]]\n"
 212         if buildfails and not app.Disabled:
 213             wikidata += "\n[[Category:Apps with failing builds]]\n"
 214         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 215             wikidata += '\n[[Category:Apps to Update]]\n'
 216         if app.Disabled:
 217             wikidata += '\n[[Category:Apps that are disabled]]\n'
 218         if app.UpdateCheckMode == 'None' and not app.Disabled:
 219             wikidata += '\n[[Category:Apps with no update check]]\n'
 220         for appcat in app.Categories:
 221             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 222
 223         # We can't have underscores in the page name, even if they're in
 224         # the package ID, because MediaWiki messes with them...
 225         pagename = appid.replace('_', ' ')
 226
 227         # Drop a trailing newline, because mediawiki is going to drop it anyway
 228         # and it we don't we'll think the page has changed when it hasn't...
 229         if wikidata.endswith('\n'):
 230             wikidata = wikidata[:-1]
 231
 232         generated_pages[pagename] = wikidata
 233
 234         # Make a redirect from the name to the ID too, unless there's
 235         # already an existing page with the name and it isn't a redirect.
 236         noclobber = False
 237         apppagename = app.Name.replace('_', ' ')
 238         apppagename = apppagename.replace('{', '')
 239         apppagename = apppagename.replace('}', ' ')
 240         apppagename = apppagename.replace(':', ' ')
 241         apppagename = apppagename.replace('[', ' ')
 242         apppagename = apppagename.replace(']', ' ')
 243         # Drop double spaces caused mostly by replacing ':' above
 244         apppagename = apppagename.replace('  ', ' ')
 245         for expagename in site.allpages(prefix=apppagename,
 246                                         filterredir='nonredirects',
 247                                         generator=False):
 248             if expagename == apppagename:
 249                 noclobber = True
 250         # Another reason not to make the redirect page is if the app name
 251         # is the same as it's ID, because that will overwrite the real page
 252         # with an redirect to itself! (Although it seems like an odd
 253         # scenario this happens a lot, e.g. where there is metadata but no
 254         # builds or binaries to extract a name from.
 255         if apppagename == pagename:
 256             noclobber = True
 257         if not noclobber:
 258             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 259
 260     for tcat, genp in [(wikicat, generated_pages),
 261                        (wikiredircat, generated_redirects)]:
 262         catpages = site.Pages['Category:' + tcat]
 263         existingpages = []
 264         for page in catpages:
 265             existingpages.append(page.name)
 266             if page.name in genp:
 267                 pagetxt = page.edit()
 268                 if pagetxt != genp[page.name]:
 269                     logging.debug("Updating modified page " + page.name)
 270                     page.save(genp[page.name], summary='Auto-updated')
 271                 else:
 272                     logging.debug("Page " + page.name + " is unchanged")
 273             else:
 274                 logging.warn("Deleting page " + page.name)
 275                 page.delete('No longer published')
 276         for pagename, text in genp.items():
 277             logging.debug("Checking " + pagename)
 278             if pagename not in existingpages:
 279                 logging.debug("Creating page " + pagename)
 280                 try:
 281                     newpage = site.Pages[pagename]
 282                     newpage.save(text, summary='Auto-created')
 283                 except:
 284                     logging.error("...FAILED to create page '{0}'".format(pagename))
 285
 286     # Purge server cache to ensure counts are up to date
 287     site.pages['Repository Maintenance'].purge()
 288
 289
 290 def delete_disabled_builds(apps, apkcache, repodirs):
 291     """Delete disabled build outputs.
 292
 293     :param apps: list of all applications, as per metadata.read_metadata
 294     :param apkcache: current apk cache information
 295     :param repodirs: the repo directories to process
 296     """
 297     for appid, app in apps.items():
 298         for build in app.builds:
 299             if not build.disable:
 300                 continue
 301             apkfilename = appid + '_' + str(build.vercode) + '.apk'
 302             iconfilename = "%s.%s.png" % (
 303                 appid,
 304                 build.vercode)
 305             for repodir in repodirs:
 306                 files = [
 307                     os.path.join(repodir, apkfilename),
 308                     os.path.join(repodir, apkfilename + '.asc'),
 309                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 310                 ]
 311                 for density in all_screen_densities:
 312                     repo_dir = get_icon_dir(repodir, density)
 313                     files.append(os.path.join(repo_dir, iconfilename))
 314
 315                 for f in files:
 316                     if os.path.exists(f):
 317                         logging.info("Deleting disabled build output " + f)
 318                         os.remove(f)
 319             if apkfilename in apkcache:
 320                 del apkcache[apkfilename]
 321
 322
 323 def resize_icon(iconpath, density):
 324
 325     if not os.path.isfile(iconpath):
 326         return
 327
 328     fp = None
 329     try:
 330         fp = open(iconpath, 'rb')
 331         im = Image.open(fp)
 332         size = dpi_to_px(density)
 333
 334         if any(length > size for length in im.size):
 335             oldsize = im.size
 336             im.thumbnail((size, size), Image.ANTIALIAS)
 337             logging.debug("%s was too large at %s - new size is %s" % (
 338                 iconpath, oldsize, im.size))
 339             im.save(iconpath, "PNG")
 340
 341     except Exception as e:
 342         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 343
 344     finally:
 345         if fp:
 346             fp.close()
 347
 348
 349 def resize_all_icons(repodirs):
 350     """Resize all icons that exceed the max size
 351
 352     :param repodirs: the repo directories to process
 353     """
 354     for repodir in repodirs:
 355         for density in screen_densities:
 356             icon_dir = get_icon_dir(repodir, density)
 357             icon_glob = os.path.join(icon_dir, '*.png')
 358             for iconpath in glob.glob(icon_glob):
 359                 resize_icon(iconpath, density)
 360
 361
 362 # A signature block file with a .DSA, .RSA, or .EC extension
 363 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 364
 365
 366 def getsig(apkpath):
 367     """ Get the signing certificate of an apk. To get the same md5 has that
 368     Android gets, we encode the .RSA certificate in a specific format and pass
 369     it hex-encoded to the md5 digest algorithm.
 370
 371     :param apkpath: path to the apk
 372     :returns: A string containing the md5 of the signature of the apk or None
 373               if an error occurred.
 374     """
 375
 376     cert = None
 377
 378     # verify the jar signature is correct
 379     args = [config['jarsigner'], '-verify', apkpath]
 380     p = FDroidPopen(args)
 381     if p.returncode != 0:
 382         logging.critical(apkpath + " has a bad signature!")
 383         return None
 384
 385     with zipfile.ZipFile(apkpath, 'r') as apk:
 386
 387         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 388
 389         if len(certs) < 1:
 390             logging.error("Found no signing certificates on %s" % apkpath)
 391             return None
 392         if len(certs) > 1:
 393             logging.error("Found multiple signing certificates on %s" % apkpath)
 394             return None
 395
 396         cert = apk.read(certs[0])
 397
 398     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 399     if content.getComponentByName('contentType') != rfc2315.signedData:
 400         logging.error("Unexpected format.")
 401         return None
 402
 403     content = decoder.decode(content.getComponentByName('content'),
 404                              asn1Spec=rfc2315.SignedData())[0]
 405     try:
 406         certificates = content.getComponentByName('certificates')
 407     except PyAsn1Error:
 408         logging.error("Certificates not found.")
 409         return None
 410
 411     cert_encoded = encoder.encode(certificates)[4:]
 412
 413     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 414
 415
 416 def get_icon_bytes(apkzip, iconsrc):
 417     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 418     try:
 419         return apkzip.read(iconsrc)
 420     except KeyError:
 421         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 422
 423
 424 def sha256sum(filename):
 425     '''Calculate the sha256 of the given file'''
 426     sha = hashlib.sha256()
 427     with open(filename, 'rb') as f:
 428         while True:
 429             t = f.read(16384)
 430             if len(t) == 0:
 431                 break
 432             sha.update(t)
 433     return sha.hexdigest()
 434
 435
 436 def insert_obbs(repodir, apps, apks):
 437     """Scans the .obb files in a given repo directory and adds them to the
 438     relevant APK instances.  OBB files have versionCodes like APK
 439     files, and they are loosely associated.  If there is an OBB file
 440     present, then any APK with the same or higher versionCode will use
 441     that OBB file.  There are two OBB types: main and patch, each APK
 442     can only have only have one of each.
 443
 444     https://developer.android.com/google/play/expansion-files.html
 445
 446     :param repodir: repo directory to scan
 447     :param apps: list of current, valid apps
 448     :param apks: current information on all APKs
 449
 450     """
 451
 452     def obbWarnDelete(f, msg):
 453         logging.warning(msg + f)
 454         if options.delete_unknown:
 455             logging.error("Deleting unknown file: " + f)
 456             os.remove(f)
 457
 458     obbs = []
 459     java_Integer_MIN_VALUE = -pow(2, 31)
 460     for f in glob.glob(os.path.join(repodir, '*.obb')):
 461         obbfile = os.path.basename(f)
 462         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 463         chunks = obbfile.split('.')
 464         if chunks[0] != 'main' and chunks[0] != 'patch':
 465             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 466             continue
 467         if not re.match(r'^-?[0-9]+$', chunks[1]):
 468             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 469             continue
 470         versioncode = int(chunks[1])
 471         packagename = ".".join(chunks[2:-1])
 472
 473         highestVersionCode = java_Integer_MIN_VALUE
 474         if packagename not in apps.keys():
 475             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 476             continue
 477         for apk in apks:
 478             if packagename == apk['id'] and apk['versioncode'] > highestVersionCode:
 479                 highestVersionCode = apk['versioncode']
 480         if versioncode > highestVersionCode:
 481             obbWarnDelete(f, 'OBB file has newer versioncode(' + str(versioncode)
 482                           + ') than any APK: ')
 483             continue
 484         obbsha256 = sha256sum(f)
 485         obbs.append((packagename, versioncode, obbfile, obbsha256))
 486
 487     for apk in apks:
 488         for (packagename, versioncode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 489             if versioncode <= apk['versioncode'] and packagename == apk['id']:
 490                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 491                     apk['obbMainFile'] = obbfile
 492                     apk['obbMainFileSha256'] = obbsha256
 493                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 494                     apk['obbPatchFile'] = obbfile
 495                     apk['obbPatchFileSha256'] = obbsha256
 496             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 497                 break
 498
 499
 500 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
 501     """Scan the apks in the given repo directory.
 502
 503     This also extracts the icons.
 504
 505     :param apps: list of all applications, as per metadata.read_metadata
 506     :param apkcache: current apk cache information
 507     :param repodir: repo directory to scan
 508     :param knownapks: known apks info
 509     :param use_date_from_apk: use date from APK (instead of current date)
 510                               for newly added APKs
 511     :returns: (apks, cachechanged) where apks is a list of apk information,
 512               and cachechanged is True if the apkcache got changed.
 513     """
 514
 515     cachechanged = False
 516
 517     for icon_dir in get_all_icon_dirs(repodir):
 518         if os.path.exists(icon_dir):
 519             if options.clean:
 520                 shutil.rmtree(icon_dir)
 521                 os.makedirs(icon_dir)
 522         else:
 523             os.makedirs(icon_dir)
 524
 525     apks = []
 526     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 527     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 528     vername_pat = re.compile(".*versionName='([^']*)'.*")
 529     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 530     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 531     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 532     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 533     string_pat = re.compile(".* name='([^']*)'.*")
 534     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 535
 536         apkfilename = apkfile[len(repodir) + 1:]
 537         if ' ' in apkfilename:
 538             logging.critical("Spaces in filenames are not allowed.")
 539             sys.exit(1)
 540
 541         shasum = sha256sum(apkfile)
 542
 543         usecache = False
 544         if apkfilename in apkcache:
 545             apk = apkcache[apkfilename]
 546             if apk['sha256'] == shasum:
 547                 logging.debug("Reading " + apkfilename + " from cache")
 548                 usecache = True
 549             else:
 550                 logging.debug("Ignoring stale cache data for " + apkfilename)
 551
 552         if not usecache:
 553             logging.debug("Processing " + apkfilename)
 554             apk = {}
 555             apk['apkname'] = apkfilename
 556             apk['sha256'] = shasum
 557             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 558             if os.path.exists(os.path.join(repodir, srcfilename)):
 559                 apk['srcname'] = srcfilename
 560             apk['size'] = os.path.getsize(apkfile)
 561             apk['permissions'] = set()
 562             apk['features'] = set()
 563             apk['icons_src'] = {}
 564             apk['icons'] = {}
 565             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 566             if p.returncode != 0:
 567                 if options.delete_unknown:
 568                     if os.path.exists(apkfile):
 569                         logging.error("Failed to get apk information, deleting " + apkfile)
 570                         os.remove(apkfile)
 571                     else:
 572                         logging.error("Could not find {0} to remove it".format(apkfile))
 573                 else:
 574                     logging.error("Failed to get apk information, skipping " + apkfile)
 575                 continue
 576             for line in p.output.splitlines():
 577                 if line.startswith("package:"):
 578                     try:
 579                         apk['id'] = re.match(name_pat, line).group(1)
 580                         apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
 581                         apk['version'] = re.match(vername_pat, line).group(1)
 582                     except Exception as e:
 583                         logging.error("Package matching failed: " + str(e))
 584                         logging.info("Line was: " + line)
 585                         sys.exit(1)
 586                 elif line.startswith("application:"):
 587                     apk['name'] = re.match(label_pat, line).group(1)
 588                     # Keep path to non-dpi icon in case we need it
 589                     match = re.match(icon_pat_nodpi, line)
 590                     if match:
 591                         apk['icons_src']['-1'] = match.group(1)
 592                 elif line.startswith("launchable-activity:"):
 593                     # Only use launchable-activity as fallback to application
 594                     if not apk['name']:
 595                         apk['name'] = re.match(label_pat, line).group(1)
 596                     if '-1' not in apk['icons_src']:
 597                         match = re.match(icon_pat_nodpi, line)
 598                         if match:
 599                             apk['icons_src']['-1'] = match.group(1)
 600                 elif line.startswith("application-icon-"):
 601                     match = re.match(icon_pat, line)
 602                     if match:
 603                         density = match.group(1)
 604                         path = match.group(2)
 605                         apk['icons_src'][density] = path
 606                 elif line.startswith("sdkVersion:"):
 607                     m = re.match(sdkversion_pat, line)
 608                     if m is None:
 609                         logging.error(line.replace('sdkVersion:', '')
 610                                       + ' is not a valid minSdkVersion!')
 611                     else:
 612                         apk['minSdkVersion'] = m.group(1)
 613                         # if target not set, default to min
 614                         if 'targetSdkVersion' not in apk:
 615                             apk['targetSdkVersion'] = m.group(1)
 616                 elif line.startswith("targetSdkVersion:"):
 617                     m = re.match(sdkversion_pat, line)
 618                     if m is None:
 619                         logging.error(line.replace('targetSdkVersion:', '')
 620                                       + ' is not a valid targetSdkVersion!')
 621                     else:
 622                         apk['targetSdkVersion'] = m.group(1)
 623                 elif line.startswith("maxSdkVersion:"):
 624                     apk['maxSdkVersion'] = re.match(sdkversion_pat, line).group(1)
 625                 elif line.startswith("native-code:"):
 626                     apk['nativecode'] = []
 627                     for arch in line[13:].split(' '):
 628                         apk['nativecode'].append(arch[1:-1])
 629                 elif line.startswith("uses-permission:"):
 630                     perm = re.match(string_pat, line).group(1)
 631                     if perm.startswith("android.permission."):
 632                         perm = perm[19:]
 633                     apk['permissions'].add(perm)
 634                 elif line.startswith("uses-feature:"):
 635                     perm = re.match(string_pat, line).group(1)
 636                     # Filter out this, it's only added with the latest SDK tools and
 637                     # causes problems for lots of apps.
 638                     if perm != "android.hardware.screen.portrait" \
 639                             and perm != "android.hardware.screen.landscape":
 640                         if perm.startswith("android.feature."):
 641                             perm = perm[16:]
 642                         apk['features'].add(perm)
 643
 644             if 'minSdkVersion' not in apk:
 645                 logging.warn("No SDK version information found in {0}".format(apkfile))
 646                 apk['minSdkVersion'] = 1
 647
 648             # Check for debuggable apks...
 649             if common.isApkDebuggable(apkfile, config):
 650                 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
 651
 652             # Get the signature (or md5 of, to be precise)...
 653             logging.debug('Getting signature of {0}'.format(apkfile))
 654             apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 655             if not apk['sig']:
 656                 logging.critical("Failed to get apk signature")
 657                 sys.exit(1)
 658
 659             apkzip = zipfile.ZipFile(apkfile, 'r')
 660
 661             # if an APK has files newer than the system time, suggest updating
 662             # the system clock.  This is useful for offline systems, used for
 663             # signing, which do not have another source of clock sync info. It
 664             # has to be more than 24 hours newer because ZIP/APK files do not
 665             # store timezone info
 666             manifest = apkzip.getinfo('AndroidManifest.xml')
 667             if manifest.date_time[1] == 0:  # month can't be zero
 668                 logging.debug('AndroidManifest.xml has no date')
 669             else:
 670                 dt_obj = datetime(*manifest.date_time)
 671                 checkdt = dt_obj - timedelta(1)
 672                 if datetime.today() < checkdt:
 673                     logging.warn('System clock is older than manifest in: '
 674                                  + apkfilename
 675                                  + '\nSet clock to that time using:\n'
 676                                  + 'sudo date -s "' + str(dt_obj) + '"')
 677
 678             iconfilename = "%s.%s.png" % (
 679                 apk['id'],
 680                 apk['versioncode'])
 681
 682             # Extract the icon file...
 683             empty_densities = []
 684             for density in screen_densities:
 685                 if density not in apk['icons_src']:
 686                     empty_densities.append(density)
 687                     continue
 688                 iconsrc = apk['icons_src'][density]
 689                 icon_dir = get_icon_dir(repodir, density)
 690                 icondest = os.path.join(icon_dir, iconfilename)
 691
 692                 try:
 693                     with open(icondest, 'wb') as f:
 694                         f.write(get_icon_bytes(apkzip, iconsrc))
 695                     apk['icons'][density] = iconfilename
 696
 697                 except:
 698                     logging.warn("Error retrieving icon file")
 699                     del apk['icons'][density]
 700                     del apk['icons_src'][density]
 701                     empty_densities.append(density)
 702
 703             if '-1' in apk['icons_src']:
 704                 iconsrc = apk['icons_src']['-1']
 705                 iconpath = os.path.join(
 706                     get_icon_dir(repodir, '0'), iconfilename)
 707                 with open(iconpath, 'wb') as f:
 708                     f.write(get_icon_bytes(apkzip, iconsrc))
 709                 try:
 710                     im = Image.open(iconpath)
 711                     dpi = px_to_dpi(im.size[0])
 712                     for density in screen_densities:
 713                         if density in apk['icons']:
 714                             break
 715                         if density == screen_densities[-1] or dpi >= int(density):
 716                             apk['icons'][density] = iconfilename
 717                             shutil.move(iconpath,
 718                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 719                             empty_densities.remove(density)
 720                             break
 721                 except Exception as e:
 722                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 723
 724             if apk['icons']:
 725                 apk['icon'] = iconfilename
 726
 727             apkzip.close()
 728
 729             # First try resizing down to not lose quality
 730             last_density = None
 731             for density in screen_densities:
 732                 if density not in empty_densities:
 733                     last_density = density
 734                     continue
 735                 if last_density is None:
 736                     continue
 737                 logging.debug("Density %s not available, resizing down from %s"
 738                               % (density, last_density))
 739
 740                 last_iconpath = os.path.join(
 741                     get_icon_dir(repodir, last_density), iconfilename)
 742                 iconpath = os.path.join(
 743                     get_icon_dir(repodir, density), iconfilename)
 744                 fp = None
 745                 try:
 746                     fp = open(last_iconpath, 'rb')
 747                     im = Image.open(fp)
 748
 749                     size = dpi_to_px(density)
 750
 751                     im.thumbnail((size, size), Image.ANTIALIAS)
 752                     im.save(iconpath, "PNG")
 753                     empty_densities.remove(density)
 754                 except:
 755                     logging.warning("Invalid image file at %s" % last_iconpath)
 756                 finally:
 757                     if fp:
 758                         fp.close()
 759
 760             # Then just copy from the highest resolution available
 761             last_density = None
 762             for density in reversed(screen_densities):
 763                 if density not in empty_densities:
 764                     last_density = density
 765                     continue
 766                 if last_density is None:
 767                     continue
 768                 logging.debug("Density %s not available, copying from lower density %s"
 769                               % (density, last_density))
 770
 771                 shutil.copyfile(
 772                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 773                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 774
 775                 empty_densities.remove(density)
 776
 777             for density in screen_densities:
 778                 icon_dir = get_icon_dir(repodir, density)
 779                 icondest = os.path.join(icon_dir, iconfilename)
 780                 resize_icon(icondest, density)
 781
 782             # Copy from icons-mdpi to icons since mdpi is the baseline density
 783             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 784             if os.path.isfile(baseline):
 785                 apk['icons']['0'] = iconfilename
 786                 shutil.copyfile(baseline,
 787                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 788
 789             if use_date_from_apk and manifest.date_time[1] != 0:
 790                 default_date_param = datetime(*manifest.date_time).utctimetuple()
 791             else:
 792                 default_date_param = None
 793
 794             # Record in known apks, getting the added date at the same time..
 795             added = knownapks.recordapk(apk['apkname'], apk['id'], default_date=default_date_param)
 796             if added:
 797                 apk['added'] = added
 798
 799             apkcache[apkfilename] = apk
 800             cachechanged = True
 801
 802         apks.append(apk)
 803
 804     return apks, cachechanged
 805
 806
 807 repo_pubkey_fingerprint = None
 808
 809
 810 # Generate a certificate fingerprint the same way keytool does it
 811 # (but with slightly different formatting)
 812 def cert_fingerprint(data):
 813     digest = hashlib.sha256(data).digest()
 814     ret = []
 815     ret.append(' '.join("%02X" % b for b in bytearray(digest)))
 816     return " ".join(ret)
 817
 818
 819 def extract_pubkey():
 820     global repo_pubkey_fingerprint
 821     if 'repo_pubkey' in config:
 822         pubkey = unhexlify(config['repo_pubkey'])
 823     else:
 824         p = FDroidPopenBytes([config['keytool'], '-exportcert',
 825                               '-alias', config['repo_keyalias'],
 826                               '-keystore', config['keystore'],
 827                               '-storepass:file', config['keystorepassfile']]
 828                              + config['smartcardoptions'],
 829                              output=False, stderr_to_stdout=False)
 830         if p.returncode != 0 or len(p.output) < 20:
 831             msg = "Failed to get repo pubkey!"
 832             if config['keystore'] == 'NONE':
 833                 msg += ' Is your crypto smartcard plugged in?'
 834             logging.critical(msg)
 835             sys.exit(1)
 836         pubkey = p.output
 837     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 838     return hexlify(pubkey)
 839
 840
 841 def make_index(apps, sortedids, apks, repodir, archive, categories):
 842     """Make a repo index.
 843
 844     :param apps: fully populated apps list
 845     :param apks: full populated apks list
 846     :param repodir: the repo directory
 847     :param archive: True if this is the archive repo, False if it's the
 848                     main one.
 849     :param categories: list of categories
 850     """
 851
 852     doc = Document()
 853
 854     def addElement(name, value, doc, parent):
 855         el = doc.createElement(name)
 856         el.appendChild(doc.createTextNode(value))
 857         parent.appendChild(el)
 858
 859     def addElementNonEmpty(name, value, doc, parent):
 860         if not value:
 861             return
 862         addElement(name, value, doc, parent)
 863
 864     def addElementCDATA(name, value, doc, parent):
 865         el = doc.createElement(name)
 866         el.appendChild(doc.createCDATASection(value))
 867         parent.appendChild(el)
 868
 869     root = doc.createElement("fdroid")
 870     doc.appendChild(root)
 871
 872     repoel = doc.createElement("repo")
 873
 874     mirrorcheckfailed = False
 875     for mirror in config.get('mirrors', []):
 876         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
 877         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
 878             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 879             mirrorcheckfailed = True
 880     if mirrorcheckfailed:
 881         sys.exit(1)
 882
 883     if archive:
 884         repoel.setAttribute("name", config['archive_name'])
 885         if config['repo_maxage'] != 0:
 886             repoel.setAttribute("maxage", str(config['repo_maxage']))
 887         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 888         repoel.setAttribute("url", config['archive_url'])
 889         addElement('description', config['archive_description'], doc, repoel)
 890         urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
 891         for mirror in config.get('mirrors', []):
 892             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 893
 894     else:
 895         repoel.setAttribute("name", config['repo_name'])
 896         if config['repo_maxage'] != 0:
 897             repoel.setAttribute("maxage", str(config['repo_maxage']))
 898         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 899         repoel.setAttribute("url", config['repo_url'])
 900         addElement('description', config['repo_description'], doc, repoel)
 901         urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
 902         for mirror in config.get('mirrors', []):
 903             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 904
 905     repoel.setAttribute("version", str(METADATA_VERSION))
 906     repoel.setAttribute("timestamp", str(int(time.time())))
 907
 908     nosigningkey = False
 909     if not options.nosign:
 910         if 'repo_keyalias' not in config:
 911             nosigningkey = True
 912             logging.critical("'repo_keyalias' not found in config.py!")
 913         if 'keystore' not in config:
 914             nosigningkey = True
 915             logging.critical("'keystore' not found in config.py!")
 916         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 917             nosigningkey = True
 918             logging.critical("'keystorepass' not found in config.py!")
 919         if 'keypass' not in config and 'keypassfile' not in config:
 920             nosigningkey = True
 921             logging.critical("'keypass' not found in config.py!")
 922         if not os.path.exists(config['keystore']):
 923             nosigningkey = True
 924             logging.critical("'" + config['keystore'] + "' does not exist!")
 925         if nosigningkey:
 926             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 927             logging.warning("\tfdroid update --create-key")
 928             sys.exit(1)
 929
 930     repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
 931     root.appendChild(repoel)
 932
 933     for appid in sortedids:
 934         app = apps[appid]
 935
 936         if app.Disabled is not None:
 937             continue
 938
 939         # Get a list of the apks for this app...
 940         apklist = []
 941         for apk in apks:
 942             if apk['id'] == appid:
 943                 apklist.append(apk)
 944
 945         if len(apklist) == 0:
 946             continue
 947
 948         apel = doc.createElement("application")
 949         apel.setAttribute("id", app.id)
 950         root.appendChild(apel)
 951
 952         addElement('id', app.id, doc, apel)
 953         if app.added:
 954             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
 955         if app.lastupdated:
 956             addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
 957         addElement('name', app.Name, doc, apel)
 958         addElement('summary', app.Summary, doc, apel)
 959         if app.icon:
 960             addElement('icon', app.icon, doc, apel)
 961
 962         def linkres(appid):
 963             if appid in apps:
 964                 return ("fdroid.app:" + appid, apps[appid].Name)
 965             raise MetaDataException("Cannot resolve app id " + appid)
 966
 967         addElement('desc',
 968                    metadata.description_html(app.Description, linkres),
 969                    doc, apel)
 970         addElement('license', app.License, doc, apel)
 971         if app.Categories:
 972             addElement('categories', ','.join(app.Categories), doc, apel)
 973             # We put the first (primary) category in LAST, which will have
 974             # the desired effect of making clients that only understand one
 975             # category see that one.
 976             addElement('category', app.Categories[0], doc, apel)
 977         addElement('web', app.WebSite, doc, apel)
 978         addElement('source', app.SourceCode, doc, apel)
 979         addElement('tracker', app.IssueTracker, doc, apel)
 980         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 981         addElementNonEmpty('author', app.AuthorName, doc, apel)
 982         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 983         addElementNonEmpty('donate', app.Donate, doc, apel)
 984         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 985         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 986         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 987
 988         # These elements actually refer to the current version (i.e. which
 989         # one is recommended. They are historically mis-named, and need
 990         # changing, but stay like this for now to support existing clients.
 991         addElement('marketversion', app.CurrentVersion, doc, apel)
 992         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 993
 994         if app.AntiFeatures:
 995             af = app.AntiFeatures
 996             if af:
 997                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 998         if app.Provides:
 999             pv = app.Provides.split(',')
1000             addElementNonEmpty('provides', ','.join(pv), doc, apel)
1001         if app.RequiresRoot:
1002             addElement('requirements', 'root', doc, apel)
1003
1004         # Sort the apk list into version order, just so the web site
1005         # doesn't have to do any work by default...
1006         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
1007
1008         # Check for duplicates - they will make the client unhappy...
1009         for i in range(len(apklist) - 1):
1010             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
1011                 logging.critical("duplicate versions: '%s' - '%s'" % (
1012                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
1013                 sys.exit(1)
1014
1015         current_version_code = 0
1016         current_version_file = None
1017         for apk in apklist:
1018             # find the APK for the "Current Version"
1019             if current_version_code < apk['versioncode']:
1020                 current_version_code = apk['versioncode']
1021             if current_version_code < int(app.CurrentVersionCode):
1022                 current_version_file = apk['apkname']
1023
1024             apkel = doc.createElement("package")
1025             apel.appendChild(apkel)
1026             addElement('version', apk['version'], doc, apkel)
1027             addElement('versioncode', str(apk['versioncode']), doc, apkel)
1028             addElement('apkname', apk['apkname'], doc, apkel)
1029             if 'srcname' in apk:
1030                 addElement('srcname', apk['srcname'], doc, apkel)
1031             for hash_type in ['sha256']:
1032                 if hash_type not in apk:
1033                     continue
1034                 hashel = doc.createElement("hash")
1035                 hashel.setAttribute("type", hash_type)
1036                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
1037                 apkel.appendChild(hashel)
1038             addElement('sig', apk['sig'], doc, apkel)
1039             addElement('size', str(apk['size']), doc, apkel)
1040             addElement('sdkver', str(apk['minSdkVersion']), doc, apkel)
1041             if 'targetSdkVersion' in apk:
1042                 addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
1043             if 'maxSdkVersion' in apk:
1044                 addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
1045             addElementNonEmpty('obbMainFile', apk.get('obbMainFile'), doc, apkel)
1046             addElementNonEmpty('obbMainFileSha256', apk.get('obbMainFileSha256'), doc, apkel)
1047             addElementNonEmpty('obbPatchFile', apk.get('obbPatchFile'), doc, apkel)
1048             addElementNonEmpty('obbPatchFileSha256', apk.get('obbPatchFileSha256'), doc, apkel)
1049             if 'added' in apk:
1050                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
1051             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
1052             if 'nativecode' in apk:
1053                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
1054             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
1055
1056         if current_version_file is not None \
1057                 and config['make_current_version_link'] \
1058                 and repodir == 'repo':  # only create these
1059             namefield = config['current_version_name_source']
1060             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
1061             apklinkname = sanitized_name + '.apk'
1062             current_version_path = os.path.join(repodir, current_version_file)
1063             if os.path.islink(apklinkname):
1064                 os.remove(apklinkname)
1065             os.symlink(current_version_path, apklinkname)
1066             # also symlink gpg signature, if it exists
1067             for extension in ('.asc', '.sig'):
1068                 sigfile_path = current_version_path + extension
1069                 if os.path.exists(sigfile_path):
1070                     siglinkname = apklinkname + extension
1071                     if os.path.islink(siglinkname):
1072                         os.remove(siglinkname)
1073                     os.symlink(sigfile_path, siglinkname)
1074
1075     if options.pretty:
1076         output = doc.toprettyxml(encoding='utf-8')
1077     else:
1078         output = doc.toxml(encoding='utf-8')
1079
1080     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
1081         f.write(output)
1082
1083     if 'repo_keyalias' in config:
1084
1085         if options.nosign:
1086             logging.info("Creating unsigned index in preparation for signing")
1087         else:
1088             logging.info("Creating signed index with this key (SHA256):")
1089             logging.info("%s" % repo_pubkey_fingerprint)
1090
1091         # Create a jar of the index...
1092         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
1093         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
1094         if p.returncode != 0:
1095             logging.critical("Failed to create {0}".format(jar_output))
1096             sys.exit(1)
1097
1098         # Sign the index...
1099         signed = os.path.join(repodir, 'index.jar')
1100         if options.nosign:
1101             # Remove old signed index if not signing
1102             if os.path.exists(signed):
1103                 os.remove(signed)
1104         else:
1105             args = [config['jarsigner'], '-keystore', config['keystore'],
1106                     '-storepass:file', config['keystorepassfile'],
1107                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1108                     signed, config['repo_keyalias']]
1109             if config['keystore'] == 'NONE':
1110                 args += config['smartcardoptions']
1111             else:  # smardcards never use -keypass
1112                 args += ['-keypass:file', config['keypassfile']]
1113             p = FDroidPopen(args)
1114             if p.returncode != 0:
1115                 logging.critical("Failed to sign index")
1116                 sys.exit(1)
1117
1118     # Copy the repo icon into the repo directory...
1119     icon_dir = os.path.join(repodir, 'icons')
1120     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1121     shutil.copyfile(config['repo_icon'], iconfilename)
1122
1123     # Write a category list in the repo to allow quick access...
1124     catdata = ''
1125     for cat in categories:
1126         catdata += cat + '\n'
1127     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1128         f.write(catdata)
1129
1130
1131 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1132
1133     for appid, app in apps.items():
1134
1135         if app.ArchivePolicy:
1136             keepversions = int(app.ArchivePolicy[:-9])
1137         else:
1138             keepversions = defaultkeepversions
1139
1140         def filter_apk_list_sorted(apk_list):
1141             res = []
1142             for apk in apk_list:
1143                 if apk['id'] == appid:
1144                     res.append(apk)
1145
1146             # Sort the apk list by version code. First is highest/newest.
1147             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1148
1149         def move_file(from_dir, to_dir, filename, ignore_missing):
1150             from_path = os.path.join(from_dir, filename)
1151             if ignore_missing and not os.path.exists(from_path):
1152                 return
1153             to_path = os.path.join(to_dir, filename)
1154             shutil.move(from_path, to_path)
1155
1156         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1157                       .format(appid, len(apks), keepversions, len(archapks)))
1158
1159         if len(apks) > keepversions:
1160             apklist = filter_apk_list_sorted(apks)
1161             # Move back the ones we don't want.
1162             for apk in apklist[keepversions:]:
1163                 logging.info("Moving " + apk['apkname'] + " to archive")
1164                 move_file(repodir, archivedir, apk['apkname'], False)
1165                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1166                 for density in all_screen_densities:
1167                     repo_icon_dir = get_icon_dir(repodir, density)
1168                     archive_icon_dir = get_icon_dir(archivedir, density)
1169                     if density not in apk['icons']:
1170                         continue
1171                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1172                 if 'srcname' in apk:
1173                     move_file(repodir, archivedir, apk['srcname'], False)
1174                 archapks.append(apk)
1175                 apks.remove(apk)
1176         elif len(apks) < keepversions and len(archapks) > 0:
1177             required = keepversions - len(apks)
1178             archapklist = filter_apk_list_sorted(archapks)
1179             # Move forward the ones we want again.
1180             for apk in archapklist[:required]:
1181                 logging.info("Moving " + apk['apkname'] + " from archive")
1182                 move_file(archivedir, repodir, apk['apkname'], False)
1183                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1184                 for density in all_screen_densities:
1185                     repo_icon_dir = get_icon_dir(repodir, density)
1186                     archive_icon_dir = get_icon_dir(archivedir, density)
1187                     if density not in apk['icons']:
1188                         continue
1189                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1190                 if 'srcname' in apk:
1191                     move_file(archivedir, repodir, apk['srcname'], False)
1192                 archapks.remove(apk)
1193                 apks.append(apk)
1194
1195
1196 def add_apks_to_per_app_repos(repodir, apks):
1197     apks_per_app = dict()
1198     for apk in apks:
1199         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1200         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1201         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1202         apks_per_app[apk['id']] = apk
1203
1204         if not os.path.exists(apk['per_app_icons']):
1205             logging.info('Adding new repo for only ' + apk['id'])
1206             os.makedirs(apk['per_app_icons'])
1207
1208         apkpath = os.path.join(repodir, apk['apkname'])
1209         shutil.copy(apkpath, apk['per_app_repo'])
1210         apksigpath = apkpath + '.sig'
1211         if os.path.exists(apksigpath):
1212             shutil.copy(apksigpath, apk['per_app_repo'])
1213         apkascpath = apkpath + '.asc'
1214         if os.path.exists(apkascpath):
1215             shutil.copy(apkascpath, apk['per_app_repo'])
1216
1217
1218 config = None
1219 options = None
1220
1221
1222 def main():
1223
1224     global config, options
1225
1226     # Parse command line...
1227     parser = ArgumentParser()
1228     common.setup_global_opts(parser)
1229     parser.add_argument("--create-key", action="store_true", default=False,
1230                         help="Create a repo signing key in a keystore")
1231     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1232                         help="Create skeleton metadata files that are missing")
1233     parser.add_argument("--delete-unknown", action="store_true", default=False,
1234                         help="Delete APKs and/or OBBs without metadata from the repo")
1235     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1236                         help="Report on build data status")
1237     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1238                         help="Interactively ask about things that need updating.")
1239     parser.add_argument("-I", "--icons", action="store_true", default=False,
1240                         help="Resize all the icons exceeding the max pixel size and exit")
1241     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1242                         help="Specify editor to use in interactive mode. Default " +
1243                         "is /etc/alternatives/editor")
1244     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1245                         help="Update the wiki")
1246     parser.add_argument("--pretty", action="store_true", default=False,
1247                         help="Produce human-readable index.xml")
1248     parser.add_argument("--clean", action="store_true", default=False,
1249                         help="Clean update - don't uses caches, reprocess all apks")
1250     parser.add_argument("--nosign", action="store_true", default=False,
1251                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1252     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1253                         help="Use date from apk instead of current time for newly added apks")
1254     options = parser.parse_args()
1255
1256     config = common.read_config(options)
1257
1258     if not ('jarsigner' in config and 'keytool' in config):
1259         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1260         sys.exit(1)
1261
1262     repodirs = ['repo']
1263     if config['archive_older'] != 0:
1264         repodirs.append('archive')
1265         if not os.path.exists('archive'):
1266             os.mkdir('archive')
1267
1268     if options.icons:
1269         resize_all_icons(repodirs)
1270         sys.exit(0)
1271
1272     # check that icons exist now, rather than fail at the end of `fdroid update`
1273     for k in ['repo_icon', 'archive_icon']:
1274         if k in config:
1275             if not os.path.exists(config[k]):
1276                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1277                 sys.exit(1)
1278
1279     # if the user asks to create a keystore, do it now, reusing whatever it can
1280     if options.create_key:
1281         if os.path.exists(config['keystore']):
1282             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1283             logging.critical("\t'" + config['keystore'] + "'")
1284             sys.exit(1)
1285
1286         if 'repo_keyalias' not in config:
1287             config['repo_keyalias'] = socket.getfqdn()
1288             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1289         if 'keydname' not in config:
1290             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1291             common.write_to_config(config, 'keydname', config['keydname'])
1292         if 'keystore' not in config:
1293             config['keystore'] = common.default_config.keystore
1294             common.write_to_config(config, 'keystore', config['keystore'])
1295
1296         password = common.genpassword()
1297         if 'keystorepass' not in config:
1298             config['keystorepass'] = password
1299             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1300         if 'keypass' not in config:
1301             config['keypass'] = password
1302             common.write_to_config(config, 'keypass', config['keypass'])
1303         common.genkeystore(config)
1304
1305     # Get all apps...
1306     apps = metadata.read_metadata()
1307
1308     # Generate a list of categories...
1309     categories = set()
1310     for app in apps.values():
1311         categories.update(app.Categories)
1312
1313     # Read known apks data (will be updated and written back when we've finished)
1314     knownapks = common.KnownApks()
1315
1316     # Gather information about all the apk files in the repo directory, using
1317     # cached data if possible.
1318     apkcachefile = os.path.join('tmp', 'apkcache')
1319     if not options.clean and os.path.exists(apkcachefile):
1320         with open(apkcachefile, 'rb') as cf:
1321             apkcache = pickle.load(cf, encoding='utf-8')
1322         if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
1323             apkcache = {}
1324     else:
1325         apkcache = {}
1326
1327     delete_disabled_builds(apps, apkcache, repodirs)
1328
1329     # Scan all apks in the main repo
1330     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1331
1332     # Generate warnings for apk's with no metadata (or create skeleton
1333     # metadata files, if requested on the command line)
1334     newmetadata = False
1335     for apk in apks:
1336         if apk['id'] not in apps:
1337             if options.create_metadata:
1338                 if 'name' not in apk:
1339                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1340                     continue
1341                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
1342                 f.write("License:Unknown\n")
1343                 f.write("Web Site:\n")
1344                 f.write("Source Code:\n")
1345                 f.write("Issue Tracker:\n")
1346                 f.write("Changelog:\n")
1347                 f.write("Summary:" + apk['name'] + "\n")
1348                 f.write("Description:\n")
1349                 f.write(apk['name'] + "\n")
1350                 f.write(".\n")
1351                 f.close()
1352                 logging.info("Generated skeleton metadata for " + apk['id'])
1353                 newmetadata = True
1354             else:
1355                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1356                 if options.delete_unknown:
1357                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1358                     rmf = os.path.join(repodirs[0], apk['apkname'])
1359                     if not os.path.exists(rmf):
1360                         logging.error("Could not find {0} to remove it".format(rmf))
1361                     else:
1362                         os.remove(rmf)
1363                 else:
1364                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1365
1366     # update the metadata with the newly created ones included
1367     if newmetadata:
1368         apps = metadata.read_metadata()
1369
1370     insert_obbs(repodirs[0], apps, apks)
1371
1372     # Scan the archive repo for apks as well
1373     if len(repodirs) > 1:
1374         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1375         if cc:
1376             cachechanged = True
1377     else:
1378         archapks = []
1379
1380     # Some information from the apks needs to be applied up to the application
1381     # level. When doing this, we use the info from the most recent version's apk.
1382     # We deal with figuring out when the app was added and last updated at the
1383     # same time.
1384     for appid, app in apps.items():
1385         bestver = 0
1386         for apk in apks + archapks:
1387             if apk['id'] == appid:
1388                 if apk['versioncode'] > bestver:
1389                     bestver = apk['versioncode']
1390                     bestapk = apk
1391
1392                 if 'added' in apk:
1393                     if not app.added or apk['added'] < app.added:
1394                         app.added = apk['added']
1395                     if not app.lastupdated or apk['added'] > app.lastupdated:
1396                         app.lastupdated = apk['added']
1397
1398         if not app.added:
1399             logging.debug("Don't know when " + appid + " was added")
1400         if not app.lastupdated:
1401             logging.debug("Don't know when " + appid + " was last updated")
1402
1403         if bestver == 0:
1404             if app.Name is None:
1405                 app.Name = app.AutoName or appid
1406             app.icon = None
1407             logging.debug("Application " + appid + " has no packages")
1408         else:
1409             if app.Name is None:
1410                 app.Name = bestapk['name']
1411             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1412             if app.CurrentVersionCode is None:
1413                 app.CurrentVersionCode = str(bestver)
1414
1415     # Sort the app list by name, then the web site doesn't have to by default.
1416     # (we had to wait until we'd scanned the apks to do this, because mostly the
1417     # name comes from there!)
1418     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1419
1420     # APKs are placed into multiple repos based on the app package, providing
1421     # per-app subscription feeds for nightly builds and things like it
1422     if config['per_app_repos']:
1423         add_apks_to_per_app_repos(repodirs[0], apks)
1424         for appid, app in apps.items():
1425             repodir = os.path.join(appid, 'fdroid', 'repo')
1426             appdict = dict()
1427             appdict[appid] = app
1428             if os.path.isdir(repodir):
1429                 make_index(appdict, [appid], apks, repodir, False, categories)
1430             else:
1431                 logging.info('Skipping index generation for ' + appid)
1432         return
1433
1434     if len(repodirs) > 1:
1435         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1436
1437     # Make the index for the main repo...
1438     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1439
1440     # If there's an archive repo,  make the index for it. We already scanned it
1441     # earlier on.
1442     if len(repodirs) > 1:
1443         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1444
1445     if config['update_stats']:
1446
1447         # Update known apks info...
1448         knownapks.writeifchanged()
1449
1450         # Generate latest apps data for widget
1451         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1452             data = ''
1453             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1454                 for line in f:
1455                     appid = line.rstrip()
1456                     data += appid + "\t"
1457                     app = apps[appid]
1458                     data += app.Name + "\t"
1459                     if app.icon is not None:
1460                         data += app.icon + "\t"
1461                     data += app.License + "\n"
1462             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1463                 f.write(data)
1464
1465     if cachechanged:
1466         apkcache["METADATA_VERSION"] = METADATA_VERSION
1467         with open(apkcachefile, 'wb') as cf:
1468             pickle.dump(apkcache, cf)
1469
1470     # Update the wiki...
1471     if options.wiki:
1472         update_wiki(apps, sortedids, apks + archapks)
1473
1474     logging.info("Finished.")
1475
1476 if __name__ == "__main__":
1477     main()