fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   5 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU Affero General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Affero General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Affero General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 import sys
  21 import os
  22 import shutil
  23 import glob
  24 import re
  25 import socket
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 import urllib.parse
  30 from datetime import datetime, timedelta
  31 from xml.dom.minidom import Document
  32 from argparse import ArgumentParser
  33 import time
  34 from pyasn1.error import PyAsn1Error
  35 from pyasn1.codec.der import decoder, encoder
  36 from pyasn1_modules import rfc2315
  37 from binascii import hexlify, unhexlify
  38
  39 from PIL import Image
  40 import logging
  41
  42 from . import common
  43 from . import metadata
  44 from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
  45 from .metadata import MetaDataException
  46
  47 METADATA_VERSION = 16
  48
  49 screen_densities = ['640', '480', '320', '240', '160', '120']
  50
  51 all_screen_densities = ['0'] + screen_densities
  52
  53
  54 def dpi_to_px(density):
  55     return (int(density) * 48) / 160
  56
  57
  58 def px_to_dpi(px):
  59     return (int(px) * 160) / 48
  60
  61
  62 def get_icon_dir(repodir, density):
  63     if density == '0':
  64         return os.path.join(repodir, "icons")
  65     return os.path.join(repodir, "icons-%s" % density)
  66
  67
  68 def get_icon_dirs(repodir):
  69     for density in screen_densities:
  70         yield get_icon_dir(repodir, density)
  71
  72
  73 def get_all_icon_dirs(repodir):
  74     for density in all_screen_densities:
  75         yield get_icon_dir(repodir, density)
  76
  77
  78 def update_wiki(apps, sortedids, apks):
  79     """Update the wiki
  80
  81     :param apps: fully populated list of all applications
  82     :param apks: all apks, except...
  83     """
  84     logging.info("Updating wiki")
  85     wikicat = 'Apps'
  86     wikiredircat = 'App Redirects'
  87     import mwclient
  88     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  89                          path=config['wiki_path'])
  90     site.login(config['wiki_user'], config['wiki_password'])
  91     generated_pages = {}
  92     generated_redirects = {}
  93
  94     for appid in sortedids:
  95         app = apps[appid]
  96
  97         wikidata = ''
  98         if app.Disabled:
  99             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 100         if app.AntiFeatures:
 101             for af in app.AntiFeatures:
 102                 wikidata += '{{AntiFeature|' + af + '}}\n'
 103         if app.RequiresRoot:
 104             requiresroot = 'Yes'
 105         else:
 106             requiresroot = 'No'
 107         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 108             appid,
 109             app.Name,
 110             time.strftime('%Y-%m-%d', app.added) if app.added else '',
 111             time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
 112             app.SourceCode,
 113             app.IssueTracker,
 114             app.WebSite,
 115             app.Changelog,
 116             app.Donate,
 117             app.FlattrID,
 118             app.Bitcoin,
 119             app.Litecoin,
 120             app.License,
 121             requiresroot,
 122             app.AuthorName,
 123             app.AuthorEmail)
 124
 125         if app.Provides:
 126             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 127
 128         wikidata += app.Summary
 129         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 130
 131         wikidata += "=Description=\n"
 132         wikidata += metadata.description_wiki(app.Description) + "\n"
 133
 134         wikidata += "=Maintainer Notes=\n"
 135         if app.MaintainerNotes:
 136             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 137         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 138
 139         # Get a list of all packages for this application...
 140         apklist = []
 141         gotcurrentver = False
 142         cantupdate = False
 143         buildfails = False
 144         for apk in apks:
 145             if apk['id'] == appid:
 146                 if str(apk['versioncode']) == app.CurrentVersionCode:
 147                     gotcurrentver = True
 148                 apklist.append(apk)
 149         # Include ones we can't build, as a special case...
 150         for build in app.builds:
 151             if build.disable:
 152                 if build.vercode == app.CurrentVersionCode:
 153                     cantupdate = True
 154                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 155                 apklist.append({'versioncode': int(build.vercode),
 156                                 'version': build.version,
 157                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 158                                 })
 159             else:
 160                 builtit = False
 161                 for apk in apklist:
 162                     if apk['versioncode'] == int(build.vercode):
 163                         builtit = True
 164                         break
 165                 if not builtit:
 166                     buildfails = True
 167                     apklist.append({'versioncode': int(build.vercode),
 168                                     'version': build.version,
 169                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
 170                                     })
 171         if app.CurrentVersionCode == '0':
 172             cantupdate = True
 173         # Sort with most recent first...
 174         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 175
 176         wikidata += "=Versions=\n"
 177         if len(apklist) == 0:
 178             wikidata += "We currently have no versions of this app available."
 179         elif not gotcurrentver:
 180             wikidata += "We don't have the current version of this app."
 181         else:
 182             wikidata += "We have the current version of this app."
 183         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 184         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 185         if len(app.NoSourceSince) > 0:
 186             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 187         if len(app.CurrentVersion) > 0:
 188             wikidata += "The current (recommended) version is " + app.CurrentVersion
 189             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 190         validapks = 0
 191         for apk in apklist:
 192             wikidata += "==" + apk['version'] + "==\n"
 193
 194             if 'buildproblem' in apk:
 195                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 196             else:
 197                 validapks += 1
 198                 wikidata += "This version is built and signed by "
 199                 if 'srcname' in apk:
 200                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 201                 else:
 202                     wikidata += "the original developer.\n\n"
 203             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 204
 205         wikidata += '\n[[Category:' + wikicat + ']]\n'
 206         if len(app.NoSourceSince) > 0:
 207             wikidata += '\n[[Category:Apps missing source code]]\n'
 208         if validapks == 0 and not app.Disabled:
 209             wikidata += '\n[[Category:Apps with no packages]]\n'
 210         if cantupdate and not app.Disabled:
 211             wikidata += "\n[[Category:Apps we cannot update]]\n"
 212         if buildfails and not app.Disabled:
 213             wikidata += "\n[[Category:Apps with failing builds]]\n"
 214         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 215             wikidata += '\n[[Category:Apps to Update]]\n'
 216         if app.Disabled:
 217             wikidata += '\n[[Category:Apps that are disabled]]\n'
 218         if app.UpdateCheckMode == 'None' and not app.Disabled:
 219             wikidata += '\n[[Category:Apps with no update check]]\n'
 220         for appcat in app.Categories:
 221             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 222
 223         # We can't have underscores in the page name, even if they're in
 224         # the package ID, because MediaWiki messes with them...
 225         pagename = appid.replace('_', ' ')
 226
 227         # Drop a trailing newline, because mediawiki is going to drop it anyway
 228         # and it we don't we'll think the page has changed when it hasn't...
 229         if wikidata.endswith('\n'):
 230             wikidata = wikidata[:-1]
 231
 232         generated_pages[pagename] = wikidata
 233
 234         # Make a redirect from the name to the ID too, unless there's
 235         # already an existing page with the name and it isn't a redirect.
 236         noclobber = False
 237         apppagename = app.Name.replace('_', ' ')
 238         apppagename = apppagename.replace('{', '')
 239         apppagename = apppagename.replace('}', ' ')
 240         apppagename = apppagename.replace(':', ' ')
 241         # Drop double spaces caused mostly by replacing ':' above
 242         apppagename = apppagename.replace('  ', ' ')
 243         for expagename in site.allpages(prefix=apppagename,
 244                                         filterredir='nonredirects',
 245                                         generator=False):
 246             if expagename == apppagename:
 247                 noclobber = True
 248         # Another reason not to make the redirect page is if the app name
 249         # is the same as it's ID, because that will overwrite the real page
 250         # with an redirect to itself! (Although it seems like an odd
 251         # scenario this happens a lot, e.g. where there is metadata but no
 252         # builds or binaries to extract a name from.
 253         if apppagename == pagename:
 254             noclobber = True
 255         if not noclobber:
 256             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 257
 258     for tcat, genp in [(wikicat, generated_pages),
 259                        (wikiredircat, generated_redirects)]:
 260         catpages = site.Pages['Category:' + tcat]
 261         existingpages = []
 262         for page in catpages:
 263             existingpages.append(page.name)
 264             if page.name in genp:
 265                 pagetxt = page.edit()
 266                 if pagetxt != genp[page.name]:
 267                     logging.debug("Updating modified page " + page.name)
 268                     page.save(genp[page.name], summary='Auto-updated')
 269                 else:
 270                     logging.debug("Page " + page.name + " is unchanged")
 271             else:
 272                 logging.warn("Deleting page " + page.name)
 273                 page.delete('No longer published')
 274         for pagename, text in genp.items():
 275             logging.debug("Checking " + pagename)
 276             if pagename not in existingpages:
 277                 logging.debug("Creating page " + pagename)
 278                 try:
 279                     newpage = site.Pages[pagename]
 280                     newpage.save(text, summary='Auto-created')
 281                 except:
 282                     logging.error("...FAILED to create page '{0}'".format(pagename))
 283
 284     # Purge server cache to ensure counts are up to date
 285     site.pages['Repository Maintenance'].purge()
 286
 287
 288 def delete_disabled_builds(apps, apkcache, repodirs):
 289     """Delete disabled build outputs.
 290
 291     :param apps: list of all applications, as per metadata.read_metadata
 292     :param apkcache: current apk cache information
 293     :param repodirs: the repo directories to process
 294     """
 295     for appid, app in apps.items():
 296         for build in app.builds:
 297             if not build.disable:
 298                 continue
 299             apkfilename = appid + '_' + str(build.vercode) + '.apk'
 300             iconfilename = "%s.%s.png" % (
 301                 appid,
 302                 build.vercode)
 303             for repodir in repodirs:
 304                 files = [
 305                     os.path.join(repodir, apkfilename),
 306                     os.path.join(repodir, apkfilename + '.asc'),
 307                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 308                 ]
 309                 for density in all_screen_densities:
 310                     repo_dir = get_icon_dir(repodir, density)
 311                     files.append(os.path.join(repo_dir, iconfilename))
 312
 313                 for f in files:
 314                     if os.path.exists(f):
 315                         logging.info("Deleting disabled build output " + f)
 316                         os.remove(f)
 317             if apkfilename in apkcache:
 318                 del apkcache[apkfilename]
 319
 320
 321 def resize_icon(iconpath, density):
 322
 323     if not os.path.isfile(iconpath):
 324         return
 325
 326     fp = None
 327     try:
 328         fp = open(iconpath, 'rb')
 329         im = Image.open(fp)
 330         size = dpi_to_px(density)
 331
 332         if any(length > size for length in im.size):
 333             oldsize = im.size
 334             im.thumbnail((size, size), Image.ANTIALIAS)
 335             logging.debug("%s was too large at %s - new size is %s" % (
 336                 iconpath, oldsize, im.size))
 337             im.save(iconpath, "PNG")
 338
 339     except Exception as e:
 340         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 341
 342     finally:
 343         if fp:
 344             fp.close()
 345
 346
 347 def resize_all_icons(repodirs):
 348     """Resize all icons that exceed the max size
 349
 350     :param repodirs: the repo directories to process
 351     """
 352     for repodir in repodirs:
 353         for density in screen_densities:
 354             icon_dir = get_icon_dir(repodir, density)
 355             icon_glob = os.path.join(icon_dir, '*.png')
 356             for iconpath in glob.glob(icon_glob):
 357                 resize_icon(iconpath, density)
 358
 359
 360 # A signature block file with a .DSA, .RSA, or .EC extension
 361 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 362
 363
 364 def getsig(apkpath):
 365     """ Get the signing certificate of an apk. To get the same md5 has that
 366     Android gets, we encode the .RSA certificate in a specific format and pass
 367     it hex-encoded to the md5 digest algorithm.
 368
 369     :param apkpath: path to the apk
 370     :returns: A string containing the md5 of the signature of the apk or None
 371               if an error occurred.
 372     """
 373
 374     cert = None
 375
 376     # verify the jar signature is correct
 377     args = [config['jarsigner'], '-verify', apkpath]
 378     p = FDroidPopen(args)
 379     if p.returncode != 0:
 380         logging.critical(apkpath + " has a bad signature!")
 381         return None
 382
 383     with zipfile.ZipFile(apkpath, 'r') as apk:
 384
 385         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 386
 387         if len(certs) < 1:
 388             logging.error("Found no signing certificates on %s" % apkpath)
 389             return None
 390         if len(certs) > 1:
 391             logging.error("Found multiple signing certificates on %s" % apkpath)
 392             return None
 393
 394         cert = apk.read(certs[0])
 395
 396     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 397     if content.getComponentByName('contentType') != rfc2315.signedData:
 398         logging.error("Unexpected format.")
 399         return None
 400
 401     content = decoder.decode(content.getComponentByName('content'),
 402                              asn1Spec=rfc2315.SignedData())[0]
 403     try:
 404         certificates = content.getComponentByName('certificates')
 405     except PyAsn1Error:
 406         logging.error("Certificates not found.")
 407         return None
 408
 409     cert_encoded = encoder.encode(certificates)[4:]
 410
 411     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 412
 413
 414 def get_icon_bytes(apkzip, iconsrc):
 415     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 416     try:
 417         return apkzip.read(iconsrc)
 418     except KeyError:
 419         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 420
 421
 422 def sha256sum(filename):
 423     '''Calculate the sha256 of the given file'''
 424     sha = hashlib.sha256()
 425     with open(filename, 'rb') as f:
 426         while True:
 427             t = f.read(16384)
 428             if len(t) == 0:
 429                 break
 430             sha.update(t)
 431     return sha.hexdigest()
 432
 433
 434 def insert_obbs(repodir, apps, apks):
 435     """Scans the .obb files in a given repo directory and adds them to the
 436     relevant APK instances.  OBB files have versionCodes like APK
 437     files, and they are loosely associated.  If there is an OBB file
 438     present, then any APK with the same or higher versionCode will use
 439     that OBB file.  There are two OBB types: main and patch, each APK
 440     can only have only have one of each.
 441
 442     https://developer.android.com/google/play/expansion-files.html
 443
 444     :param repodir: repo directory to scan
 445     :param apps: list of current, valid apps
 446     :param apks: current information on all APKs
 447
 448     """
 449
 450     def obbWarnDelete(f, msg):
 451         logging.warning(msg + f)
 452         if options.delete_unknown:
 453             logging.error("Deleting unknown file: " + f)
 454             os.remove(f)
 455
 456     obbs = []
 457     java_Integer_MIN_VALUE = -pow(2, 31)
 458     for f in glob.glob(os.path.join(repodir, '*.obb')):
 459         obbfile = os.path.basename(f)
 460         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 461         chunks = obbfile.split('.')
 462         if chunks[0] != 'main' and chunks[0] != 'patch':
 463             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 464             continue
 465         if not re.match(r'^-?[0-9]+$', chunks[1]):
 466             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 467             continue
 468         versioncode = int(chunks[1])
 469         packagename = ".".join(chunks[2:-1])
 470
 471         highestVersionCode = java_Integer_MIN_VALUE
 472         if packagename not in apps.keys():
 473             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 474             continue
 475         for apk in apks:
 476             if packagename == apk['id'] and apk['versioncode'] > highestVersionCode:
 477                 highestVersionCode = apk['versioncode']
 478         if versioncode > highestVersionCode:
 479             obbWarnDelete(f, 'OBB file has newer versioncode(' + str(versioncode)
 480                           + ') than any APK: ')
 481             continue
 482         obbsha256 = sha256sum(f)
 483         obbs.append((packagename, versioncode, obbfile, obbsha256))
 484
 485     for apk in apks:
 486         for (packagename, versioncode, obbfile, obbsha256) in sorted(obbs, reverse=True):
 487             if versioncode <= apk['versioncode'] and packagename == apk['id']:
 488                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 489                     apk['obbMainFile'] = obbfile
 490                     apk['obbMainFileSha256'] = obbsha256
 491                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 492                     apk['obbPatchFile'] = obbfile
 493                     apk['obbPatchFileSha256'] = obbsha256
 494             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 495                 break
 496
 497
 498 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
 499     """Scan the apks in the given repo directory.
 500
 501     This also extracts the icons.
 502
 503     :param apps: list of all applications, as per metadata.read_metadata
 504     :param apkcache: current apk cache information
 505     :param repodir: repo directory to scan
 506     :param knownapks: known apks info
 507     :param use_date_from_apk: use date from APK (instead of current date)
 508                               for newly added APKs
 509     :returns: (apks, cachechanged) where apks is a list of apk information,
 510               and cachechanged is True if the apkcache got changed.
 511     """
 512
 513     cachechanged = False
 514
 515     for icon_dir in get_all_icon_dirs(repodir):
 516         if os.path.exists(icon_dir):
 517             if options.clean:
 518                 shutil.rmtree(icon_dir)
 519                 os.makedirs(icon_dir)
 520         else:
 521             os.makedirs(icon_dir)
 522
 523     apks = []
 524     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 525     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 526     vername_pat = re.compile(".*versionName='([^']*)'.*")
 527     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 528     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 529     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 530     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 531     string_pat = re.compile(".* name='([^']*)'.*")
 532     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 533
 534         apkfilename = apkfile[len(repodir) + 1:]
 535         if ' ' in apkfilename:
 536             logging.critical("Spaces in filenames are not allowed.")
 537             sys.exit(1)
 538
 539         shasum = sha256sum(apkfile)
 540
 541         usecache = False
 542         if apkfilename in apkcache:
 543             apk = apkcache[apkfilename]
 544             if apk['sha256'] == shasum:
 545                 logging.debug("Reading " + apkfilename + " from cache")
 546                 usecache = True
 547             else:
 548                 logging.debug("Ignoring stale cache data for " + apkfilename)
 549
 550         if not usecache:
 551             logging.debug("Processing " + apkfilename)
 552             apk = {}
 553             apk['apkname'] = apkfilename
 554             apk['sha256'] = shasum
 555             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 556             if os.path.exists(os.path.join(repodir, srcfilename)):
 557                 apk['srcname'] = srcfilename
 558             apk['size'] = os.path.getsize(apkfile)
 559             apk['permissions'] = set()
 560             apk['features'] = set()
 561             apk['icons_src'] = {}
 562             apk['icons'] = {}
 563             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 564             if p.returncode != 0:
 565                 if options.delete_unknown:
 566                     if os.path.exists(apkfile):
 567                         logging.error("Failed to get apk information, deleting " + apkfile)
 568                         os.remove(apkfile)
 569                     else:
 570                         logging.error("Could not find {0} to remove it".format(apkfile))
 571                 else:
 572                     logging.error("Failed to get apk information, skipping " + apkfile)
 573                 continue
 574             for line in p.output.splitlines():
 575                 if line.startswith("package:"):
 576                     try:
 577                         apk['id'] = re.match(name_pat, line).group(1)
 578                         apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
 579                         apk['version'] = re.match(vername_pat, line).group(1)
 580                     except Exception as e:
 581                         logging.error("Package matching failed: " + str(e))
 582                         logging.info("Line was: " + line)
 583                         sys.exit(1)
 584                 elif line.startswith("application:"):
 585                     apk['name'] = re.match(label_pat, line).group(1)
 586                     # Keep path to non-dpi icon in case we need it
 587                     match = re.match(icon_pat_nodpi, line)
 588                     if match:
 589                         apk['icons_src']['-1'] = match.group(1)
 590                 elif line.startswith("launchable-activity:"):
 591                     # Only use launchable-activity as fallback to application
 592                     if not apk['name']:
 593                         apk['name'] = re.match(label_pat, line).group(1)
 594                     if '-1' not in apk['icons_src']:
 595                         match = re.match(icon_pat_nodpi, line)
 596                         if match:
 597                             apk['icons_src']['-1'] = match.group(1)
 598                 elif line.startswith("application-icon-"):
 599                     match = re.match(icon_pat, line)
 600                     if match:
 601                         density = match.group(1)
 602                         path = match.group(2)
 603                         apk['icons_src'][density] = path
 604                 elif line.startswith("sdkVersion:"):
 605                     m = re.match(sdkversion_pat, line)
 606                     if m is None:
 607                         logging.error(line.replace('sdkVersion:', '')
 608                                       + ' is not a valid minSdkVersion!')
 609                     else:
 610                         apk['minSdkVersion'] = m.group(1)
 611                         # if target not set, default to min
 612                         if 'targetSdkVersion' not in apk:
 613                             apk['targetSdkVersion'] = m.group(1)
 614                 elif line.startswith("targetSdkVersion:"):
 615                     m = re.match(sdkversion_pat, line)
 616                     if m is None:
 617                         logging.error(line.replace('targetSdkVersion:', '')
 618                                       + ' is not a valid targetSdkVersion!')
 619                     else:
 620                         apk['targetSdkVersion'] = m.group(1)
 621                 elif line.startswith("maxSdkVersion:"):
 622                     apk['maxSdkVersion'] = re.match(sdkversion_pat, line).group(1)
 623                 elif line.startswith("native-code:"):
 624                     apk['nativecode'] = []
 625                     for arch in line[13:].split(' '):
 626                         apk['nativecode'].append(arch[1:-1])
 627                 elif line.startswith("uses-permission:"):
 628                     perm = re.match(string_pat, line).group(1)
 629                     if perm.startswith("android.permission."):
 630                         perm = perm[19:]
 631                     apk['permissions'].add(perm)
 632                 elif line.startswith("uses-feature:"):
 633                     perm = re.match(string_pat, line).group(1)
 634                     # Filter out this, it's only added with the latest SDK tools and
 635                     # causes problems for lots of apps.
 636                     if perm != "android.hardware.screen.portrait" \
 637                             and perm != "android.hardware.screen.landscape":
 638                         if perm.startswith("android.feature."):
 639                             perm = perm[16:]
 640                         apk['features'].add(perm)
 641
 642             if 'minSdkVersion' not in apk:
 643                 logging.warn("No SDK version information found in {0}".format(apkfile))
 644                 apk['minSdkVersion'] = 1
 645
 646             # Check for debuggable apks...
 647             if common.isApkDebuggable(apkfile, config):
 648                 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
 649
 650             # Get the signature (or md5 of, to be precise)...
 651             logging.debug('Getting signature of {0}'.format(apkfile))
 652             apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 653             if not apk['sig']:
 654                 logging.critical("Failed to get apk signature")
 655                 sys.exit(1)
 656
 657             apkzip = zipfile.ZipFile(apkfile, 'r')
 658
 659             # if an APK has files newer than the system time, suggest updating
 660             # the system clock.  This is useful for offline systems, used for
 661             # signing, which do not have another source of clock sync info. It
 662             # has to be more than 24 hours newer because ZIP/APK files do not
 663             # store timezone info
 664             manifest = apkzip.getinfo('AndroidManifest.xml')
 665             if manifest.date_time[1] == 0:  # month can't be zero
 666                 logging.debug('AndroidManifest.xml has no date')
 667             else:
 668                 dt_obj = datetime(*manifest.date_time)
 669                 checkdt = dt_obj - timedelta(1)
 670                 if datetime.today() < checkdt:
 671                     logging.warn('System clock is older than manifest in: '
 672                                  + apkfilename
 673                                  + '\nSet clock to that time using:\n'
 674                                  + 'sudo date -s "' + str(dt_obj) + '"')
 675
 676             iconfilename = "%s.%s.png" % (
 677                 apk['id'],
 678                 apk['versioncode'])
 679
 680             # Extract the icon file...
 681             empty_densities = []
 682             for density in screen_densities:
 683                 if density not in apk['icons_src']:
 684                     empty_densities.append(density)
 685                     continue
 686                 iconsrc = apk['icons_src'][density]
 687                 icon_dir = get_icon_dir(repodir, density)
 688                 icondest = os.path.join(icon_dir, iconfilename)
 689
 690                 try:
 691                     with open(icondest, 'wb') as f:
 692                         f.write(get_icon_bytes(apkzip, iconsrc))
 693                     apk['icons'][density] = iconfilename
 694
 695                 except:
 696                     logging.warn("Error retrieving icon file")
 697                     del apk['icons'][density]
 698                     del apk['icons_src'][density]
 699                     empty_densities.append(density)
 700
 701             if '-1' in apk['icons_src']:
 702                 iconsrc = apk['icons_src']['-1']
 703                 iconpath = os.path.join(
 704                     get_icon_dir(repodir, '0'), iconfilename)
 705                 with open(iconpath, 'wb') as f:
 706                     f.write(get_icon_bytes(apkzip, iconsrc))
 707                 try:
 708                     im = Image.open(iconpath)
 709                     dpi = px_to_dpi(im.size[0])
 710                     for density in screen_densities:
 711                         if density in apk['icons']:
 712                             break
 713                         if density == screen_densities[-1] or dpi >= int(density):
 714                             apk['icons'][density] = iconfilename
 715                             shutil.move(iconpath,
 716                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 717                             empty_densities.remove(density)
 718                             break
 719                 except Exception as e:
 720                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 721
 722             if apk['icons']:
 723                 apk['icon'] = iconfilename
 724
 725             apkzip.close()
 726
 727             # First try resizing down to not lose quality
 728             last_density = None
 729             for density in screen_densities:
 730                 if density not in empty_densities:
 731                     last_density = density
 732                     continue
 733                 if last_density is None:
 734                     continue
 735                 logging.debug("Density %s not available, resizing down from %s"
 736                               % (density, last_density))
 737
 738                 last_iconpath = os.path.join(
 739                     get_icon_dir(repodir, last_density), iconfilename)
 740                 iconpath = os.path.join(
 741                     get_icon_dir(repodir, density), iconfilename)
 742                 fp = None
 743                 try:
 744                     fp = open(last_iconpath, 'rb')
 745                     im = Image.open(fp)
 746
 747                     size = dpi_to_px(density)
 748
 749                     im.thumbnail((size, size), Image.ANTIALIAS)
 750                     im.save(iconpath, "PNG")
 751                     empty_densities.remove(density)
 752                 except:
 753                     logging.warning("Invalid image file at %s" % last_iconpath)
 754                 finally:
 755                     if fp:
 756                         fp.close()
 757
 758             # Then just copy from the highest resolution available
 759             last_density = None
 760             for density in reversed(screen_densities):
 761                 if density not in empty_densities:
 762                     last_density = density
 763                     continue
 764                 if last_density is None:
 765                     continue
 766                 logging.debug("Density %s not available, copying from lower density %s"
 767                               % (density, last_density))
 768
 769                 shutil.copyfile(
 770                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 771                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 772
 773                 empty_densities.remove(density)
 774
 775             for density in screen_densities:
 776                 icon_dir = get_icon_dir(repodir, density)
 777                 icondest = os.path.join(icon_dir, iconfilename)
 778                 resize_icon(icondest, density)
 779
 780             # Copy from icons-mdpi to icons since mdpi is the baseline density
 781             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 782             if os.path.isfile(baseline):
 783                 apk['icons']['0'] = iconfilename
 784                 shutil.copyfile(baseline,
 785                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 786
 787             if use_date_from_apk and manifest.date_time[1] != 0:
 788                 default_date_param = datetime(*manifest.date_time).utctimetuple()
 789             else:
 790                 default_date_param = None
 791
 792             # Record in known apks, getting the added date at the same time..
 793             added = knownapks.recordapk(apk['apkname'], apk['id'], default_date=default_date_param)
 794             if added:
 795                 apk['added'] = added
 796
 797             apkcache[apkfilename] = apk
 798             cachechanged = True
 799
 800         apks.append(apk)
 801
 802     return apks, cachechanged
 803
 804
 805 repo_pubkey_fingerprint = None
 806
 807
 808 # Generate a certificate fingerprint the same way keytool does it
 809 # (but with slightly different formatting)
 810 def cert_fingerprint(data):
 811     digest = hashlib.sha256(data).digest()
 812     ret = []
 813     ret.append(' '.join("%02X" % b for b in bytearray(digest)))
 814     return " ".join(ret)
 815
 816
 817 def extract_pubkey():
 818     global repo_pubkey_fingerprint
 819     if 'repo_pubkey' in config:
 820         pubkey = unhexlify(config['repo_pubkey'])
 821     else:
 822         p = FDroidPopenBytes([config['keytool'], '-exportcert',
 823                               '-alias', config['repo_keyalias'],
 824                               '-keystore', config['keystore'],
 825                               '-storepass:file', config['keystorepassfile']]
 826                              + config['smartcardoptions'],
 827                              output=False, stderr_to_stdout=False)
 828         if p.returncode != 0 or len(p.output) < 20:
 829             msg = "Failed to get repo pubkey!"
 830             if config['keystore'] == 'NONE':
 831                 msg += ' Is your crypto smartcard plugged in?'
 832             logging.critical(msg)
 833             sys.exit(1)
 834         pubkey = p.output
 835     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 836     return hexlify(pubkey)
 837
 838
 839 def make_index(apps, sortedids, apks, repodir, archive, categories):
 840     """Make a repo index.
 841
 842     :param apps: fully populated apps list
 843     :param apks: full populated apks list
 844     :param repodir: the repo directory
 845     :param archive: True if this is the archive repo, False if it's the
 846                     main one.
 847     :param categories: list of categories
 848     """
 849
 850     doc = Document()
 851
 852     def addElement(name, value, doc, parent):
 853         el = doc.createElement(name)
 854         el.appendChild(doc.createTextNode(value))
 855         parent.appendChild(el)
 856
 857     def addElementNonEmpty(name, value, doc, parent):
 858         if not value:
 859             return
 860         addElement(name, value, doc, parent)
 861
 862     def addElementCDATA(name, value, doc, parent):
 863         el = doc.createElement(name)
 864         el.appendChild(doc.createCDATASection(value))
 865         parent.appendChild(el)
 866
 867     root = doc.createElement("fdroid")
 868     doc.appendChild(root)
 869
 870     repoel = doc.createElement("repo")
 871
 872     mirrorcheckfailed = False
 873     for mirror in config.get('mirrors', []):
 874         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
 875         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
 876             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 877             mirrorcheckfailed = True
 878     if mirrorcheckfailed:
 879         sys.exit(1)
 880
 881     if archive:
 882         repoel.setAttribute("name", config['archive_name'])
 883         if config['repo_maxage'] != 0:
 884             repoel.setAttribute("maxage", str(config['repo_maxage']))
 885         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 886         repoel.setAttribute("url", config['archive_url'])
 887         addElement('description', config['archive_description'], doc, repoel)
 888         urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
 889         for mirror in config.get('mirrors', []):
 890             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 891
 892     else:
 893         repoel.setAttribute("name", config['repo_name'])
 894         if config['repo_maxage'] != 0:
 895             repoel.setAttribute("maxage", str(config['repo_maxage']))
 896         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 897         repoel.setAttribute("url", config['repo_url'])
 898         addElement('description', config['repo_description'], doc, repoel)
 899         urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
 900         for mirror in config.get('mirrors', []):
 901             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 902
 903     repoel.setAttribute("version", str(METADATA_VERSION))
 904     repoel.setAttribute("timestamp", str(int(time.time())))
 905
 906     nosigningkey = False
 907     if not options.nosign:
 908         if 'repo_keyalias' not in config:
 909             nosigningkey = True
 910             logging.critical("'repo_keyalias' not found in config.py!")
 911         if 'keystore' not in config:
 912             nosigningkey = True
 913             logging.critical("'keystore' not found in config.py!")
 914         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 915             nosigningkey = True
 916             logging.critical("'keystorepass' not found in config.py!")
 917         if 'keypass' not in config and 'keypassfile' not in config:
 918             nosigningkey = True
 919             logging.critical("'keypass' not found in config.py!")
 920         if not os.path.exists(config['keystore']):
 921             nosigningkey = True
 922             logging.critical("'" + config['keystore'] + "' does not exist!")
 923         if nosigningkey:
 924             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 925             logging.warning("\tfdroid update --create-key")
 926             sys.exit(1)
 927
 928     repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
 929     root.appendChild(repoel)
 930
 931     for appid in sortedids:
 932         app = apps[appid]
 933
 934         if app.Disabled is not None:
 935             continue
 936
 937         # Get a list of the apks for this app...
 938         apklist = []
 939         for apk in apks:
 940             if apk['id'] == appid:
 941                 apklist.append(apk)
 942
 943         if len(apklist) == 0:
 944             continue
 945
 946         apel = doc.createElement("application")
 947         apel.setAttribute("id", app.id)
 948         root.appendChild(apel)
 949
 950         addElement('id', app.id, doc, apel)
 951         if app.added:
 952             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
 953         if app.lastupdated:
 954             addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
 955         addElement('name', app.Name, doc, apel)
 956         addElement('summary', app.Summary, doc, apel)
 957         if app.icon:
 958             addElement('icon', app.icon, doc, apel)
 959
 960         def linkres(appid):
 961             if appid in apps:
 962                 return ("fdroid.app:" + appid, apps[appid].Name)
 963             raise MetaDataException("Cannot resolve app id " + appid)
 964
 965         addElement('desc',
 966                    metadata.description_html(app.Description, linkres),
 967                    doc, apel)
 968         addElement('license', app.License, doc, apel)
 969         if app.Categories:
 970             addElement('categories', ','.join(app.Categories), doc, apel)
 971             # We put the first (primary) category in LAST, which will have
 972             # the desired effect of making clients that only understand one
 973             # category see that one.
 974             addElement('category', app.Categories[0], doc, apel)
 975         addElement('web', app.WebSite, doc, apel)
 976         addElement('source', app.SourceCode, doc, apel)
 977         addElement('tracker', app.IssueTracker, doc, apel)
 978         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 979         addElementNonEmpty('author', app.AuthorName, doc, apel)
 980         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 981         addElementNonEmpty('donate', app.Donate, doc, apel)
 982         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 983         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 984         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 985
 986         # These elements actually refer to the current version (i.e. which
 987         # one is recommended. They are historically mis-named, and need
 988         # changing, but stay like this for now to support existing clients.
 989         addElement('marketversion', app.CurrentVersion, doc, apel)
 990         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 991
 992         if app.AntiFeatures:
 993             af = app.AntiFeatures
 994             if af:
 995                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 996         if app.Provides:
 997             pv = app.Provides.split(',')
 998             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 999         if app.RequiresRoot:
1000             addElement('requirements', 'root', doc, apel)
1001
1002         # Sort the apk list into version order, just so the web site
1003         # doesn't have to do any work by default...
1004         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
1005
1006         # Check for duplicates - they will make the client unhappy...
1007         for i in range(len(apklist) - 1):
1008             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
1009                 logging.critical("duplicate versions: '%s' - '%s'" % (
1010                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
1011                 sys.exit(1)
1012
1013         current_version_code = 0
1014         current_version_file = None
1015         for apk in apklist:
1016             # find the APK for the "Current Version"
1017             if current_version_code < apk['versioncode']:
1018                 current_version_code = apk['versioncode']
1019             if current_version_code < int(app.CurrentVersionCode):
1020                 current_version_file = apk['apkname']
1021
1022             apkel = doc.createElement("package")
1023             apel.appendChild(apkel)
1024             addElement('version', apk['version'], doc, apkel)
1025             addElement('versioncode', str(apk['versioncode']), doc, apkel)
1026             addElement('apkname', apk['apkname'], doc, apkel)
1027             if 'srcname' in apk:
1028                 addElement('srcname', apk['srcname'], doc, apkel)
1029             for hash_type in ['sha256']:
1030                 if hash_type not in apk:
1031                     continue
1032                 hashel = doc.createElement("hash")
1033                 hashel.setAttribute("type", hash_type)
1034                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
1035                 apkel.appendChild(hashel)
1036             addElement('sig', apk['sig'], doc, apkel)
1037             addElement('size', str(apk['size']), doc, apkel)
1038             addElement('sdkver', str(apk['minSdkVersion']), doc, apkel)
1039             if 'targetSdkVersion' in apk:
1040                 addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
1041             if 'maxSdkVersion' in apk:
1042                 addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
1043             addElementNonEmpty('obbMainFile', apk.get('obbMainFile'), doc, apkel)
1044             addElementNonEmpty('obbMainFileSha256', apk.get('obbMainFileSha256'), doc, apkel)
1045             addElementNonEmpty('obbPatchFile', apk.get('obbPatchFile'), doc, apkel)
1046             addElementNonEmpty('obbPatchFileSha256', apk.get('obbPatchFileSha256'), doc, apkel)
1047             if 'added' in apk:
1048                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
1049             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
1050             if 'nativecode' in apk:
1051                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
1052             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
1053
1054         if current_version_file is not None \
1055                 and config['make_current_version_link'] \
1056                 and repodir == 'repo':  # only create these
1057             namefield = config['current_version_name_source']
1058             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
1059             apklinkname = sanitized_name + '.apk'
1060             current_version_path = os.path.join(repodir, current_version_file)
1061             if os.path.islink(apklinkname):
1062                 os.remove(apklinkname)
1063             os.symlink(current_version_path, apklinkname)
1064             # also symlink gpg signature, if it exists
1065             for extension in ('.asc', '.sig'):
1066                 sigfile_path = current_version_path + extension
1067                 if os.path.exists(sigfile_path):
1068                     siglinkname = apklinkname + extension
1069                     if os.path.islink(siglinkname):
1070                         os.remove(siglinkname)
1071                     os.symlink(sigfile_path, siglinkname)
1072
1073     if options.pretty:
1074         output = doc.toprettyxml(encoding='utf-8')
1075     else:
1076         output = doc.toxml(encoding='utf-8')
1077
1078     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
1079         f.write(output)
1080
1081     if 'repo_keyalias' in config:
1082
1083         if options.nosign:
1084             logging.info("Creating unsigned index in preparation for signing")
1085         else:
1086             logging.info("Creating signed index with this key (SHA256):")
1087             logging.info("%s" % repo_pubkey_fingerprint)
1088
1089         # Create a jar of the index...
1090         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
1091         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
1092         if p.returncode != 0:
1093             logging.critical("Failed to create {0}".format(jar_output))
1094             sys.exit(1)
1095
1096         # Sign the index...
1097         signed = os.path.join(repodir, 'index.jar')
1098         if options.nosign:
1099             # Remove old signed index if not signing
1100             if os.path.exists(signed):
1101                 os.remove(signed)
1102         else:
1103             args = [config['jarsigner'], '-keystore', config['keystore'],
1104                     '-storepass:file', config['keystorepassfile'],
1105                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1106                     signed, config['repo_keyalias']]
1107             if config['keystore'] == 'NONE':
1108                 args += config['smartcardoptions']
1109             else:  # smardcards never use -keypass
1110                 args += ['-keypass:file', config['keypassfile']]
1111             p = FDroidPopen(args)
1112             if p.returncode != 0:
1113                 logging.critical("Failed to sign index")
1114                 sys.exit(1)
1115
1116     # Copy the repo icon into the repo directory...
1117     icon_dir = os.path.join(repodir, 'icons')
1118     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1119     shutil.copyfile(config['repo_icon'], iconfilename)
1120
1121     # Write a category list in the repo to allow quick access...
1122     catdata = ''
1123     for cat in categories:
1124         catdata += cat + '\n'
1125     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1126         f.write(catdata)
1127
1128
1129 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1130
1131     for appid, app in apps.items():
1132
1133         if app.ArchivePolicy:
1134             keepversions = int(app.ArchivePolicy[:-9])
1135         else:
1136             keepversions = defaultkeepversions
1137
1138         def filter_apk_list_sorted(apk_list):
1139             res = []
1140             for apk in apk_list:
1141                 if apk['id'] == appid:
1142                     res.append(apk)
1143
1144             # Sort the apk list by version code. First is highest/newest.
1145             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1146
1147         def move_file(from_dir, to_dir, filename, ignore_missing):
1148             from_path = os.path.join(from_dir, filename)
1149             if ignore_missing and not os.path.exists(from_path):
1150                 return
1151             to_path = os.path.join(to_dir, filename)
1152             shutil.move(from_path, to_path)
1153
1154         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1155                       .format(appid, len(apks), keepversions, len(archapks)))
1156
1157         if len(apks) > keepversions:
1158             apklist = filter_apk_list_sorted(apks)
1159             # Move back the ones we don't want.
1160             for apk in apklist[keepversions:]:
1161                 logging.info("Moving " + apk['apkname'] + " to archive")
1162                 move_file(repodir, archivedir, apk['apkname'], False)
1163                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1164                 for density in all_screen_densities:
1165                     repo_icon_dir = get_icon_dir(repodir, density)
1166                     archive_icon_dir = get_icon_dir(archivedir, density)
1167                     if density not in apk['icons']:
1168                         continue
1169                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1170                 if 'srcname' in apk:
1171                     move_file(repodir, archivedir, apk['srcname'], False)
1172                 archapks.append(apk)
1173                 apks.remove(apk)
1174         elif len(apks) < keepversions and len(archapks) > 0:
1175             required = keepversions - len(apks)
1176             archapklist = filter_apk_list_sorted(archapks)
1177             # Move forward the ones we want again.
1178             for apk in archapklist[:required]:
1179                 logging.info("Moving " + apk['apkname'] + " from archive")
1180                 move_file(archivedir, repodir, apk['apkname'], False)
1181                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1182                 for density in all_screen_densities:
1183                     repo_icon_dir = get_icon_dir(repodir, density)
1184                     archive_icon_dir = get_icon_dir(archivedir, density)
1185                     if density not in apk['icons']:
1186                         continue
1187                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1188                 if 'srcname' in apk:
1189                     move_file(archivedir, repodir, apk['srcname'], False)
1190                 archapks.remove(apk)
1191                 apks.append(apk)
1192
1193
1194 def add_apks_to_per_app_repos(repodir, apks):
1195     apks_per_app = dict()
1196     for apk in apks:
1197         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1198         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1199         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1200         apks_per_app[apk['id']] = apk
1201
1202         if not os.path.exists(apk['per_app_icons']):
1203             logging.info('Adding new repo for only ' + apk['id'])
1204             os.makedirs(apk['per_app_icons'])
1205
1206         apkpath = os.path.join(repodir, apk['apkname'])
1207         shutil.copy(apkpath, apk['per_app_repo'])
1208         apksigpath = apkpath + '.sig'
1209         if os.path.exists(apksigpath):
1210             shutil.copy(apksigpath, apk['per_app_repo'])
1211         apkascpath = apkpath + '.asc'
1212         if os.path.exists(apkascpath):
1213             shutil.copy(apkascpath, apk['per_app_repo'])
1214
1215
1216 config = None
1217 options = None
1218
1219
1220 def main():
1221
1222     global config, options
1223
1224     # Parse command line...
1225     parser = ArgumentParser()
1226     common.setup_global_opts(parser)
1227     parser.add_argument("--create-key", action="store_true", default=False,
1228                         help="Create a repo signing key in a keystore")
1229     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1230                         help="Create skeleton metadata files that are missing")
1231     parser.add_argument("--delete-unknown", action="store_true", default=False,
1232                         help="Delete APKs and/or OBBs without metadata from the repo")
1233     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1234                         help="Report on build data status")
1235     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1236                         help="Interactively ask about things that need updating.")
1237     parser.add_argument("-I", "--icons", action="store_true", default=False,
1238                         help="Resize all the icons exceeding the max pixel size and exit")
1239     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1240                         help="Specify editor to use in interactive mode. Default " +
1241                         "is /etc/alternatives/editor")
1242     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1243                         help="Update the wiki")
1244     parser.add_argument("--pretty", action="store_true", default=False,
1245                         help="Produce human-readable index.xml")
1246     parser.add_argument("--clean", action="store_true", default=False,
1247                         help="Clean update - don't uses caches, reprocess all apks")
1248     parser.add_argument("--nosign", action="store_true", default=False,
1249                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1250     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1251                         help="Use date from apk instead of current time for newly added apks")
1252     options = parser.parse_args()
1253
1254     config = common.read_config(options)
1255
1256     if not ('jarsigner' in config and 'keytool' in config):
1257         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1258         sys.exit(1)
1259
1260     repodirs = ['repo']
1261     if config['archive_older'] != 0:
1262         repodirs.append('archive')
1263         if not os.path.exists('archive'):
1264             os.mkdir('archive')
1265
1266     if options.icons:
1267         resize_all_icons(repodirs)
1268         sys.exit(0)
1269
1270     # check that icons exist now, rather than fail at the end of `fdroid update`
1271     for k in ['repo_icon', 'archive_icon']:
1272         if k in config:
1273             if not os.path.exists(config[k]):
1274                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1275                 sys.exit(1)
1276
1277     # if the user asks to create a keystore, do it now, reusing whatever it can
1278     if options.create_key:
1279         if os.path.exists(config['keystore']):
1280             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1281             logging.critical("\t'" + config['keystore'] + "'")
1282             sys.exit(1)
1283
1284         if 'repo_keyalias' not in config:
1285             config['repo_keyalias'] = socket.getfqdn()
1286             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1287         if 'keydname' not in config:
1288             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1289             common.write_to_config(config, 'keydname', config['keydname'])
1290         if 'keystore' not in config:
1291             config['keystore'] = common.default_config.keystore
1292             common.write_to_config(config, 'keystore', config['keystore'])
1293
1294         password = common.genpassword()
1295         if 'keystorepass' not in config:
1296             config['keystorepass'] = password
1297             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1298         if 'keypass' not in config:
1299             config['keypass'] = password
1300             common.write_to_config(config, 'keypass', config['keypass'])
1301         common.genkeystore(config)
1302
1303     # Get all apps...
1304     apps = metadata.read_metadata()
1305
1306     # Generate a list of categories...
1307     categories = set()
1308     for app in apps.values():
1309         categories.update(app.Categories)
1310
1311     # Read known apks data (will be updated and written back when we've finished)
1312     knownapks = common.KnownApks()
1313
1314     # Gather information about all the apk files in the repo directory, using
1315     # cached data if possible.
1316     apkcachefile = os.path.join('tmp', 'apkcache')
1317     if not options.clean and os.path.exists(apkcachefile):
1318         with open(apkcachefile, 'rb') as cf:
1319             apkcache = pickle.load(cf, encoding='utf-8')
1320         if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
1321             apkcache = {}
1322     else:
1323         apkcache = {}
1324
1325     delete_disabled_builds(apps, apkcache, repodirs)
1326
1327     # Scan all apks in the main repo
1328     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1329
1330     # Generate warnings for apk's with no metadata (or create skeleton
1331     # metadata files, if requested on the command line)
1332     newmetadata = False
1333     for apk in apks:
1334         if apk['id'] not in apps:
1335             if options.create_metadata:
1336                 if 'name' not in apk:
1337                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1338                     continue
1339                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
1340                 f.write("License:Unknown\n")
1341                 f.write("Web Site:\n")
1342                 f.write("Source Code:\n")
1343                 f.write("Issue Tracker:\n")
1344                 f.write("Changelog:\n")
1345                 f.write("Summary:" + apk['name'] + "\n")
1346                 f.write("Description:\n")
1347                 f.write(apk['name'] + "\n")
1348                 f.write(".\n")
1349                 f.close()
1350                 logging.info("Generated skeleton metadata for " + apk['id'])
1351                 newmetadata = True
1352             else:
1353                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1354                 if options.delete_unknown:
1355                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1356                     rmf = os.path.join(repodirs[0], apk['apkname'])
1357                     if not os.path.exists(rmf):
1358                         logging.error("Could not find {0} to remove it".format(rmf))
1359                     else:
1360                         os.remove(rmf)
1361                 else:
1362                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1363
1364     # update the metadata with the newly created ones included
1365     if newmetadata:
1366         apps = metadata.read_metadata()
1367
1368     insert_obbs(repodirs[0], apps, apks)
1369
1370     # Scan the archive repo for apks as well
1371     if len(repodirs) > 1:
1372         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1373         if cc:
1374             cachechanged = True
1375     else:
1376         archapks = []
1377
1378     # Some information from the apks needs to be applied up to the application
1379     # level. When doing this, we use the info from the most recent version's apk.
1380     # We deal with figuring out when the app was added and last updated at the
1381     # same time.
1382     for appid, app in apps.items():
1383         bestver = 0
1384         for apk in apks + archapks:
1385             if apk['id'] == appid:
1386                 if apk['versioncode'] > bestver:
1387                     bestver = apk['versioncode']
1388                     bestapk = apk
1389
1390                 if 'added' in apk:
1391                     if not app.added or apk['added'] < app.added:
1392                         app.added = apk['added']
1393                     if not app.lastupdated or apk['added'] > app.lastupdated:
1394                         app.lastupdated = apk['added']
1395
1396         if not app.added:
1397             logging.debug("Don't know when " + appid + " was added")
1398         if not app.lastupdated:
1399             logging.debug("Don't know when " + appid + " was last updated")
1400
1401         if bestver == 0:
1402             if app.Name is None:
1403                 app.Name = app.AutoName or appid
1404             app.icon = None
1405             logging.debug("Application " + appid + " has no packages")
1406         else:
1407             if app.Name is None:
1408                 app.Name = bestapk['name']
1409             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1410             if app.CurrentVersionCode is None:
1411                 app.CurrentVersionCode = str(bestver)
1412
1413     # Sort the app list by name, then the web site doesn't have to by default.
1414     # (we had to wait until we'd scanned the apks to do this, because mostly the
1415     # name comes from there!)
1416     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1417
1418     # APKs are placed into multiple repos based on the app package, providing
1419     # per-app subscription feeds for nightly builds and things like it
1420     if config['per_app_repos']:
1421         add_apks_to_per_app_repos(repodirs[0], apks)
1422         for appid, app in apps.items():
1423             repodir = os.path.join(appid, 'fdroid', 'repo')
1424             appdict = dict()
1425             appdict[appid] = app
1426             if os.path.isdir(repodir):
1427                 make_index(appdict, [appid], apks, repodir, False, categories)
1428             else:
1429                 logging.info('Skipping index generation for ' + appid)
1430         return
1431
1432     if len(repodirs) > 1:
1433         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1434
1435     # Make the index for the main repo...
1436     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1437
1438     # If there's an archive repo,  make the index for it. We already scanned it
1439     # earlier on.
1440     if len(repodirs) > 1:
1441         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1442
1443     if config['update_stats']:
1444
1445         # Update known apks info...
1446         knownapks.writeifchanged()
1447
1448         # Generate latest apps data for widget
1449         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1450             data = ''
1451             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1452                 for line in f:
1453                     appid = line.rstrip()
1454                     data += appid + "\t"
1455                     app = apps[appid]
1456                     data += app.Name + "\t"
1457                     if app.icon is not None:
1458                         data += app.icon + "\t"
1459                     data += app.License + "\n"
1460             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1461                 f.write(data)
1462
1463     if cachechanged:
1464         apkcache["METADATA_VERSION"] = METADATA_VERSION
1465         with open(apkcachefile, 'wb') as cf:
1466             pickle.dump(apkcache, cf)
1467
1468     # Update the wiki...
1469     if options.wiki:
1470         update_wiki(apps, sortedids, apks + archapks)
1471
1472     logging.info("Finished.")
1473
1474 if __name__ == "__main__":
1475     main()