fdroidserver/update.py

   1 #!/usr/bin/env python3
   2 #
   3 # update.py - part of the FDroid server tools
   4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
   5 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU Affero General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Affero General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Affero General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 import sys
  21 import os
  22 import shutil
  23 import glob
  24 import re
  25 import socket
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 import urllib.parse
  30 from datetime import datetime, timedelta
  31 from xml.dom.minidom import Document
  32 from argparse import ArgumentParser
  33 import time
  34 from pyasn1.error import PyAsn1Error
  35 from pyasn1.codec.der import decoder, encoder
  36 from pyasn1_modules import rfc2315
  37 from binascii import hexlify, unhexlify
  38
  39 from PIL import Image
  40 import logging
  41
  42 from . import common
  43 from . import metadata
  44 from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
  45 from .metadata import MetaDataException
  46
  47 METADATA_VERSION = 16
  48
  49 screen_densities = ['640', '480', '320', '240', '160', '120']
  50
  51 all_screen_densities = ['0'] + screen_densities
  52
  53
  54 def dpi_to_px(density):
  55     return (int(density) * 48) / 160
  56
  57
  58 def px_to_dpi(px):
  59     return (int(px) * 160) / 48
  60
  61
  62 def get_icon_dir(repodir, density):
  63     if density == '0':
  64         return os.path.join(repodir, "icons")
  65     return os.path.join(repodir, "icons-%s" % density)
  66
  67
  68 def get_icon_dirs(repodir):
  69     for density in screen_densities:
  70         yield get_icon_dir(repodir, density)
  71
  72
  73 def get_all_icon_dirs(repodir):
  74     for density in all_screen_densities:
  75         yield get_icon_dir(repodir, density)
  76
  77
  78 def update_wiki(apps, sortedids, apks):
  79     """Update the wiki
  80
  81     :param apps: fully populated list of all applications
  82     :param apks: all apks, except...
  83     """
  84     logging.info("Updating wiki")
  85     wikicat = 'Apps'
  86     wikiredircat = 'App Redirects'
  87     import mwclient
  88     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  89                          path=config['wiki_path'])
  90     site.login(config['wiki_user'], config['wiki_password'])
  91     generated_pages = {}
  92     generated_redirects = {}
  93
  94     for appid in sortedids:
  95         app = apps[appid]
  96
  97         wikidata = ''
  98         if app.Disabled:
  99             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
 100         if app.AntiFeatures:
 101             for af in app.AntiFeatures:
 102                 wikidata += '{{AntiFeature|' + af + '}}\n'
 103         if app.RequiresRoot:
 104             requiresroot = 'Yes'
 105         else:
 106             requiresroot = 'No'
 107         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
 108             appid,
 109             app.Name,
 110             time.strftime('%Y-%m-%d', app.added) if app.added else '',
 111             time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
 112             app.SourceCode,
 113             app.IssueTracker,
 114             app.WebSite,
 115             app.Changelog,
 116             app.Donate,
 117             app.FlattrID,
 118             app.Bitcoin,
 119             app.Litecoin,
 120             app.License,
 121             requiresroot,
 122             app.AuthorName,
 123             app.AuthorEmail)
 124
 125         if app.Provides:
 126             wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
 127
 128         wikidata += app.Summary
 129         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 130
 131         wikidata += "=Description=\n"
 132         wikidata += metadata.description_wiki(app.Description) + "\n"
 133
 134         wikidata += "=Maintainer Notes=\n"
 135         if app.MaintainerNotes:
 136             wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
 137         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 138
 139         # Get a list of all packages for this application...
 140         apklist = []
 141         gotcurrentver = False
 142         cantupdate = False
 143         buildfails = False
 144         for apk in apks:
 145             if apk['id'] == appid:
 146                 if str(apk['versioncode']) == app.CurrentVersionCode:
 147                     gotcurrentver = True
 148                 apklist.append(apk)
 149         # Include ones we can't build, as a special case...
 150         for build in app.builds:
 151             if build.disable:
 152                 if build.vercode == app.CurrentVersionCode:
 153                     cantupdate = True
 154                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 155                 apklist.append({'versioncode': int(build.vercode),
 156                                 'version': build.version,
 157                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
 158                                 })
 159             else:
 160                 builtit = False
 161                 for apk in apklist:
 162                     if apk['versioncode'] == int(build.vercode):
 163                         builtit = True
 164                         break
 165                 if not builtit:
 166                     buildfails = True
 167                     apklist.append({'versioncode': int(build.vercode),
 168                                     'version': build.version,
 169                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
 170                                     })
 171         if app.CurrentVersionCode == '0':
 172             cantupdate = True
 173         # Sort with most recent first...
 174         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 175
 176         wikidata += "=Versions=\n"
 177         if len(apklist) == 0:
 178             wikidata += "We currently have no versions of this app available."
 179         elif not gotcurrentver:
 180             wikidata += "We don't have the current version of this app."
 181         else:
 182             wikidata += "We have the current version of this app."
 183         wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
 184         wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
 185         if len(app.NoSourceSince) > 0:
 186             wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
 187         if len(app.CurrentVersion) > 0:
 188             wikidata += "The current (recommended) version is " + app.CurrentVersion
 189             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
 190         validapks = 0
 191         for apk in apklist:
 192             wikidata += "==" + apk['version'] + "==\n"
 193
 194             if 'buildproblem' in apk:
 195                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 196             else:
 197                 validapks += 1
 198                 wikidata += "This version is built and signed by "
 199                 if 'srcname' in apk:
 200                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 201                 else:
 202                     wikidata += "the original developer.\n\n"
 203             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 204
 205         wikidata += '\n[[Category:' + wikicat + ']]\n'
 206         if len(app.NoSourceSince) > 0:
 207             wikidata += '\n[[Category:Apps missing source code]]\n'
 208         if validapks == 0 and not app.Disabled:
 209             wikidata += '\n[[Category:Apps with no packages]]\n'
 210         if cantupdate and not app.Disabled:
 211             wikidata += "\n[[Category:Apps we cannot update]]\n"
 212         if buildfails and not app.Disabled:
 213             wikidata += "\n[[Category:Apps with failing builds]]\n"
 214         elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
 215             wikidata += '\n[[Category:Apps to Update]]\n'
 216         if app.Disabled:
 217             wikidata += '\n[[Category:Apps that are disabled]]\n'
 218         if app.UpdateCheckMode == 'None' and not app.Disabled:
 219             wikidata += '\n[[Category:Apps with no update check]]\n'
 220         for appcat in app.Categories:
 221             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 222
 223         # We can't have underscores in the page name, even if they're in
 224         # the package ID, because MediaWiki messes with them...
 225         pagename = appid.replace('_', ' ')
 226
 227         # Drop a trailing newline, because mediawiki is going to drop it anyway
 228         # and it we don't we'll think the page has changed when it hasn't...
 229         if wikidata.endswith('\n'):
 230             wikidata = wikidata[:-1]
 231
 232         generated_pages[pagename] = wikidata
 233
 234         # Make a redirect from the name to the ID too, unless there's
 235         # already an existing page with the name and it isn't a redirect.
 236         noclobber = False
 237         apppagename = app.Name.replace('_', ' ')
 238         apppagename = apppagename.replace('{', '')
 239         apppagename = apppagename.replace('}', ' ')
 240         apppagename = apppagename.replace(':', ' ')
 241         # Drop double spaces caused mostly by replacing ':' above
 242         apppagename = apppagename.replace('  ', ' ')
 243         for expagename in site.allpages(prefix=apppagename,
 244                                         filterredir='nonredirects',
 245                                         generator=False):
 246             if expagename == apppagename:
 247                 noclobber = True
 248         # Another reason not to make the redirect page is if the app name
 249         # is the same as it's ID, because that will overwrite the real page
 250         # with an redirect to itself! (Although it seems like an odd
 251         # scenario this happens a lot, e.g. where there is metadata but no
 252         # builds or binaries to extract a name from.
 253         if apppagename == pagename:
 254             noclobber = True
 255         if not noclobber:
 256             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 257
 258     for tcat, genp in [(wikicat, generated_pages),
 259                        (wikiredircat, generated_redirects)]:
 260         catpages = site.Pages['Category:' + tcat]
 261         existingpages = []
 262         for page in catpages:
 263             existingpages.append(page.name)
 264             if page.name in genp:
 265                 pagetxt = page.edit()
 266                 if pagetxt != genp[page.name]:
 267                     logging.debug("Updating modified page " + page.name)
 268                     page.save(genp[page.name], summary='Auto-updated')
 269                 else:
 270                     logging.debug("Page " + page.name + " is unchanged")
 271             else:
 272                 logging.warn("Deleting page " + page.name)
 273                 page.delete('No longer published')
 274         for pagename, text in genp.items():
 275             logging.debug("Checking " + pagename)
 276             if pagename not in existingpages:
 277                 logging.debug("Creating page " + pagename)
 278                 try:
 279                     newpage = site.Pages[pagename]
 280                     newpage.save(text, summary='Auto-created')
 281                 except:
 282                     logging.error("...FAILED to create page '{0}'".format(pagename))
 283
 284     # Purge server cache to ensure counts are up to date
 285     site.pages['Repository Maintenance'].purge()
 286
 287
 288 def delete_disabled_builds(apps, apkcache, repodirs):
 289     """Delete disabled build outputs.
 290
 291     :param apps: list of all applications, as per metadata.read_metadata
 292     :param apkcache: current apk cache information
 293     :param repodirs: the repo directories to process
 294     """
 295     for appid, app in apps.items():
 296         for build in app.builds:
 297             if not build.disable:
 298                 continue
 299             apkfilename = appid + '_' + str(build.vercode) + '.apk'
 300             iconfilename = "%s.%s.png" % (
 301                 appid,
 302                 build.vercode)
 303             for repodir in repodirs:
 304                 files = [
 305                     os.path.join(repodir, apkfilename),
 306                     os.path.join(repodir, apkfilename + '.asc'),
 307                     os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
 308                 ]
 309                 for density in all_screen_densities:
 310                     repo_dir = get_icon_dir(repodir, density)
 311                     files.append(os.path.join(repo_dir, iconfilename))
 312
 313                 for f in files:
 314                     if os.path.exists(f):
 315                         logging.info("Deleting disabled build output " + f)
 316                         os.remove(f)
 317             if apkfilename in apkcache:
 318                 del apkcache[apkfilename]
 319
 320
 321 def resize_icon(iconpath, density):
 322
 323     if not os.path.isfile(iconpath):
 324         return
 325
 326     fp = None
 327     try:
 328         fp = open(iconpath, 'rb')
 329         im = Image.open(fp)
 330         size = dpi_to_px(density)
 331
 332         if any(length > size for length in im.size):
 333             oldsize = im.size
 334             im.thumbnail((size, size), Image.ANTIALIAS)
 335             logging.debug("%s was too large at %s - new size is %s" % (
 336                 iconpath, oldsize, im.size))
 337             im.save(iconpath, "PNG")
 338
 339     except Exception as e:
 340         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 341
 342     finally:
 343         if fp:
 344             fp.close()
 345
 346
 347 def resize_all_icons(repodirs):
 348     """Resize all icons that exceed the max size
 349
 350     :param repodirs: the repo directories to process
 351     """
 352     for repodir in repodirs:
 353         for density in screen_densities:
 354             icon_dir = get_icon_dir(repodir, density)
 355             icon_glob = os.path.join(icon_dir, '*.png')
 356             for iconpath in glob.glob(icon_glob):
 357                 resize_icon(iconpath, density)
 358
 359
 360 # A signature block file with a .DSA, .RSA, or .EC extension
 361 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
 362
 363
 364 def getsig(apkpath):
 365     """ Get the signing certificate of an apk. To get the same md5 has that
 366     Android gets, we encode the .RSA certificate in a specific format and pass
 367     it hex-encoded to the md5 digest algorithm.
 368
 369     :param apkpath: path to the apk
 370     :returns: A string containing the md5 of the signature of the apk or None
 371               if an error occurred.
 372     """
 373
 374     cert = None
 375
 376     # verify the jar signature is correct
 377     args = [config['jarsigner'], '-verify', apkpath]
 378     p = FDroidPopen(args)
 379     if p.returncode != 0:
 380         logging.critical(apkpath + " has a bad signature!")
 381         return None
 382
 383     with zipfile.ZipFile(apkpath, 'r') as apk:
 384
 385         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 386
 387         if len(certs) < 1:
 388             logging.error("Found no signing certificates on %s" % apkpath)
 389             return None
 390         if len(certs) > 1:
 391             logging.error("Found multiple signing certificates on %s" % apkpath)
 392             return None
 393
 394         cert = apk.read(certs[0])
 395
 396     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 397     if content.getComponentByName('contentType') != rfc2315.signedData:
 398         logging.error("Unexpected format.")
 399         return None
 400
 401     content = decoder.decode(content.getComponentByName('content'),
 402                              asn1Spec=rfc2315.SignedData())[0]
 403     try:
 404         certificates = content.getComponentByName('certificates')
 405     except PyAsn1Error:
 406         logging.error("Certificates not found.")
 407         return None
 408
 409     cert_encoded = encoder.encode(certificates)[4:]
 410
 411     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 412
 413
 414 def get_icon_bytes(apkzip, iconsrc):
 415     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
 416     try:
 417         return apkzip.read(iconsrc)
 418     except KeyError:
 419         return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
 420
 421
 422 def insert_obbs(repodir, apps, apks):
 423     """Scans the .obb files in a given repo directory and adds them to the
 424     relevant APK instances.
 425
 426     :param repodir: repo directory to scan
 427     :param apps: list of current, valid apps
 428     :param apks: current information on all APKs
 429     """
 430
 431     def obbWarnDelete(f, msg):
 432         logging.warning(msg + f)
 433         if options.delete_unknown:
 434             logging.error("Deleting unknown file: " + f)
 435             os.remove(f)
 436
 437     obbs = []
 438     java_Integer_MIN_VALUE = -pow(2, 31)
 439     for f in glob.glob(os.path.join(repodir, '*.obb')):
 440         obbfile = os.path.basename(f)
 441         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
 442         chunks = obbfile.split('.')
 443         if chunks[0] != 'main' and chunks[0] != 'patch':
 444             obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
 445             continue
 446         if not re.match(r'^-?[0-9]+$', chunks[1]):
 447             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
 448             continue
 449         versioncode = int(chunks[1])
 450         packagename = ".".join(chunks[2:-1])
 451
 452         highestVersionCode = java_Integer_MIN_VALUE
 453         if packagename not in apps.keys():
 454             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
 455             continue
 456         for apk in apks:
 457             if packagename == apk['id'] and apk['versioncode'] > highestVersionCode:
 458                 highestVersionCode = apk['versioncode']
 459         if versioncode > highestVersionCode:
 460             obbWarnDelete(f, 'OBB file has newer versioncode(' + str(versioncode)
 461                           + ') than any APK: ')
 462             continue
 463
 464         obbs.append((packagename, versioncode, obbfile))
 465
 466     for apk in apks:
 467         for (packagename, versioncode, obbfile) in sorted(obbs, reverse=True):
 468             if versioncode <= apk['versioncode'] and packagename == apk['id']:
 469                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
 470                     apk['obbMainFile'] = obbfile
 471                 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
 472                     apk['obbPatchFile'] = obbfile
 473             if 'obbMainFile' in apk and 'obbPatchFile' in apk:
 474                 break
 475
 476
 477 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
 478     """Scan the apks in the given repo directory.
 479
 480     This also extracts the icons.
 481
 482     :param apps: list of all applications, as per metadata.read_metadata
 483     :param apkcache: current apk cache information
 484     :param repodir: repo directory to scan
 485     :param knownapks: known apks info
 486     :param use_date_from_apk: use date from APK (instead of current date)
 487                               for newly added APKs
 488     :returns: (apks, cachechanged) where apks is a list of apk information,
 489               and cachechanged is True if the apkcache got changed.
 490     """
 491
 492     cachechanged = False
 493
 494     for icon_dir in get_all_icon_dirs(repodir):
 495         if os.path.exists(icon_dir):
 496             if options.clean:
 497                 shutil.rmtree(icon_dir)
 498                 os.makedirs(icon_dir)
 499         else:
 500             os.makedirs(icon_dir)
 501
 502     apks = []
 503     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 504     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 505     vername_pat = re.compile(".*versionName='([^']*)'.*")
 506     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 507     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 508     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 509     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 510     string_pat = re.compile(".* name='([^']*)'.*")
 511     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 512
 513         apkfilename = apkfile[len(repodir) + 1:]
 514         if ' ' in apkfilename:
 515             logging.critical("Spaces in filenames are not allowed.")
 516             sys.exit(1)
 517
 518         # Calculate the sha256...
 519         sha = hashlib.sha256()
 520         with open(apkfile, 'rb') as f:
 521             while True:
 522                 t = f.read(16384)
 523                 if len(t) == 0:
 524                     break
 525                 sha.update(t)
 526             shasum = sha.hexdigest()
 527
 528         usecache = False
 529         if apkfilename in apkcache:
 530             apk = apkcache[apkfilename]
 531             if apk['sha256'] == shasum:
 532                 logging.debug("Reading " + apkfilename + " from cache")
 533                 usecache = True
 534             else:
 535                 logging.debug("Ignoring stale cache data for " + apkfilename)
 536
 537         if not usecache:
 538             logging.debug("Processing " + apkfilename)
 539             apk = {}
 540             apk['apkname'] = apkfilename
 541             apk['sha256'] = shasum
 542             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 543             if os.path.exists(os.path.join(repodir, srcfilename)):
 544                 apk['srcname'] = srcfilename
 545             apk['size'] = os.path.getsize(apkfile)
 546             apk['permissions'] = set()
 547             apk['features'] = set()
 548             apk['icons_src'] = {}
 549             apk['icons'] = {}
 550             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 551             if p.returncode != 0:
 552                 if options.delete_unknown:
 553                     if os.path.exists(apkfile):
 554                         logging.error("Failed to get apk information, deleting " + apkfile)
 555                         os.remove(apkfile)
 556                     else:
 557                         logging.error("Could not find {0} to remove it".format(apkfile))
 558                 else:
 559                     logging.error("Failed to get apk information, skipping " + apkfile)
 560                 continue
 561             for line in p.output.splitlines():
 562                 if line.startswith("package:"):
 563                     try:
 564                         apk['id'] = re.match(name_pat, line).group(1)
 565                         apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
 566                         apk['version'] = re.match(vername_pat, line).group(1)
 567                     except Exception as e:
 568                         logging.error("Package matching failed: " + str(e))
 569                         logging.info("Line was: " + line)
 570                         sys.exit(1)
 571                 elif line.startswith("application:"):
 572                     apk['name'] = re.match(label_pat, line).group(1)
 573                     # Keep path to non-dpi icon in case we need it
 574                     match = re.match(icon_pat_nodpi, line)
 575                     if match:
 576                         apk['icons_src']['-1'] = match.group(1)
 577                 elif line.startswith("launchable-activity:"):
 578                     # Only use launchable-activity as fallback to application
 579                     if not apk['name']:
 580                         apk['name'] = re.match(label_pat, line).group(1)
 581                     if '-1' not in apk['icons_src']:
 582                         match = re.match(icon_pat_nodpi, line)
 583                         if match:
 584                             apk['icons_src']['-1'] = match.group(1)
 585                 elif line.startswith("application-icon-"):
 586                     match = re.match(icon_pat, line)
 587                     if match:
 588                         density = match.group(1)
 589                         path = match.group(2)
 590                         apk['icons_src'][density] = path
 591                 elif line.startswith("sdkVersion:"):
 592                     m = re.match(sdkversion_pat, line)
 593                     if m is None:
 594                         logging.error(line.replace('sdkVersion:', '')
 595                                       + ' is not a valid minSdkVersion!')
 596                     else:
 597                         apk['minSdkVersion'] = m.group(1)
 598                         # if target not set, default to min
 599                         if 'targetSdkVersion' not in apk:
 600                             apk['targetSdkVersion'] = m.group(1)
 601                 elif line.startswith("targetSdkVersion:"):
 602                     m = re.match(sdkversion_pat, line)
 603                     if m is None:
 604                         logging.error(line.replace('targetSdkVersion:', '')
 605                                       + ' is not a valid targetSdkVersion!')
 606                     else:
 607                         apk['targetSdkVersion'] = m.group(1)
 608                 elif line.startswith("maxSdkVersion:"):
 609                     apk['maxSdkVersion'] = re.match(sdkversion_pat, line).group(1)
 610                 elif line.startswith("native-code:"):
 611                     apk['nativecode'] = []
 612                     for arch in line[13:].split(' '):
 613                         apk['nativecode'].append(arch[1:-1])
 614                 elif line.startswith("uses-permission:"):
 615                     perm = re.match(string_pat, line).group(1)
 616                     if perm.startswith("android.permission."):
 617                         perm = perm[19:]
 618                     apk['permissions'].add(perm)
 619                 elif line.startswith("uses-feature:"):
 620                     perm = re.match(string_pat, line).group(1)
 621                     # Filter out this, it's only added with the latest SDK tools and
 622                     # causes problems for lots of apps.
 623                     if perm != "android.hardware.screen.portrait" \
 624                             and perm != "android.hardware.screen.landscape":
 625                         if perm.startswith("android.feature."):
 626                             perm = perm[16:]
 627                         apk['features'].add(perm)
 628
 629             if 'minSdkVersion' not in apk:
 630                 logging.warn("No SDK version information found in {0}".format(apkfile))
 631                 apk['minSdkVersion'] = 1
 632
 633             # Check for debuggable apks...
 634             if common.isApkDebuggable(apkfile, config):
 635                 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
 636
 637             # Get the signature (or md5 of, to be precise)...
 638             logging.debug('Getting signature of {0}'.format(apkfile))
 639             apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 640             if not apk['sig']:
 641                 logging.critical("Failed to get apk signature")
 642                 sys.exit(1)
 643
 644             apkzip = zipfile.ZipFile(apkfile, 'r')
 645
 646             # if an APK has files newer than the system time, suggest updating
 647             # the system clock.  This is useful for offline systems, used for
 648             # signing, which do not have another source of clock sync info. It
 649             # has to be more than 24 hours newer because ZIP/APK files do not
 650             # store timezone info
 651             manifest = apkzip.getinfo('AndroidManifest.xml')
 652             if manifest.date_time[1] == 0:  # month can't be zero
 653                 logging.debug('AndroidManifest.xml has no date')
 654             else:
 655                 dt_obj = datetime(*manifest.date_time)
 656                 checkdt = dt_obj - timedelta(1)
 657                 if datetime.today() < checkdt:
 658                     logging.warn('System clock is older than manifest in: '
 659                                  + apkfilename
 660                                  + '\nSet clock to that time using:\n'
 661                                  + 'sudo date -s "' + str(dt_obj) + '"')
 662
 663             iconfilename = "%s.%s.png" % (
 664                 apk['id'],
 665                 apk['versioncode'])
 666
 667             # Extract the icon file...
 668             empty_densities = []
 669             for density in screen_densities:
 670                 if density not in apk['icons_src']:
 671                     empty_densities.append(density)
 672                     continue
 673                 iconsrc = apk['icons_src'][density]
 674                 icon_dir = get_icon_dir(repodir, density)
 675                 icondest = os.path.join(icon_dir, iconfilename)
 676
 677                 try:
 678                     with open(icondest, 'wb') as f:
 679                         f.write(get_icon_bytes(apkzip, iconsrc))
 680                     apk['icons'][density] = iconfilename
 681
 682                 except:
 683                     logging.warn("Error retrieving icon file")
 684                     del apk['icons'][density]
 685                     del apk['icons_src'][density]
 686                     empty_densities.append(density)
 687
 688             if '-1' in apk['icons_src']:
 689                 iconsrc = apk['icons_src']['-1']
 690                 iconpath = os.path.join(
 691                     get_icon_dir(repodir, '0'), iconfilename)
 692                 with open(iconpath, 'wb') as f:
 693                     f.write(get_icon_bytes(apkzip, iconsrc))
 694                 try:
 695                     im = Image.open(iconpath)
 696                     dpi = px_to_dpi(im.size[0])
 697                     for density in screen_densities:
 698                         if density in apk['icons']:
 699                             break
 700                         if density == screen_densities[-1] or dpi >= int(density):
 701                             apk['icons'][density] = iconfilename
 702                             shutil.move(iconpath,
 703                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 704                             empty_densities.remove(density)
 705                             break
 706                 except Exception as e:
 707                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 708
 709             if apk['icons']:
 710                 apk['icon'] = iconfilename
 711
 712             apkzip.close()
 713
 714             # First try resizing down to not lose quality
 715             last_density = None
 716             for density in screen_densities:
 717                 if density not in empty_densities:
 718                     last_density = density
 719                     continue
 720                 if last_density is None:
 721                     continue
 722                 logging.debug("Density %s not available, resizing down from %s"
 723                               % (density, last_density))
 724
 725                 last_iconpath = os.path.join(
 726                     get_icon_dir(repodir, last_density), iconfilename)
 727                 iconpath = os.path.join(
 728                     get_icon_dir(repodir, density), iconfilename)
 729                 fp = None
 730                 try:
 731                     fp = open(last_iconpath, 'rb')
 732                     im = Image.open(fp)
 733
 734                     size = dpi_to_px(density)
 735
 736                     im.thumbnail((size, size), Image.ANTIALIAS)
 737                     im.save(iconpath, "PNG")
 738                     empty_densities.remove(density)
 739                 except:
 740                     logging.warning("Invalid image file at %s" % last_iconpath)
 741                 finally:
 742                     if fp:
 743                         fp.close()
 744
 745             # Then just copy from the highest resolution available
 746             last_density = None
 747             for density in reversed(screen_densities):
 748                 if density not in empty_densities:
 749                     last_density = density
 750                     continue
 751                 if last_density is None:
 752                     continue
 753                 logging.debug("Density %s not available, copying from lower density %s"
 754                               % (density, last_density))
 755
 756                 shutil.copyfile(
 757                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 758                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 759
 760                 empty_densities.remove(density)
 761
 762             for density in screen_densities:
 763                 icon_dir = get_icon_dir(repodir, density)
 764                 icondest = os.path.join(icon_dir, iconfilename)
 765                 resize_icon(icondest, density)
 766
 767             # Copy from icons-mdpi to icons since mdpi is the baseline density
 768             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 769             if os.path.isfile(baseline):
 770                 apk['icons']['0'] = iconfilename
 771                 shutil.copyfile(baseline,
 772                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 773
 774             # Record in known apks, getting the added date at the same time..
 775             added = knownapks.recordapk(apk['apkname'], apk['id'])
 776             if added:
 777                 if use_date_from_apk and manifest.date_time[1] != 0:
 778                     added = datetime(*manifest.date_time).timetuple()
 779                     logging.debug("Using date from APK")
 780
 781                 apk['added'] = added
 782
 783             apkcache[apkfilename] = apk
 784             cachechanged = True
 785
 786         apks.append(apk)
 787
 788     return apks, cachechanged
 789
 790
 791 repo_pubkey_fingerprint = None
 792
 793
 794 # Generate a certificate fingerprint the same way keytool does it
 795 # (but with slightly different formatting)
 796 def cert_fingerprint(data):
 797     digest = hashlib.sha256(data).digest()
 798     ret = []
 799     ret.append(' '.join("%02X" % b for b in bytearray(digest)))
 800     return " ".join(ret)
 801
 802
 803 def extract_pubkey():
 804     global repo_pubkey_fingerprint
 805     if 'repo_pubkey' in config:
 806         pubkey = unhexlify(config['repo_pubkey'])
 807     else:
 808         p = FDroidPopenBytes([config['keytool'], '-exportcert',
 809                               '-alias', config['repo_keyalias'],
 810                               '-keystore', config['keystore'],
 811                               '-storepass:file', config['keystorepassfile']]
 812                              + config['smartcardoptions'],
 813                              output=False, stderr_to_stdout=False)
 814         if p.returncode != 0 or len(p.output) < 20:
 815             msg = "Failed to get repo pubkey!"
 816             if config['keystore'] == 'NONE':
 817                 msg += ' Is your crypto smartcard plugged in?'
 818             logging.critical(msg)
 819             sys.exit(1)
 820         pubkey = p.output
 821     repo_pubkey_fingerprint = cert_fingerprint(pubkey)
 822     return hexlify(pubkey)
 823
 824
 825 def make_index(apps, sortedids, apks, repodir, archive, categories):
 826     """Make a repo index.
 827
 828     :param apps: fully populated apps list
 829     :param apks: full populated apks list
 830     :param repodir: the repo directory
 831     :param archive: True if this is the archive repo, False if it's the
 832                     main one.
 833     :param categories: list of categories
 834     """
 835
 836     doc = Document()
 837
 838     def addElement(name, value, doc, parent):
 839         el = doc.createElement(name)
 840         el.appendChild(doc.createTextNode(value))
 841         parent.appendChild(el)
 842
 843     def addElementNonEmpty(name, value, doc, parent):
 844         if not value:
 845             return
 846         addElement(name, value, doc, parent)
 847
 848     def addElementCDATA(name, value, doc, parent):
 849         el = doc.createElement(name)
 850         el.appendChild(doc.createCDATASection(value))
 851         parent.appendChild(el)
 852
 853     root = doc.createElement("fdroid")
 854     doc.appendChild(root)
 855
 856     repoel = doc.createElement("repo")
 857
 858     mirrorcheckfailed = False
 859     for mirror in config.get('mirrors', []):
 860         base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
 861         if config.get('nonstandardwebroot') is not True and base != 'fdroid':
 862             logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
 863             mirrorcheckfailed = True
 864     if mirrorcheckfailed:
 865         sys.exit(1)
 866
 867     if archive:
 868         repoel.setAttribute("name", config['archive_name'])
 869         if config['repo_maxage'] != 0:
 870             repoel.setAttribute("maxage", str(config['repo_maxage']))
 871         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 872         repoel.setAttribute("url", config['archive_url'])
 873         addElement('description', config['archive_description'], doc, repoel)
 874         urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
 875         for mirror in config.get('mirrors', []):
 876             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 877
 878     else:
 879         repoel.setAttribute("name", config['repo_name'])
 880         if config['repo_maxage'] != 0:
 881             repoel.setAttribute("maxage", str(config['repo_maxage']))
 882         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 883         repoel.setAttribute("url", config['repo_url'])
 884         addElement('description', config['repo_description'], doc, repoel)
 885         urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
 886         for mirror in config.get('mirrors', []):
 887             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 888
 889     repoel.setAttribute("version", str(METADATA_VERSION))
 890     repoel.setAttribute("timestamp", str(int(time.time())))
 891
 892     nosigningkey = False
 893     if not options.nosign:
 894         if 'repo_keyalias' not in config:
 895             nosigningkey = True
 896             logging.critical("'repo_keyalias' not found in config.py!")
 897         if 'keystore' not in config:
 898             nosigningkey = True
 899             logging.critical("'keystore' not found in config.py!")
 900         if 'keystorepass' not in config and 'keystorepassfile' not in config:
 901             nosigningkey = True
 902             logging.critical("'keystorepass' not found in config.py!")
 903         if 'keypass' not in config and 'keypassfile' not in config:
 904             nosigningkey = True
 905             logging.critical("'keypass' not found in config.py!")
 906         if not os.path.exists(config['keystore']):
 907             nosigningkey = True
 908             logging.critical("'" + config['keystore'] + "' does not exist!")
 909         if nosigningkey:
 910             logging.warning("`fdroid update` requires a signing key, you can create one using:")
 911             logging.warning("\tfdroid update --create-key")
 912             sys.exit(1)
 913
 914     repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
 915     root.appendChild(repoel)
 916
 917     for appid in sortedids:
 918         app = apps[appid]
 919
 920         if app.Disabled is not None:
 921             continue
 922
 923         # Get a list of the apks for this app...
 924         apklist = []
 925         for apk in apks:
 926             if apk['id'] == appid:
 927                 apklist.append(apk)
 928
 929         if len(apklist) == 0:
 930             continue
 931
 932         apel = doc.createElement("application")
 933         apel.setAttribute("id", app.id)
 934         root.appendChild(apel)
 935
 936         addElement('id', app.id, doc, apel)
 937         if app.added:
 938             addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
 939         if app.lastupdated:
 940             addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
 941         addElement('name', app.Name, doc, apel)
 942         addElement('summary', app.Summary, doc, apel)
 943         if app.icon:
 944             addElement('icon', app.icon, doc, apel)
 945
 946         def linkres(appid):
 947             if appid in apps:
 948                 return ("fdroid.app:" + appid, apps[appid].Name)
 949             raise MetaDataException("Cannot resolve app id " + appid)
 950
 951         addElement('desc',
 952                    metadata.description_html(app.Description, linkres),
 953                    doc, apel)
 954         addElement('license', app.License, doc, apel)
 955         if app.Categories:
 956             addElement('categories', ','.join(app.Categories), doc, apel)
 957             # We put the first (primary) category in LAST, which will have
 958             # the desired effect of making clients that only understand one
 959             # category see that one.
 960             addElement('category', app.Categories[0], doc, apel)
 961         addElement('web', app.WebSite, doc, apel)
 962         addElement('source', app.SourceCode, doc, apel)
 963         addElement('tracker', app.IssueTracker, doc, apel)
 964         addElementNonEmpty('changelog', app.Changelog, doc, apel)
 965         addElementNonEmpty('author', app.AuthorName, doc, apel)
 966         addElementNonEmpty('email', app.AuthorEmail, doc, apel)
 967         addElementNonEmpty('donate', app.Donate, doc, apel)
 968         addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
 969         addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
 970         addElementNonEmpty('flattr', app.FlattrID, doc, apel)
 971
 972         # These elements actually refer to the current version (i.e. which
 973         # one is recommended. They are historically mis-named, and need
 974         # changing, but stay like this for now to support existing clients.
 975         addElement('marketversion', app.CurrentVersion, doc, apel)
 976         addElement('marketvercode', app.CurrentVersionCode, doc, apel)
 977
 978         if app.AntiFeatures:
 979             af = app.AntiFeatures
 980             if af:
 981                 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
 982         if app.Provides:
 983             pv = app.Provides.split(',')
 984             addElementNonEmpty('provides', ','.join(pv), doc, apel)
 985         if app.RequiresRoot:
 986             addElement('requirements', 'root', doc, apel)
 987
 988         # Sort the apk list into version order, just so the web site
 989         # doesn't have to do any work by default...
 990         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 991
 992         # Check for duplicates - they will make the client unhappy...
 993         for i in range(len(apklist) - 1):
 994             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 995                 logging.critical("duplicate versions: '%s' - '%s'" % (
 996                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 997                 sys.exit(1)
 998
 999         current_version_code = 0
1000         current_version_file = None
1001         for apk in apklist:
1002             # find the APK for the "Current Version"
1003             if current_version_code < apk['versioncode']:
1004                 current_version_code = apk['versioncode']
1005             if current_version_code < int(app.CurrentVersionCode):
1006                 current_version_file = apk['apkname']
1007
1008             apkel = doc.createElement("package")
1009             apel.appendChild(apkel)
1010             addElement('version', apk['version'], doc, apkel)
1011             addElement('versioncode', str(apk['versioncode']), doc, apkel)
1012             addElement('apkname', apk['apkname'], doc, apkel)
1013             if 'srcname' in apk:
1014                 addElement('srcname', apk['srcname'], doc, apkel)
1015             for hash_type in ['sha256']:
1016                 if hash_type not in apk:
1017                     continue
1018                 hashel = doc.createElement("hash")
1019                 hashel.setAttribute("type", hash_type)
1020                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
1021                 apkel.appendChild(hashel)
1022             addElement('sig', apk['sig'], doc, apkel)
1023             addElement('size', str(apk['size']), doc, apkel)
1024             addElement('sdkver', str(apk['minSdkVersion']), doc, apkel)
1025             if 'targetSdkVersion' in apk:
1026                 addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
1027             if 'maxSdkVersion' in apk:
1028                 addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
1029             addElementNonEmpty('obbMainFile', apk.get('obbMainFile'), doc, apkel)
1030             addElementNonEmpty('obbPatchFile', apk.get('obbPatchFile'), doc, apkel)
1031             if 'added' in apk:
1032                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
1033             addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
1034             if 'nativecode' in apk:
1035                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
1036             addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
1037
1038         if current_version_file is not None \
1039                 and config['make_current_version_link'] \
1040                 and repodir == 'repo':  # only create these
1041             namefield = config['current_version_name_source']
1042             sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
1043             apklinkname = sanitized_name + '.apk'
1044             current_version_path = os.path.join(repodir, current_version_file)
1045             if os.path.islink(apklinkname):
1046                 os.remove(apklinkname)
1047             os.symlink(current_version_path, apklinkname)
1048             # also symlink gpg signature, if it exists
1049             for extension in ('.asc', '.sig'):
1050                 sigfile_path = current_version_path + extension
1051                 if os.path.exists(sigfile_path):
1052                     siglinkname = apklinkname + extension
1053                     if os.path.islink(siglinkname):
1054                         os.remove(siglinkname)
1055                     os.symlink(sigfile_path, siglinkname)
1056
1057     if options.pretty:
1058         output = doc.toprettyxml(encoding='utf-8')
1059     else:
1060         output = doc.toxml(encoding='utf-8')
1061
1062     with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
1063         f.write(output)
1064
1065     if 'repo_keyalias' in config:
1066
1067         if options.nosign:
1068             logging.info("Creating unsigned index in preparation for signing")
1069         else:
1070             logging.info("Creating signed index with this key (SHA256):")
1071             logging.info("%s" % repo_pubkey_fingerprint)
1072
1073         # Create a jar of the index...
1074         jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
1075         p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
1076         if p.returncode != 0:
1077             logging.critical("Failed to create {0}".format(jar_output))
1078             sys.exit(1)
1079
1080         # Sign the index...
1081         signed = os.path.join(repodir, 'index.jar')
1082         if options.nosign:
1083             # Remove old signed index if not signing
1084             if os.path.exists(signed):
1085                 os.remove(signed)
1086         else:
1087             args = [config['jarsigner'], '-keystore', config['keystore'],
1088                     '-storepass:file', config['keystorepassfile'],
1089                     '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1090                     signed, config['repo_keyalias']]
1091             if config['keystore'] == 'NONE':
1092                 args += config['smartcardoptions']
1093             else:  # smardcards never use -keypass
1094                 args += ['-keypass:file', config['keypassfile']]
1095             p = FDroidPopen(args)
1096             if p.returncode != 0:
1097                 logging.critical("Failed to sign index")
1098                 sys.exit(1)
1099
1100     # Copy the repo icon into the repo directory...
1101     icon_dir = os.path.join(repodir, 'icons')
1102     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1103     shutil.copyfile(config['repo_icon'], iconfilename)
1104
1105     # Write a category list in the repo to allow quick access...
1106     catdata = ''
1107     for cat in categories:
1108         catdata += cat + '\n'
1109     with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1110         f.write(catdata)
1111
1112
1113 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1114
1115     for appid, app in apps.items():
1116
1117         if app.ArchivePolicy:
1118             keepversions = int(app.ArchivePolicy[:-9])
1119         else:
1120             keepversions = defaultkeepversions
1121
1122         def filter_apk_list_sorted(apk_list):
1123             res = []
1124             for apk in apk_list:
1125                 if apk['id'] == appid:
1126                     res.append(apk)
1127
1128             # Sort the apk list by version code. First is highest/newest.
1129             return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1130
1131         def move_file(from_dir, to_dir, filename, ignore_missing):
1132             from_path = os.path.join(from_dir, filename)
1133             if ignore_missing and not os.path.exists(from_path):
1134                 return
1135             to_path = os.path.join(to_dir, filename)
1136             shutil.move(from_path, to_path)
1137
1138         logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1139                       .format(appid, len(apks), keepversions, len(archapks)))
1140
1141         if len(apks) > keepversions:
1142             apklist = filter_apk_list_sorted(apks)
1143             # Move back the ones we don't want.
1144             for apk in apklist[keepversions:]:
1145                 logging.info("Moving " + apk['apkname'] + " to archive")
1146                 move_file(repodir, archivedir, apk['apkname'], False)
1147                 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1148                 for density in all_screen_densities:
1149                     repo_icon_dir = get_icon_dir(repodir, density)
1150                     archive_icon_dir = get_icon_dir(archivedir, density)
1151                     if density not in apk['icons']:
1152                         continue
1153                     move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1154                 if 'srcname' in apk:
1155                     move_file(repodir, archivedir, apk['srcname'], False)
1156                 archapks.append(apk)
1157                 apks.remove(apk)
1158         elif len(apks) < keepversions and len(archapks) > 0:
1159             required = keepversions - len(apks)
1160             archapklist = filter_apk_list_sorted(archapks)
1161             # Move forward the ones we want again.
1162             for apk in archapklist[:required]:
1163                 logging.info("Moving " + apk['apkname'] + " from archive")
1164                 move_file(archivedir, repodir, apk['apkname'], False)
1165                 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1166                 for density in all_screen_densities:
1167                     repo_icon_dir = get_icon_dir(repodir, density)
1168                     archive_icon_dir = get_icon_dir(archivedir, density)
1169                     if density not in apk['icons']:
1170                         continue
1171                     move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1172                 if 'srcname' in apk:
1173                     move_file(archivedir, repodir, apk['srcname'], False)
1174                 archapks.remove(apk)
1175                 apks.append(apk)
1176
1177
1178 def add_apks_to_per_app_repos(repodir, apks):
1179     apks_per_app = dict()
1180     for apk in apks:
1181         apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1182         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1183         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1184         apks_per_app[apk['id']] = apk
1185
1186         if not os.path.exists(apk['per_app_icons']):
1187             logging.info('Adding new repo for only ' + apk['id'])
1188             os.makedirs(apk['per_app_icons'])
1189
1190         apkpath = os.path.join(repodir, apk['apkname'])
1191         shutil.copy(apkpath, apk['per_app_repo'])
1192         apksigpath = apkpath + '.sig'
1193         if os.path.exists(apksigpath):
1194             shutil.copy(apksigpath, apk['per_app_repo'])
1195         apkascpath = apkpath + '.asc'
1196         if os.path.exists(apkascpath):
1197             shutil.copy(apkascpath, apk['per_app_repo'])
1198
1199
1200 config = None
1201 options = None
1202
1203
1204 def main():
1205
1206     global config, options
1207
1208     # Parse command line...
1209     parser = ArgumentParser()
1210     common.setup_global_opts(parser)
1211     parser.add_argument("--create-key", action="store_true", default=False,
1212                         help="Create a repo signing key in a keystore")
1213     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1214                         help="Create skeleton metadata files that are missing")
1215     parser.add_argument("--delete-unknown", action="store_true", default=False,
1216                         help="Delete APKs and/or OBBs without metadata from the repo")
1217     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1218                         help="Report on build data status")
1219     parser.add_argument("-i", "--interactive", default=False, action="store_true",
1220                         help="Interactively ask about things that need updating.")
1221     parser.add_argument("-I", "--icons", action="store_true", default=False,
1222                         help="Resize all the icons exceeding the max pixel size and exit")
1223     parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1224                         help="Specify editor to use in interactive mode. Default " +
1225                         "is /etc/alternatives/editor")
1226     parser.add_argument("-w", "--wiki", default=False, action="store_true",
1227                         help="Update the wiki")
1228     parser.add_argument("--pretty", action="store_true", default=False,
1229                         help="Produce human-readable index.xml")
1230     parser.add_argument("--clean", action="store_true", default=False,
1231                         help="Clean update - don't uses caches, reprocess all apks")
1232     parser.add_argument("--nosign", action="store_true", default=False,
1233                         help="When configured for signed indexes, create only unsigned indexes at this stage")
1234     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1235                         help="Use date from apk instead of current time for newly added apks")
1236     options = parser.parse_args()
1237
1238     config = common.read_config(options)
1239
1240     if not ('jarsigner' in config and 'keytool' in config):
1241         logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1242         sys.exit(1)
1243
1244     repodirs = ['repo']
1245     if config['archive_older'] != 0:
1246         repodirs.append('archive')
1247         if not os.path.exists('archive'):
1248             os.mkdir('archive')
1249
1250     if options.icons:
1251         resize_all_icons(repodirs)
1252         sys.exit(0)
1253
1254     # check that icons exist now, rather than fail at the end of `fdroid update`
1255     for k in ['repo_icon', 'archive_icon']:
1256         if k in config:
1257             if not os.path.exists(config[k]):
1258                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1259                 sys.exit(1)
1260
1261     # if the user asks to create a keystore, do it now, reusing whatever it can
1262     if options.create_key:
1263         if os.path.exists(config['keystore']):
1264             logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1265             logging.critical("\t'" + config['keystore'] + "'")
1266             sys.exit(1)
1267
1268         if 'repo_keyalias' not in config:
1269             config['repo_keyalias'] = socket.getfqdn()
1270             common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1271         if 'keydname' not in config:
1272             config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1273             common.write_to_config(config, 'keydname', config['keydname'])
1274         if 'keystore' not in config:
1275             config['keystore'] = common.default_config.keystore
1276             common.write_to_config(config, 'keystore', config['keystore'])
1277
1278         password = common.genpassword()
1279         if 'keystorepass' not in config:
1280             config['keystorepass'] = password
1281             common.write_to_config(config, 'keystorepass', config['keystorepass'])
1282         if 'keypass' not in config:
1283             config['keypass'] = password
1284             common.write_to_config(config, 'keypass', config['keypass'])
1285         common.genkeystore(config)
1286
1287     # Get all apps...
1288     apps = metadata.read_metadata()
1289
1290     # Generate a list of categories...
1291     categories = set()
1292     for app in apps.values():
1293         categories.update(app.Categories)
1294
1295     # Read known apks data (will be updated and written back when we've finished)
1296     knownapks = common.KnownApks()
1297
1298     # Gather information about all the apk files in the repo directory, using
1299     # cached data if possible.
1300     apkcachefile = os.path.join('tmp', 'apkcache')
1301     if not options.clean and os.path.exists(apkcachefile):
1302         with open(apkcachefile, 'rb') as cf:
1303             apkcache = pickle.load(cf, encoding='utf-8')
1304         if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
1305             apkcache = {}
1306     else:
1307         apkcache = {}
1308
1309     delete_disabled_builds(apps, apkcache, repodirs)
1310
1311     # Scan all apks in the main repo
1312     apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1313
1314     # Generate warnings for apk's with no metadata (or create skeleton
1315     # metadata files, if requested on the command line)
1316     newmetadata = False
1317     for apk in apks:
1318         if apk['id'] not in apps:
1319             if options.create_metadata:
1320                 if 'name' not in apk:
1321                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1322                     continue
1323                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
1324                 f.write("License:Unknown\n")
1325                 f.write("Web Site:\n")
1326                 f.write("Source Code:\n")
1327                 f.write("Issue Tracker:\n")
1328                 f.write("Changelog:\n")
1329                 f.write("Summary:" + apk['name'] + "\n")
1330                 f.write("Description:\n")
1331                 f.write(apk['name'] + "\n")
1332                 f.write(".\n")
1333                 f.close()
1334                 logging.info("Generated skeleton metadata for " + apk['id'])
1335                 newmetadata = True
1336             else:
1337                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1338                 if options.delete_unknown:
1339                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1340                     rmf = os.path.join(repodirs[0], apk['apkname'])
1341                     if not os.path.exists(rmf):
1342                         logging.error("Could not find {0} to remove it".format(rmf))
1343                     else:
1344                         os.remove(rmf)
1345                 else:
1346                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1347
1348     # update the metadata with the newly created ones included
1349     if newmetadata:
1350         apps = metadata.read_metadata()
1351
1352     insert_obbs(repodirs[0], apps, apks)
1353
1354     # Scan the archive repo for apks as well
1355     if len(repodirs) > 1:
1356         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1357         if cc:
1358             cachechanged = True
1359     else:
1360         archapks = []
1361
1362     # Some information from the apks needs to be applied up to the application
1363     # level. When doing this, we use the info from the most recent version's apk.
1364     # We deal with figuring out when the app was added and last updated at the
1365     # same time.
1366     for appid, app in apps.items():
1367         bestver = 0
1368         for apk in apks + archapks:
1369             if apk['id'] == appid:
1370                 if apk['versioncode'] > bestver:
1371                     bestver = apk['versioncode']
1372                     bestapk = apk
1373
1374                 if 'added' in apk:
1375                     if not app.added or apk['added'] < app.added:
1376                         app.added = apk['added']
1377                     if not app.lastupdated or apk['added'] > app.lastupdated:
1378                         app.lastupdated = apk['added']
1379
1380         if not app.added:
1381             logging.debug("Don't know when " + appid + " was added")
1382         if not app.lastupdated:
1383             logging.debug("Don't know when " + appid + " was last updated")
1384
1385         if bestver == 0:
1386             if app.Name is None:
1387                 app.Name = app.AutoName or appid
1388             app.icon = None
1389             logging.debug("Application " + appid + " has no packages")
1390         else:
1391             if app.Name is None:
1392                 app.Name = bestapk['name']
1393             app.icon = bestapk['icon'] if 'icon' in bestapk else None
1394             if app.CurrentVersionCode is None:
1395                 app.CurrentVersionCode = str(bestver)
1396
1397     # Sort the app list by name, then the web site doesn't have to by default.
1398     # (we had to wait until we'd scanned the apks to do this, because mostly the
1399     # name comes from there!)
1400     sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1401
1402     # APKs are placed into multiple repos based on the app package, providing
1403     # per-app subscription feeds for nightly builds and things like it
1404     if config['per_app_repos']:
1405         add_apks_to_per_app_repos(repodirs[0], apks)
1406         for appid, app in apps.items():
1407             repodir = os.path.join(appid, 'fdroid', 'repo')
1408             appdict = dict()
1409             appdict[appid] = app
1410             if os.path.isdir(repodir):
1411                 make_index(appdict, [appid], apks, repodir, False, categories)
1412             else:
1413                 logging.info('Skipping index generation for ' + appid)
1414         return
1415
1416     if len(repodirs) > 1:
1417         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1418
1419     # Make the index for the main repo...
1420     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1421
1422     # If there's an archive repo,  make the index for it. We already scanned it
1423     # earlier on.
1424     if len(repodirs) > 1:
1425         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1426
1427     if config['update_stats']:
1428
1429         # Update known apks info...
1430         knownapks.writeifchanged()
1431
1432         # Generate latest apps data for widget
1433         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1434             data = ''
1435             with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1436                 for line in f:
1437                     appid = line.rstrip()
1438                     data += appid + "\t"
1439                     app = apps[appid]
1440                     data += app.Name + "\t"
1441                     if app.icon is not None:
1442                         data += app.icon + "\t"
1443                     data += app.License + "\n"
1444             with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1445                 f.write(data)
1446
1447     if cachechanged:
1448         apkcache["METADATA_VERSION"] = METADATA_VERSION
1449         with open(apkcachefile, 'wb') as cf:
1450             pickle.dump(apkcache, cf)
1451
1452     # Update the wiki...
1453     if options.wiki:
1454         update_wiki(apps, sortedids, apks + archapks)
1455
1456     logging.info("Finished.")
1457
1458 if __name__ == "__main__":
1459     main()