fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from pyasn1.error import PyAsn1Error
  33 from pyasn1.codec.der import decoder, encoder
  34 from pyasn1_modules import rfc2315
  35 from hashlib import md5
  36
  37 from PIL import Image
  38 import logging
  39
  40 import common
  41 import metadata
  42 from common import FDroidPopen, SdkToolsPopen
  43 from metadata import MetaDataException
  44
  45
  46 def get_densities():
  47     return ['640', '480', '320', '240', '160', '120']
  48
  49
  50 def dpi_to_px(density):
  51     return (int(density) * 48) / 160
  52
  53
  54 def px_to_dpi(px):
  55     return (int(px) * 160) / 48
  56
  57
  58 def get_icon_dir(repodir, density):
  59     if density is None:
  60         return os.path.join(repodir, "icons")
  61     return os.path.join(repodir, "icons-%s" % density)
  62
  63
  64 def get_icon_dirs(repodir):
  65     for density in get_densities():
  66         yield get_icon_dir(repodir, density)
  67     yield os.path.join(repodir, "icons")
  68
  69
  70 def update_wiki(apps, sortedids, apks):
  71     """Update the wiki
  72
  73     :param apps: fully populated list of all applications
  74     :param apks: all apks, except...
  75     """
  76     logging.info("Updating wiki")
  77     wikicat = 'Apps'
  78     wikiredircat = 'App Redirects'
  79     import mwclient
  80     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  81                          path=config['wiki_path'])
  82     site.login(config['wiki_user'], config['wiki_password'])
  83     generated_pages = {}
  84     generated_redirects = {}
  85
  86     for appid in sortedids:
  87         app = apps[appid]
  88
  89         wikidata = ''
  90         if app['Disabled']:
  91             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  92         if app['AntiFeatures']:
  93             for af in app['AntiFeatures'].split(','):
  94                 wikidata += '{{AntiFeature|' + af + '}}\n'
  95         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
  96             appid,
  97             app['Name'],
  98             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  99             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
 100             app['Source Code'],
 101             app['Issue Tracker'],
 102             app['Web Site'],
 103             app['Changelog'],
 104             app['Donate'],
 105             app['FlattrID'],
 106             app['Bitcoin'],
 107             app['Litecoin'],
 108             app['Dogecoin'],
 109             app['License'],
 110             app.get('Requires Root', 'No'))
 111
 112         if app['Provides']:
 113             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 114
 115         wikidata += app['Summary']
 116         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 117
 118         wikidata += "=Description=\n"
 119         wikidata += metadata.description_wiki(app['Description']) + "\n"
 120
 121         wikidata += "=Maintainer Notes=\n"
 122         if 'Maintainer Notes' in app:
 123             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 124         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 125
 126         # Get a list of all packages for this application...
 127         apklist = []
 128         gotcurrentver = False
 129         cantupdate = False
 130         buildfails = False
 131         for apk in apks:
 132             if apk['id'] == appid:
 133                 if str(apk['versioncode']) == app['Current Version Code']:
 134                     gotcurrentver = True
 135                 apklist.append(apk)
 136         # Include ones we can't build, as a special case...
 137         for thisbuild in app['builds']:
 138             if thisbuild['disable']:
 139                 if thisbuild['vercode'] == app['Current Version Code']:
 140                     cantupdate = True
 141                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 142                 apklist.append({'versioncode': int(thisbuild['vercode']),
 143                                 'version': thisbuild['version'],
 144                                 'buildproblem': thisbuild['disable']
 145                                 })
 146             else:
 147                 builtit = False
 148                 for apk in apklist:
 149                     if apk['versioncode'] == int(thisbuild['vercode']):
 150                         builtit = True
 151                         break
 152                 if not builtit:
 153                     buildfails = True
 154                     apklist.append({'versioncode': int(thisbuild['vercode']),
 155                                     'version': thisbuild['version'],
 156                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, thisbuild['vercode'])
 157                                     })
 158         if app['Current Version Code'] == '0':
 159             cantupdate = True
 160         # Sort with most recent first...
 161         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 162
 163         wikidata += "=Versions=\n"
 164         if len(apklist) == 0:
 165             wikidata += "We currently have no versions of this app available."
 166         elif not gotcurrentver:
 167             wikidata += "We don't have the current version of this app."
 168         else:
 169             wikidata += "We have the current version of this app."
 170         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 171         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 172         if len(app['No Source Since']) > 0:
 173             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 174         if len(app['Current Version']) > 0:
 175             wikidata += "The current (recommended) version is " + app['Current Version']
 176             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 177         validapks = 0
 178         for apk in apklist:
 179             wikidata += "==" + apk['version'] + "==\n"
 180
 181             if 'buildproblem' in apk:
 182                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 183             else:
 184                 validapks += 1
 185                 wikidata += "This version is built and signed by "
 186                 if 'srcname' in apk:
 187                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 188                 else:
 189                     wikidata += "the original developer.\n\n"
 190             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 191
 192         wikidata += '\n[[Category:' + wikicat + ']]\n'
 193         if len(app['No Source Since']) > 0:
 194             wikidata += '\n[[Category:Apps missing source code]]\n'
 195         if validapks == 0 and not app['Disabled']:
 196             wikidata += '\n[[Category:Apps with no packages]]\n'
 197         if cantupdate and not app['Disabled']:
 198             wikidata += "\n[[Category:Apps we can't update]]\n"
 199         if buildfails and not app['Disabled']:
 200             wikidata += "\n[[Category:Apps with failing builds]]\n"
 201         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 202             wikidata += '\n[[Category:Apps to Update]]\n'
 203         if app['Disabled']:
 204             wikidata += '\n[[Category:Apps that are disabled]]\n'
 205         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 206             wikidata += '\n[[Category:Apps with no update check]]\n'
 207         for appcat in app['Categories']:
 208             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 209
 210         # We can't have underscores in the page name, even if they're in
 211         # the package ID, because MediaWiki messes with them...
 212         pagename = appid.replace('_', ' ')
 213
 214         # Drop a trailing newline, because mediawiki is going to drop it anyway
 215         # and it we don't we'll think the page has changed when it hasn't...
 216         if wikidata.endswith('\n'):
 217             wikidata = wikidata[:-1]
 218
 219         generated_pages[pagename] = wikidata
 220
 221         # Make a redirect from the name to the ID too, unless there's
 222         # already an existing page with the name and it isn't a redirect.
 223         noclobber = False
 224         apppagename = app['Name'].replace('_', ' ')
 225         apppagename = apppagename.replace('{', '')
 226         apppagename = apppagename.replace('}', ' ')
 227         apppagename = apppagename.replace(':', ' ')
 228         # Drop double spaces caused mostly by replacing ':' above
 229         apppagename = apppagename.replace('  ', ' ')
 230         for expagename in site.allpages(prefix=apppagename,
 231                                         filterredir='nonredirects',
 232                                         generator=False):
 233             if expagename == apppagename:
 234                 noclobber = True
 235         # Another reason not to make the redirect page is if the app name
 236         # is the same as it's ID, because that will overwrite the real page
 237         # with an redirect to itself! (Although it seems like an odd
 238         # scenario this happens a lot, e.g. where there is metadata but no
 239         # builds or binaries to extract a name from.
 240         if apppagename == pagename:
 241             noclobber = True
 242         if not noclobber:
 243             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 244
 245     for tcat, genp in [(wikicat, generated_pages),
 246                        (wikiredircat, generated_redirects)]:
 247         catpages = site.Pages['Category:' + tcat]
 248         existingpages = []
 249         for page in catpages:
 250             existingpages.append(page.name)
 251             if page.name in genp:
 252                 pagetxt = page.edit()
 253                 if pagetxt != genp[page.name]:
 254                     logging.debug("Updating modified page " + page.name)
 255                     page.save(genp[page.name], summary='Auto-updated')
 256                 else:
 257                     logging.debug("Page " + page.name + " is unchanged")
 258             else:
 259                 logging.warn("Deleting page " + page.name)
 260                 page.delete('No longer published')
 261         for pagename, text in genp.items():
 262             logging.debug("Checking " + pagename)
 263             if pagename not in existingpages:
 264                 logging.debug("Creating page " + pagename)
 265                 try:
 266                     newpage = site.Pages[pagename]
 267                     newpage.save(text, summary='Auto-created')
 268                 except:
 269                     logging.error("...FAILED to create page")
 270
 271     # Purge server cache to ensure counts are up to date
 272     site.pages['Repository Maintenance'].purge()
 273
 274
 275 def delete_disabled_builds(apps, apkcache, repodirs):
 276     """Delete disabled build outputs.
 277
 278     :param apps: list of all applications, as per metadata.read_metadata
 279     :param apkcache: current apk cache information
 280     :param repodirs: the repo directories to process
 281     """
 282     for appid, app in apps.iteritems():
 283         for build in app['builds']:
 284             if build['disable']:
 285                 apkfilename = appid + '_' + str(build['vercode']) + '.apk'
 286                 for repodir in repodirs:
 287                     apkpath = os.path.join(repodir, apkfilename)
 288                     ascpath = apkpath + ".asc"
 289                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 290                     for name in [apkpath, srcpath, ascpath]:
 291                         if os.path.exists(name):
 292                             logging.warn("Deleting disabled build output " + apkfilename)
 293                             os.remove(name)
 294                 if apkfilename in apkcache:
 295                     del apkcache[apkfilename]
 296
 297
 298 def resize_icon(iconpath, density):
 299
 300     if not os.path.isfile(iconpath):
 301         return
 302
 303     try:
 304         im = Image.open(iconpath)
 305         size = dpi_to_px(density)
 306
 307         if any(length > size for length in im.size):
 308             oldsize = im.size
 309             im.thumbnail((size, size), Image.ANTIALIAS)
 310             logging.debug("%s was too large at %s - new size is %s" % (
 311                 iconpath, oldsize, im.size))
 312             im.save(iconpath, "PNG")
 313
 314     except Exception, e:
 315         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 316
 317
 318 def resize_all_icons(repodirs):
 319     """Resize all icons that exceed the max size
 320
 321     :param repodirs: the repo directories to process
 322     """
 323     for repodir in repodirs:
 324         for density in get_densities():
 325             icon_dir = get_icon_dir(repodir, density)
 326             icon_glob = os.path.join(icon_dir, '*.png')
 327             for iconpath in glob.glob(icon_glob):
 328                 resize_icon(iconpath, density)
 329
 330
 331 cert_path_regex = re.compile(r'^META-INF/.*\.RSA$')
 332
 333
 334 def getsig(apkpath):
 335     """ Get the signing certificate of an apk. To get the same md5 has that
 336     Android gets, we encode the .RSA certificate in a specific format and pass
 337     it hex-encoded to the md5 digest algorithm.
 338
 339     :param apkpath: path to the apk
 340     :returns: A string containing the md5 of the signature of the apk or None
 341               if an error occurred.
 342     """
 343
 344     cert = None
 345
 346     # verify the jar signature is correct
 347     args = ['jarsigner', '-verify', apkpath]
 348     p = FDroidPopen(args)
 349     if p.returncode != 0:
 350         logging.critical(apkpath + " has a bad signature!")
 351         return None
 352
 353     with zipfile.ZipFile(apkpath, 'r') as apk:
 354
 355         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 356
 357         if len(certs) < 1:
 358             logging.error("Found no signing certificates on %s" % apkpath)
 359             return None
 360         if len(certs) > 1:
 361             logging.error("Found multiple signing certificates on %s" % apkpath)
 362             return None
 363
 364         cert = apk.read(certs[0])
 365
 366     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 367     if content.getComponentByName('contentType') != rfc2315.signedData:
 368         logging.error("Unexpected format.")
 369         return None
 370
 371     content = decoder.decode(content.getComponentByName('content'),
 372                              asn1Spec=rfc2315.SignedData())[0]
 373     try:
 374         certificates = content.getComponentByName('certificates')
 375     except PyAsn1Error:
 376         logging.error("Certificates not found.")
 377         return None
 378
 379     cert_encoded = encoder.encode(certificates)[4:]
 380
 381     return md5(cert_encoded.encode('hex')).hexdigest()
 382
 383
 384 def scan_apks(apps, apkcache, repodir, knownapks):
 385     """Scan the apks in the given repo directory.
 386
 387     This also extracts the icons.
 388
 389     :param apps: list of all applications, as per metadata.read_metadata
 390     :param apkcache: current apk cache information
 391     :param repodir: repo directory to scan
 392     :param knownapks: known apks info
 393     :returns: (apks, cachechanged) where apks is a list of apk information,
 394               and cachechanged is True if the apkcache got changed.
 395     """
 396
 397     cachechanged = False
 398
 399     icon_dirs = get_icon_dirs(repodir)
 400     for icon_dir in icon_dirs:
 401         if os.path.exists(icon_dir):
 402             if options.clean:
 403                 shutil.rmtree(icon_dir)
 404                 os.makedirs(icon_dir)
 405         else:
 406             os.makedirs(icon_dir)
 407
 408     apks = []
 409     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 410     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 411     vername_pat = re.compile(".*versionName='([^']*)'.*")
 412     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 413     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 414     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 415     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 416     string_pat = re.compile(".*'([^']*)'.*")
 417     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 418
 419         apkfilename = apkfile[len(repodir) + 1:]
 420         if ' ' in apkfilename:
 421             logging.critical("Spaces in filenames are not allowed.")
 422             sys.exit(1)
 423
 424         if apkfilename in apkcache:
 425             logging.debug("Reading " + apkfilename + " from cache")
 426             thisinfo = apkcache[apkfilename]
 427
 428         else:
 429             logging.debug("Processing " + apkfilename)
 430             thisinfo = {}
 431             thisinfo['apkname'] = apkfilename
 432             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 433             if os.path.exists(os.path.join(repodir, srcfilename)):
 434                 thisinfo['srcname'] = srcfilename
 435             thisinfo['size'] = os.path.getsize(apkfile)
 436             thisinfo['permissions'] = set()
 437             thisinfo['features'] = set()
 438             thisinfo['icons_src'] = {}
 439             thisinfo['icons'] = {}
 440             p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
 441             if p.returncode != 0:
 442                 if options.delete_unknown:
 443                     if os.path.exists(apkfile):
 444                         logging.error("Failed to get apk information, deleting " + apkfile)
 445                         os.remove(apkfile)
 446                     else:
 447                         logging.error("Could not find {0} to remove it".format(apkfile))
 448                 else:
 449                     logging.error("Failed to get apk information, skipping " + apkfile)
 450                 continue
 451             for line in p.output.splitlines():
 452                 if line.startswith("package:"):
 453                     try:
 454                         thisinfo['id'] = re.match(name_pat, line).group(1)
 455                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 456                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 457                     except Exception, e:
 458                         logging.error("Package matching failed: " + str(e))
 459                         logging.info("Line was: " + line)
 460                         sys.exit(1)
 461                 elif line.startswith("application:"):
 462                     thisinfo['name'] = re.match(label_pat, line).group(1)
 463                     # Keep path to non-dpi icon in case we need it
 464                     match = re.match(icon_pat_nodpi, line)
 465                     if match:
 466                         thisinfo['icons_src']['-1'] = match.group(1)
 467                 elif line.startswith("launchable-activity:"):
 468                     # Only use launchable-activity as fallback to application
 469                     if not thisinfo['name']:
 470                         thisinfo['name'] = re.match(label_pat, line).group(1)
 471                     if '-1' not in thisinfo['icons_src']:
 472                         match = re.match(icon_pat_nodpi, line)
 473                         if match:
 474                             thisinfo['icons_src']['-1'] = match.group(1)
 475                 elif line.startswith("application-icon-"):
 476                     match = re.match(icon_pat, line)
 477                     if match:
 478                         density = match.group(1)
 479                         path = match.group(2)
 480                         thisinfo['icons_src'][density] = path
 481                 elif line.startswith("sdkVersion:"):
 482                     m = re.match(sdkversion_pat, line)
 483                     if m is None:
 484                         logging.error(line.replace('sdkVersion:', '')
 485                                       + ' is not a valid minSdkVersion!')
 486                     else:
 487                         thisinfo['sdkversion'] = m.group(1)
 488                 elif line.startswith("maxSdkVersion:"):
 489                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 490                 elif line.startswith("native-code:"):
 491                     thisinfo['nativecode'] = []
 492                     for arch in line[13:].split(' '):
 493                         thisinfo['nativecode'].append(arch[1:-1])
 494                 elif line.startswith("uses-permission:"):
 495                     perm = re.match(string_pat, line).group(1)
 496                     if perm.startswith("android.permission."):
 497                         perm = perm[19:]
 498                     thisinfo['permissions'].add(perm)
 499                 elif line.startswith("uses-feature:"):
 500                     perm = re.match(string_pat, line).group(1)
 501                     # Filter out this, it's only added with the latest SDK tools and
 502                     # causes problems for lots of apps.
 503                     if perm != "android.hardware.screen.portrait" \
 504                             and perm != "android.hardware.screen.landscape":
 505                         if perm.startswith("android.feature."):
 506                             perm = perm[16:]
 507                         thisinfo['features'].add(perm)
 508
 509             if 'sdkversion' not in thisinfo:
 510                 logging.warn("No SDK version information found in {0}".format(apkfile))
 511                 thisinfo['sdkversion'] = 0
 512
 513             # Check for debuggable apks...
 514             if common.isApkDebuggable(apkfile, config):
 515                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 516
 517             # Calculate the sha256...
 518             sha = hashlib.sha256()
 519             with open(apkfile, 'rb') as f:
 520                 while True:
 521                     t = f.read(1024)
 522                     if len(t) == 0:
 523                         break
 524                     sha.update(t)
 525                 thisinfo['sha256'] = sha.hexdigest()
 526
 527             # Get the signature (or md5 of, to be precise)...
 528             thisinfo['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 529             if not thisinfo['sig']:
 530                 logging.critical("Failed to get apk signature")
 531                 sys.exit(1)
 532
 533             apk = zipfile.ZipFile(apkfile, 'r')
 534
 535             iconfilename = "%s.%s.png" % (
 536                 thisinfo['id'],
 537                 thisinfo['versioncode'])
 538
 539             # Extract the icon file...
 540             densities = get_densities()
 541             empty_densities = []
 542             for density in densities:
 543                 if density not in thisinfo['icons_src']:
 544                     empty_densities.append(density)
 545                     continue
 546                 iconsrc = thisinfo['icons_src'][density]
 547                 icon_dir = get_icon_dir(repodir, density)
 548                 icondest = os.path.join(icon_dir, iconfilename)
 549
 550                 try:
 551                     iconfile = open(icondest, 'wb')
 552                     iconfile.write(apk.read(iconsrc))
 553                     iconfile.close()
 554                     thisinfo['icons'][density] = iconfilename
 555
 556                 except:
 557                     logging.warn("Error retrieving icon file")
 558                     del thisinfo['icons'][density]
 559                     del thisinfo['icons_src'][density]
 560                     empty_densities.append(density)
 561
 562             if '-1' in thisinfo['icons_src']:
 563                 iconsrc = thisinfo['icons_src']['-1']
 564                 iconpath = os.path.join(
 565                     get_icon_dir(repodir, None), iconfilename)
 566                 iconfile = open(iconpath, 'wb')
 567                 iconfile.write(apk.read(iconsrc))
 568                 iconfile.close()
 569                 try:
 570                     im = Image.open(iconpath)
 571                     dpi = px_to_dpi(im.size[0])
 572                     for density in densities:
 573                         if density in thisinfo['icons']:
 574                             break
 575                         if density == densities[-1] or dpi >= int(density):
 576                             thisinfo['icons'][density] = iconfilename
 577                             shutil.move(iconpath,
 578                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 579                             empty_densities.remove(density)
 580                             break
 581                 except Exception, e:
 582                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 583
 584             if thisinfo['icons']:
 585                 thisinfo['icon'] = iconfilename
 586
 587             apk.close()
 588
 589             # First try resizing down to not lose quality
 590             last_density = None
 591             for density in densities:
 592                 if density not in empty_densities:
 593                     last_density = density
 594                     continue
 595                 if last_density is None:
 596                     continue
 597                 logging.debug("Density %s not available, resizing down from %s"
 598                               % (density, last_density))
 599
 600                 last_iconpath = os.path.join(
 601                     get_icon_dir(repodir, last_density), iconfilename)
 602                 iconpath = os.path.join(
 603                     get_icon_dir(repodir, density), iconfilename)
 604                 try:
 605                     im = Image.open(last_iconpath)
 606                 except:
 607                     logging.warn("Invalid image file at %s" % last_iconpath)
 608                     continue
 609
 610                 size = dpi_to_px(density)
 611
 612                 im.thumbnail((size, size), Image.ANTIALIAS)
 613                 im.save(iconpath, "PNG")
 614                 empty_densities.remove(density)
 615
 616             # Then just copy from the highest resolution available
 617             last_density = None
 618             for density in reversed(densities):
 619                 if density not in empty_densities:
 620                     last_density = density
 621                     continue
 622                 if last_density is None:
 623                     continue
 624                 logging.debug("Density %s not available, copying from lower density %s"
 625                               % (density, last_density))
 626
 627                 shutil.copyfile(
 628                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 629                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 630
 631                 empty_densities.remove(density)
 632
 633             for density in densities:
 634                 icon_dir = get_icon_dir(repodir, density)
 635                 icondest = os.path.join(icon_dir, iconfilename)
 636                 resize_icon(icondest, density)
 637
 638             # Copy from icons-mdpi to icons since mdpi is the baseline density
 639             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 640             if os.path.isfile(baseline):
 641                 shutil.copyfile(baseline,
 642                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 643
 644             # Record in known apks, getting the added date at the same time..
 645             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 646             if added:
 647                 thisinfo['added'] = added
 648
 649             apkcache[apkfilename] = thisinfo
 650             cachechanged = True
 651
 652         apks.append(thisinfo)
 653
 654     return apks, cachechanged
 655
 656
 657 repo_pubkey_fingerprint = None
 658
 659
 660 def make_index(apps, sortedids, apks, repodir, archive, categories):
 661     """Make a repo index.
 662
 663     :param apps: fully populated apps list
 664     :param apks: full populated apks list
 665     :param repodir: the repo directory
 666     :param archive: True if this is the archive repo, False if it's the
 667                     main one.
 668     :param categories: list of categories
 669     """
 670
 671     doc = Document()
 672
 673     def addElement(name, value, doc, parent):
 674         el = doc.createElement(name)
 675         el.appendChild(doc.createTextNode(value))
 676         parent.appendChild(el)
 677
 678     def addElementCDATA(name, value, doc, parent):
 679         el = doc.createElement(name)
 680         el.appendChild(doc.createCDATASection(value))
 681         parent.appendChild(el)
 682
 683     root = doc.createElement("fdroid")
 684     doc.appendChild(root)
 685
 686     repoel = doc.createElement("repo")
 687
 688     if archive:
 689         repoel.setAttribute("name", config['archive_name'])
 690         if config['repo_maxage'] != 0:
 691             repoel.setAttribute("maxage", str(config['repo_maxage']))
 692         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 693         repoel.setAttribute("url", config['archive_url'])
 694         addElement('description', config['archive_description'], doc, repoel)
 695
 696     else:
 697         repoel.setAttribute("name", config['repo_name'])
 698         if config['repo_maxage'] != 0:
 699             repoel.setAttribute("maxage", str(config['repo_maxage']))
 700         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 701         repoel.setAttribute("url", config['repo_url'])
 702         addElement('description', config['repo_description'], doc, repoel)
 703
 704     repoel.setAttribute("version", "12")
 705     repoel.setAttribute("timestamp", str(int(time.time())))
 706
 707     if 'repo_keyalias' in config:
 708
 709         # Generate a certificate fingerprint the same way keytool does it
 710         # (but with slightly different formatting)
 711         def cert_fingerprint(data):
 712             digest = hashlib.sha256(data).digest()
 713             ret = []
 714             ret.append(' '.join("%02X" % ord(b) for b in digest))
 715             return " ".join(ret)
 716
 717         def extract_pubkey():
 718             p = FDroidPopen(['keytool', '-exportcert',
 719                              '-alias', config['repo_keyalias'],
 720                              '-keystore', config['keystore'],
 721                              '-storepass:file', config['keystorepassfile']]
 722                             + config['smartcardoptions'], output=False)
 723             if p.returncode != 0:
 724                 msg = "Failed to get repo pubkey!"
 725                 if config['keystore'] == 'NONE':
 726                     msg += ' Is your crypto smartcard plugged in?'
 727                 logging.critical(msg)
 728                 sys.exit(1)
 729             global repo_pubkey_fingerprint
 730             repo_pubkey_fingerprint = cert_fingerprint(p.output)
 731             return "".join("%02x" % ord(b) for b in p.output)
 732
 733         repoel.setAttribute("pubkey", extract_pubkey())
 734
 735     root.appendChild(repoel)
 736
 737     for appid in sortedids:
 738         app = apps[appid]
 739
 740         if app['Disabled'] is not None:
 741             continue
 742
 743         # Get a list of the apks for this app...
 744         apklist = []
 745         for apk in apks:
 746             if apk['id'] == appid:
 747                 apklist.append(apk)
 748
 749         if len(apklist) == 0:
 750             continue
 751
 752         apel = doc.createElement("application")
 753         apel.setAttribute("id", app['id'])
 754         root.appendChild(apel)
 755
 756         addElement('id', app['id'], doc, apel)
 757         if 'added' in app:
 758             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 759         if 'lastupdated' in app:
 760             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 761         addElement('name', app['Name'], doc, apel)
 762         addElement('summary', app['Summary'], doc, apel)
 763         if app['icon']:
 764             addElement('icon', app['icon'], doc, apel)
 765
 766         def linkres(appid):
 767             if appid in apps:
 768                 return ("fdroid.app:" + appid, apps[appid]['Name'])
 769             raise MetaDataException("Cannot resolve app id " + appid)
 770
 771         addElement('desc',
 772                    metadata.description_html(app['Description'], linkres),
 773                    doc, apel)
 774         addElement('license', app['License'], doc, apel)
 775         if 'Categories' in app:
 776             addElement('categories', ','.join(app["Categories"]), doc, apel)
 777             # We put the first (primary) category in LAST, which will have
 778             # the desired effect of making clients that only understand one
 779             # category see that one.
 780             addElement('category', app["Categories"][0], doc, apel)
 781         addElement('web', app['Web Site'], doc, apel)
 782         addElement('source', app['Source Code'], doc, apel)
 783         addElement('tracker', app['Issue Tracker'], doc, apel)
 784         addElement('changelog', app['Changelog'], doc, apel)
 785         if app['Donate']:
 786             addElement('donate', app['Donate'], doc, apel)
 787         if app['Bitcoin']:
 788             addElement('bitcoin', app['Bitcoin'], doc, apel)
 789         if app['Litecoin']:
 790             addElement('litecoin', app['Litecoin'], doc, apel)
 791         if app['Dogecoin']:
 792             addElement('dogecoin', app['Dogecoin'], doc, apel)
 793         if app['FlattrID']:
 794             addElement('flattr', app['FlattrID'], doc, apel)
 795
 796         # These elements actually refer to the current version (i.e. which
 797         # one is recommended. They are historically mis-named, and need
 798         # changing, but stay like this for now to support existing clients.
 799         addElement('marketversion', app['Current Version'], doc, apel)
 800         addElement('marketvercode', app['Current Version Code'], doc, apel)
 801
 802         if app['AntiFeatures']:
 803             af = app['AntiFeatures'].split(',')
 804             # TODO: Temporarily not including UpstreamNonFree in the index,
 805             # because current F-Droid clients do not understand it, and also
 806             # look ugly when they encounter an unknown antifeature. This
 807             # filtering can be removed in time...
 808             if 'UpstreamNonFree' in af:
 809                 af.remove('UpstreamNonFree')
 810             if af:
 811                 addElement('antifeatures', ','.join(af), doc, apel)
 812         if app['Provides']:
 813             pv = app['Provides'].split(',')
 814             addElement('provides', ','.join(pv), doc, apel)
 815         if app['Requires Root']:
 816             addElement('requirements', 'root', doc, apel)
 817
 818         # Sort the apk list into version order, just so the web site
 819         # doesn't have to do any work by default...
 820         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 821
 822         # Check for duplicates - they will make the client unhappy...
 823         for i in range(len(apklist) - 1):
 824             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 825                 logging.critical("duplicate versions: '%s' - '%s'" % (
 826                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 827                 sys.exit(1)
 828
 829         current_version_code = 0
 830         current_version_file = None
 831         for apk in apklist:
 832             # find the APK for the "Current Version"
 833             if current_version_code < apk['versioncode']:
 834                 current_version_code = apk['versioncode']
 835             if current_version_code < int(app['Current Version Code']):
 836                 current_version_file = apk['apkname']
 837
 838             apkel = doc.createElement("package")
 839             apel.appendChild(apkel)
 840             addElement('version', apk['version'], doc, apkel)
 841             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 842             addElement('apkname', apk['apkname'], doc, apkel)
 843             if 'srcname' in apk:
 844                 addElement('srcname', apk['srcname'], doc, apkel)
 845             for hash_type in ['sha256']:
 846                 if hash_type not in apk:
 847                     continue
 848                 hashel = doc.createElement("hash")
 849                 hashel.setAttribute("type", hash_type)
 850                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 851                 apkel.appendChild(hashel)
 852             addElement('sig', apk['sig'], doc, apkel)
 853             addElement('size', str(apk['size']), doc, apkel)
 854             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 855             if 'maxsdkversion' in apk:
 856                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 857             if 'added' in apk:
 858                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 859             if app['Requires Root']:
 860                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 861                     apk['permissions'].add('ACCESS_SUPERUSER')
 862
 863             if len(apk['permissions']) > 0:
 864                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 865             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 866                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 867             if len(apk['features']) > 0:
 868                 addElement('features', ','.join(apk['features']), doc, apkel)
 869
 870         if current_version_file is not None \
 871                 and config['make_current_version_link'] \
 872                 and repodir == 'repo':  # only create these
 873             sanitized_name = re.sub('''[ '"&%?+=/]''', '',
 874                                     app[config['current_version_name_source']])
 875             apklinkname = sanitized_name + '.apk'
 876             current_version_path = os.path.join(repodir, current_version_file)
 877             if os.path.exists(apklinkname):
 878                 os.remove(apklinkname)
 879             os.symlink(current_version_path, apklinkname)
 880             # also symlink gpg signature, if it exists
 881             for extension in ('.asc', '.sig'):
 882                 sigfile_path = current_version_path + extension
 883                 if os.path.exists(sigfile_path):
 884                     siglinkname = apklinkname + extension
 885                     if os.path.exists(siglinkname):
 886                         os.remove(siglinkname)
 887                     os.symlink(sigfile_path, siglinkname)
 888
 889     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 890     if options.pretty:
 891         output = doc.toprettyxml()
 892     else:
 893         output = doc.toxml()
 894     of.write(output)
 895     of.close()
 896
 897     if 'repo_keyalias' in config:
 898
 899         logging.info("Creating signed index with this key (SHA256):")
 900         logging.info("%s" % repo_pubkey_fingerprint)
 901
 902         # Create a jar of the index...
 903         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 904         if p.returncode != 0:
 905             logging.critical("Failed to create jar file")
 906             sys.exit(1)
 907
 908         # Sign the index...
 909         args = ['jarsigner', '-keystore', config['keystore'],
 910                 '-storepass:file', config['keystorepassfile'],
 911                 '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 912                 os.path.join(repodir, 'index.jar'), config['repo_keyalias']]
 913         if config['keystore'] == 'NONE':
 914             args += config['smartcardoptions']
 915         else:  # smardcards never use -keypass
 916             args += ['-keypass:file', config['keypassfile']]
 917         p = FDroidPopen(args)
 918         # TODO keypass should be sent via stdin
 919         if p.returncode != 0:
 920             logging.critical("Failed to sign index")
 921             sys.exit(1)
 922
 923     # Copy the repo icon into the repo directory...
 924     icon_dir = os.path.join(repodir, 'icons')
 925     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 926     shutil.copyfile(config['repo_icon'], iconfilename)
 927
 928     # Write a category list in the repo to allow quick access...
 929     catdata = ''
 930     for cat in categories:
 931         catdata += cat + '\n'
 932     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 933     f.write(catdata)
 934     f.close()
 935
 936
 937 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 938
 939     for appid, app in apps.iteritems():
 940
 941         # Get a list of the apks for this app...
 942         apklist = []
 943         for apk in apks:
 944             if apk['id'] == appid:
 945                 apklist.append(apk)
 946
 947         # Sort the apk list into version order...
 948         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 949
 950         if app['Archive Policy']:
 951             keepversions = int(app['Archive Policy'][:-9])
 952         else:
 953             keepversions = defaultkeepversions
 954
 955         if len(apklist) > keepversions:
 956             for apk in apklist[keepversions:]:
 957                 logging.info("Moving " + apk['apkname'] + " to archive")
 958                 shutil.move(os.path.join(repodir, apk['apkname']),
 959                             os.path.join(archivedir, apk['apkname']))
 960                 if 'srcname' in apk:
 961                     shutil.move(os.path.join(repodir, apk['srcname']),
 962                                 os.path.join(archivedir, apk['srcname']))
 963                     # Move GPG signature too...
 964                     sigfile = apk['srcname'] + '.asc'
 965                     sigsrc = os.path.join(repodir, sigfile)
 966                     if os.path.exists(sigsrc):
 967                         shutil.move(sigsrc, os.path.join(archivedir, sigfile))
 968
 969                 archapks.append(apk)
 970                 apks.remove(apk)
 971
 972
 973 config = None
 974 options = None
 975
 976
 977 def main():
 978
 979     global config, options
 980
 981     # Parse command line...
 982     parser = OptionParser()
 983     parser.add_option("-c", "--create-metadata", action="store_true", default=False,
 984                       help="Create skeleton metadata files that are missing")
 985     parser.add_option("--delete-unknown", action="store_true", default=False,
 986                       help="Delete APKs without metadata from the repo")
 987     parser.add_option("-v", "--verbose", action="store_true", default=False,
 988                       help="Spew out even more information than normal")
 989     parser.add_option("-q", "--quiet", action="store_true", default=False,
 990                       help="Restrict output to warnings and errors")
 991     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 992                       help="Report on build data status")
 993     parser.add_option("-i", "--interactive", default=False, action="store_true",
 994                       help="Interactively ask about things that need updating.")
 995     parser.add_option("-I", "--icons", action="store_true", default=False,
 996                       help="Resize all the icons exceeding the max pixel size and exit")
 997     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 998                       help="Specify editor to use in interactive mode. Default " +
 999                       "is /etc/alternatives/editor")
1000     parser.add_option("-w", "--wiki", default=False, action="store_true",
1001                       help="Update the wiki")
1002     parser.add_option("", "--pretty", action="store_true", default=False,
1003                       help="Produce human-readable index.xml")
1004     parser.add_option("--clean", action="store_true", default=False,
1005                       help="Clean update - don't uses caches, reprocess all apks")
1006     (options, args) = parser.parse_args()
1007
1008     config = common.read_config(options)
1009
1010     repodirs = ['repo']
1011     if config['archive_older'] != 0:
1012         repodirs.append('archive')
1013         if not os.path.exists('archive'):
1014             os.mkdir('archive')
1015
1016     if options.icons:
1017         resize_all_icons(repodirs)
1018         sys.exit(0)
1019
1020     # check that icons exist now, rather than fail at the end of `fdroid update`
1021     for k in ['repo_icon', 'archive_icon']:
1022         if k in config:
1023             if not os.path.exists(config[k]):
1024                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1025                 sys.exit(1)
1026
1027     # Get all apps...
1028     apps = metadata.read_metadata()
1029
1030     # Generate a list of categories...
1031     categories = set()
1032     for app in apps.itervalues():
1033         categories.update(app['Categories'])
1034
1035     # Read known apks data (will be updated and written back when we've finished)
1036     knownapks = common.KnownApks()
1037
1038     # Gather information about all the apk files in the repo directory, using
1039     # cached data if possible.
1040     apkcachefile = os.path.join('tmp', 'apkcache')
1041     if not options.clean and os.path.exists(apkcachefile):
1042         with open(apkcachefile, 'rb') as cf:
1043             apkcache = pickle.load(cf)
1044     else:
1045         apkcache = {}
1046     cachechanged = False
1047
1048     delete_disabled_builds(apps, apkcache, repodirs)
1049
1050     # Scan all apks in the main repo
1051     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
1052     if cc:
1053         cachechanged = True
1054
1055     # Generate warnings for apk's with no metadata (or create skeleton
1056     # metadata files, if requested on the command line)
1057     newmetadata = False
1058     for apk in apks:
1059         if apk['id'] not in apps:
1060             if options.create_metadata:
1061                 if 'name' not in apk:
1062                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1063                     continue
1064                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1065                 f.write("License:Unknown\n")
1066                 f.write("Web Site:\n")
1067                 f.write("Source Code:\n")
1068                 f.write("Issue Tracker:\n")
1069                 f.write("Changelog:\n")
1070                 f.write("Summary:" + apk['name'] + "\n")
1071                 f.write("Description:\n")
1072                 f.write(apk['name'] + "\n")
1073                 f.write(".\n")
1074                 f.close()
1075                 logging.info("Generated skeleton metadata for " + apk['id'])
1076                 newmetadata = True
1077             else:
1078                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1079                 if options.delete_unknown:
1080                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1081                     rmf = os.path.join(repodirs[0], apk['apkname'])
1082                     if not os.path.exists(rmf):
1083                         logging.error("Could not find {0} to remove it".format(rmf))
1084                     else:
1085                         os.remove(rmf)
1086                 else:
1087                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1088
1089     # update the metadata with the newly created ones included
1090     if newmetadata:
1091         apps = metadata.read_metadata()
1092
1093     # Scan the archive repo for apks as well
1094     if len(repodirs) > 1:
1095         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1096         if cc:
1097             cachechanged = True
1098     else:
1099         archapks = []
1100
1101     # Some information from the apks needs to be applied up to the application
1102     # level. When doing this, we use the info from the most recent version's apk.
1103     # We deal with figuring out when the app was added and last updated at the
1104     # same time.
1105     for appid, app in apps.iteritems():
1106         bestver = 0
1107         added = None
1108         lastupdated = None
1109         for apk in apks + archapks:
1110             if apk['id'] == appid:
1111                 if apk['versioncode'] > bestver:
1112                     bestver = apk['versioncode']
1113                     bestapk = apk
1114
1115                 if 'added' in apk:
1116                     if not added or apk['added'] < added:
1117                         added = apk['added']
1118                     if not lastupdated or apk['added'] > lastupdated:
1119                         lastupdated = apk['added']
1120
1121         if added:
1122             app['added'] = added
1123         else:
1124             logging.warn("Don't know when " + appid + " was added")
1125         if lastupdated:
1126             app['lastupdated'] = lastupdated
1127         else:
1128             logging.warn("Don't know when " + appid + " was last updated")
1129
1130         if bestver == 0:
1131             if app['Name'] is None:
1132                 app['Name'] = app['Auto Name'] or appid
1133             app['icon'] = None
1134             logging.warn("Application " + appid + " has no packages")
1135         else:
1136             if app['Name'] is None:
1137                 app['Name'] = bestapk['name']
1138             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1139
1140     # Sort the app list by name, then the web site doesn't have to by default.
1141     # (we had to wait until we'd scanned the apks to do this, because mostly the
1142     # name comes from there!)
1143     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid]['Name'].upper())
1144
1145     if len(repodirs) > 1:
1146         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1147
1148     # Make the index for the main repo...
1149     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1150
1151     # If there's an archive repo,  make the index for it. We already scanned it
1152     # earlier on.
1153     if len(repodirs) > 1:
1154         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1155
1156     if config['update_stats']:
1157
1158         # Update known apks info...
1159         knownapks.writeifchanged()
1160
1161         # Generate latest apps data for widget
1162         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1163             data = ''
1164             for line in file(os.path.join('stats', 'latestapps.txt')):
1165                 appid = line.rstrip()
1166                 data += appid + "\t"
1167                 app = apps[appid]
1168                 data += app['Name'] + "\t"
1169                 if app['icon'] is not None:
1170                     data += app['icon'] + "\t"
1171                 data += app['License'] + "\n"
1172             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1173             f.write(data)
1174             f.close()
1175
1176     if cachechanged:
1177         with open(apkcachefile, 'wb') as cf:
1178             pickle.dump(apkcache, cf)
1179
1180     # Update the wiki...
1181     if options.wiki:
1182         update_wiki(apps, sortedids, apks + archapks)
1183
1184     logging.info("Finished.")
1185
1186 if __name__ == "__main__":
1187     main()