fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from pyasn1.error import PyAsn1Error
  33 from pyasn1.codec.der import decoder, encoder
  34 from pyasn1_modules import rfc2315
  35 from hashlib import md5
  36
  37 from PIL import Image
  38 import logging
  39
  40 import common
  41 import metadata
  42 from common import FDroidPopen, SilentPopen
  43 from metadata import MetaDataException
  44
  45
  46 def get_densities():
  47     return ['640', '480', '320', '240', '160', '120']
  48
  49
  50 def dpi_to_px(density):
  51     return (int(density) * 48) / 160
  52
  53
  54 def px_to_dpi(px):
  55     return (int(px) * 160) / 48
  56
  57
  58 def get_icon_dir(repodir, density):
  59     if density is None:
  60         return os.path.join(repodir, "icons")
  61     return os.path.join(repodir, "icons-%s" % density)
  62
  63
  64 def get_icon_dirs(repodir):
  65     for density in get_densities():
  66         yield get_icon_dir(repodir, density)
  67     yield os.path.join(repodir, "icons")
  68
  69
  70 def update_wiki(apps, sortedids, apks):
  71     """Update the wiki
  72
  73     :param apps: fully populated list of all applications
  74     :param apks: all apks, except...
  75     """
  76     logging.info("Updating wiki")
  77     wikicat = 'Apps'
  78     wikiredircat = 'App Redirects'
  79     import mwclient
  80     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  81                          path=config['wiki_path'])
  82     site.login(config['wiki_user'], config['wiki_password'])
  83     generated_pages = {}
  84     generated_redirects = {}
  85
  86     for appid in sortedids:
  87         app = apps[appid]
  88
  89         wikidata = ''
  90         if app['Disabled']:
  91             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  92         if app['AntiFeatures']:
  93             for af in app['AntiFeatures'].split(','):
  94                 wikidata += '{{AntiFeature|' + af + '}}\n'
  95         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
  96             appid,
  97             app['Name'],
  98             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  99             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
 100             app['Source Code'],
 101             app['Issue Tracker'],
 102             app['Web Site'],
 103             app['Donate'],
 104             app['FlattrID'],
 105             app['Bitcoin'],
 106             app['Litecoin'],
 107             app['Dogecoin'],
 108             app['License'],
 109             app.get('Requires Root', 'No'))
 110
 111         if app['Provides']:
 112             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 113
 114         wikidata += app['Summary']
 115         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 116
 117         wikidata += "=Description=\n"
 118         wikidata += metadata.description_wiki(app['Description']) + "\n"
 119
 120         wikidata += "=Maintainer Notes=\n"
 121         if 'Maintainer Notes' in app:
 122             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 123         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 124
 125         # Get a list of all packages for this application...
 126         apklist = []
 127         gotcurrentver = False
 128         cantupdate = False
 129         buildfails = False
 130         for apk in apks:
 131             if apk['id'] == appid:
 132                 if str(apk['versioncode']) == app['Current Version Code']:
 133                     gotcurrentver = True
 134                 apklist.append(apk)
 135         # Include ones we can't build, as a special case...
 136         for thisbuild in app['builds']:
 137             if thisbuild['disable']:
 138                 if thisbuild['vercode'] == app['Current Version Code']:
 139                     cantupdate = True
 140                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 141                 apklist.append({'versioncode': int(thisbuild['vercode']),
 142                                 'version': thisbuild['version'],
 143                                 'buildproblem': thisbuild['disable']
 144                                 })
 145             else:
 146                 builtit = False
 147                 for apk in apklist:
 148                     if apk['versioncode'] == int(thisbuild['vercode']):
 149                         builtit = True
 150                         break
 151                 if not builtit:
 152                     buildfails = True
 153                     apklist.append({'versioncode': int(thisbuild['vercode']),
 154                                     'version': thisbuild['version'],
 155                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, thisbuild['vercode'])
 156                                     })
 157         if app['Current Version Code'] == '0':
 158             cantupdate = True
 159         # Sort with most recent first...
 160         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 161
 162         wikidata += "=Versions=\n"
 163         if len(apklist) == 0:
 164             wikidata += "We currently have no versions of this app available."
 165         elif not gotcurrentver:
 166             wikidata += "We don't have the current version of this app."
 167         else:
 168             wikidata += "We have the current version of this app."
 169         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 170         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 171         if len(app['No Source Since']) > 0:
 172             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 173         if len(app['Current Version']) > 0:
 174             wikidata += "The current (recommended) version is " + app['Current Version']
 175             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 176         validapks = 0
 177         for apk in apklist:
 178             wikidata += "==" + apk['version'] + "==\n"
 179
 180             if 'buildproblem' in apk:
 181                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 182             else:
 183                 validapks += 1
 184                 wikidata += "This version is built and signed by "
 185                 if 'srcname' in apk:
 186                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 187                 else:
 188                     wikidata += "the original developer.\n\n"
 189             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 190
 191         wikidata += '\n[[Category:' + wikicat + ']]\n'
 192         if len(app['No Source Since']) > 0:
 193             wikidata += '\n[[Category:Apps missing source code]]\n'
 194         if validapks == 0 and not app['Disabled']:
 195             wikidata += '\n[[Category:Apps with no packages]]\n'
 196         if cantupdate and not app['Disabled']:
 197             wikidata += "\n[[Category:Apps we can't update]]\n"
 198         if buildfails and not app['Disabled']:
 199             wikidata += "\n[[Category:Apps with failing builds]]\n"
 200         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 201             wikidata += '\n[[Category:Apps to Update]]\n'
 202         if app['Disabled']:
 203             wikidata += '\n[[Category:Apps that are disabled]]\n'
 204         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 205             wikidata += '\n[[Category:Apps with no update check]]\n'
 206         for appcat in app['Categories']:
 207             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 208
 209         # We can't have underscores in the page name, even if they're in
 210         # the package ID, because MediaWiki messes with them...
 211         pagename = appid.replace('_', ' ')
 212
 213         # Drop a trailing newline, because mediawiki is going to drop it anyway
 214         # and it we don't we'll think the page has changed when it hasn't...
 215         if wikidata.endswith('\n'):
 216             wikidata = wikidata[:-1]
 217
 218         generated_pages[pagename] = wikidata
 219
 220         # Make a redirect from the name to the ID too, unless there's
 221         # already an existing page with the name and it isn't a redirect.
 222         noclobber = False
 223         apppagename = app['Name'].replace('_', ' ')
 224         apppagename = apppagename.replace('{', '')
 225         apppagename = apppagename.replace('}', ' ')
 226         apppagename = apppagename.replace(':', ' ')
 227         # Drop double spaces caused mostly by replacing ':' above
 228         apppagename = apppagename.replace('  ', ' ')
 229         for expagename in site.allpages(prefix=apppagename,
 230                                         filterredir='nonredirects',
 231                                         generator=False):
 232             if expagename == apppagename:
 233                 noclobber = True
 234         # Another reason not to make the redirect page is if the app name
 235         # is the same as it's ID, because that will overwrite the real page
 236         # with an redirect to itself! (Although it seems like an odd
 237         # scenario this happens a lot, e.g. where there is metadata but no
 238         # builds or binaries to extract a name from.
 239         if apppagename == pagename:
 240             noclobber = True
 241         if not noclobber:
 242             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 243
 244     for tcat, genp in [(wikicat, generated_pages),
 245                        (wikiredircat, generated_redirects)]:
 246         catpages = site.Pages['Category:' + tcat]
 247         existingpages = []
 248         for page in catpages:
 249             existingpages.append(page.name)
 250             if page.name in genp:
 251                 pagetxt = page.edit()
 252                 if pagetxt != genp[page.name]:
 253                     logging.debug("Updating modified page " + page.name)
 254                     page.save(genp[page.name], summary='Auto-updated')
 255                 else:
 256                     logging.debug("Page " + page.name + " is unchanged")
 257             else:
 258                 logging.warn("Deleting page " + page.name)
 259                 page.delete('No longer published')
 260         for pagename, text in genp.items():
 261             logging.debug("Checking " + pagename)
 262             if pagename not in existingpages:
 263                 logging.debug("Creating page " + pagename)
 264                 try:
 265                     newpage = site.Pages[pagename]
 266                     newpage.save(text, summary='Auto-created')
 267                 except:
 268                     logging.error("...FAILED to create page")
 269
 270     # Purge server cache to ensure counts are up to date
 271     site.pages['Repository Maintenance'].purge()
 272
 273
 274 def delete_disabled_builds(apps, apkcache, repodirs):
 275     """Delete disabled build outputs.
 276
 277     :param apps: list of all applications, as per metadata.read_metadata
 278     :param apkcache: current apk cache information
 279     :param repodirs: the repo directories to process
 280     """
 281     for appid, app in apps.iteritems():
 282         for build in app['builds']:
 283             if build['disable']:
 284                 apkfilename = appid + '_' + str(build['vercode']) + '.apk'
 285                 for repodir in repodirs:
 286                     apkpath = os.path.join(repodir, apkfilename)
 287                     ascpath = apkpath + ".asc"
 288                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 289                     for name in [apkpath, srcpath, ascpath]:
 290                         if os.path.exists(name):
 291                             logging.warn("Deleting disabled build output " + apkfilename)
 292                             os.remove(name)
 293                 if apkfilename in apkcache:
 294                     del apkcache[apkfilename]
 295
 296
 297 def resize_icon(iconpath, density):
 298
 299     if not os.path.isfile(iconpath):
 300         return
 301
 302     try:
 303         im = Image.open(iconpath)
 304         size = dpi_to_px(density)
 305
 306         if any(length > size for length in im.size):
 307             oldsize = im.size
 308             im.thumbnail((size, size), Image.ANTIALIAS)
 309             logging.debug("%s was too large at %s - new size is %s" % (
 310                 iconpath, oldsize, im.size))
 311             im.save(iconpath, "PNG")
 312
 313     except Exception, e:
 314         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 315
 316
 317 def resize_all_icons(repodirs):
 318     """Resize all icons that exceed the max size
 319
 320     :param repodirs: the repo directories to process
 321     """
 322     for repodir in repodirs:
 323         for density in get_densities():
 324             icon_dir = get_icon_dir(repodir, density)
 325             icon_glob = os.path.join(icon_dir, '*.png')
 326             for iconpath in glob.glob(icon_glob):
 327                 resize_icon(iconpath, density)
 328
 329
 330 cert_path_regex = re.compile(r'^META-INF/.*\.RSA$')
 331
 332
 333 def getsig(apkpath):
 334     """ Get the signing certificate of an apk. To get the same md5 has that
 335     Android gets, we encode the .RSA certificate in a specific format and pass
 336     it hex-encoded to the md5 digest algorithm.
 337
 338     :param apkpath: path to the apk
 339     :returns: A string containing the md5 of the signature of the apk or None
 340               if an error occurred.
 341     """
 342
 343     cert = None
 344
 345     with zipfile.ZipFile(apkpath, 'r') as apk:
 346
 347         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 348
 349         if len(certs) < 1:
 350             logging.error("Found no signing certificates on %s" % apkpath)
 351             return None
 352         if len(certs) > 1:
 353             logging.error("Found multiple signing certificates on %s" % apkpath)
 354             return None
 355
 356         cert = apk.read(certs[0])
 357
 358     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 359     if content.getComponentByName('contentType') != rfc2315.signedData:
 360         logging.error("Unexpected format.")
 361         return None
 362
 363     content = decoder.decode(content.getComponentByName('content'),
 364                              asn1Spec=rfc2315.SignedData())[0]
 365     try:
 366         certificates = content.getComponentByName('certificates')
 367     except PyAsn1Error:
 368         logging.error("Certificates not found.")
 369         return None
 370
 371     cert_encoded = encoder.encode(certificates)[4:]
 372
 373     return md5(cert_encoded.encode('hex')).hexdigest()
 374
 375
 376 def scan_apks(apps, apkcache, repodir, knownapks):
 377     """Scan the apks in the given repo directory.
 378
 379     This also extracts the icons.
 380
 381     :param apps: list of all applications, as per metadata.read_metadata
 382     :param apkcache: current apk cache information
 383     :param repodir: repo directory to scan
 384     :param knownapks: known apks info
 385     :returns: (apks, cachechanged) where apks is a list of apk information,
 386               and cachechanged is True if the apkcache got changed.
 387     """
 388
 389     cachechanged = False
 390
 391     icon_dirs = get_icon_dirs(repodir)
 392     for icon_dir in icon_dirs:
 393         if os.path.exists(icon_dir):
 394             if options.clean:
 395                 shutil.rmtree(icon_dir)
 396                 os.makedirs(icon_dir)
 397         else:
 398             os.makedirs(icon_dir)
 399
 400     apks = []
 401     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 402     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 403     vername_pat = re.compile(".*versionName='([^']*)'.*")
 404     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 405     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 406     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 407     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 408     string_pat = re.compile(".*'([^']*)'.*")
 409     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 410
 411         apkfilename = apkfile[len(repodir) + 1:]
 412         if ' ' in apkfilename:
 413             logging.critical("Spaces in filenames are not allowed.")
 414             sys.exit(1)
 415
 416         if apkfilename in apkcache:
 417             logging.debug("Reading " + apkfilename + " from cache")
 418             thisinfo = apkcache[apkfilename]
 419
 420         else:
 421             logging.debug("Processing " + apkfilename)
 422             thisinfo = {}
 423             thisinfo['apkname'] = apkfilename
 424             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 425             if os.path.exists(os.path.join(repodir, srcfilename)):
 426                 thisinfo['srcname'] = srcfilename
 427             thisinfo['size'] = os.path.getsize(apkfile)
 428             thisinfo['permissions'] = set()
 429             thisinfo['features'] = set()
 430             thisinfo['icons_src'] = {}
 431             thisinfo['icons'] = {}
 432             p = SilentPopen([config['aapt'], 'dump', 'badging', apkfile])
 433             if p.returncode != 0:
 434                 if options.delete_unknown:
 435                     if os.path.exists(apkfile):
 436                         logging.error("Failed to get apk information, deleting " + apkfile)
 437                         os.remove(apkfile)
 438                     else:
 439                         logging.error("Could not find {0} to remove it".format(apkfile))
 440                 else:
 441                     logging.error("Failed to get apk information, skipping " + apkfile)
 442                 continue
 443             for line in p.output.splitlines():
 444                 if line.startswith("package:"):
 445                     try:
 446                         thisinfo['id'] = re.match(name_pat, line).group(1)
 447                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 448                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 449                     except Exception, e:
 450                         logging.error("Package matching failed: " + str(e))
 451                         logging.info("Line was: " + line)
 452                         sys.exit(1)
 453                 elif line.startswith("application:"):
 454                     thisinfo['name'] = re.match(label_pat, line).group(1)
 455                     # Keep path to non-dpi icon in case we need it
 456                     match = re.match(icon_pat_nodpi, line)
 457                     if match:
 458                         thisinfo['icons_src']['-1'] = match.group(1)
 459                 elif line.startswith("launchable-activity:"):
 460                     # Only use launchable-activity as fallback to application
 461                     if not thisinfo['name']:
 462                         thisinfo['name'] = re.match(label_pat, line).group(1)
 463                     if '-1' not in thisinfo['icons_src']:
 464                         match = re.match(icon_pat_nodpi, line)
 465                         if match:
 466                             thisinfo['icons_src']['-1'] = match.group(1)
 467                 elif line.startswith("application-icon-"):
 468                     match = re.match(icon_pat, line)
 469                     if match:
 470                         density = match.group(1)
 471                         path = match.group(2)
 472                         thisinfo['icons_src'][density] = path
 473                 elif line.startswith("sdkVersion:"):
 474                     m = re.match(sdkversion_pat, line)
 475                     if m is None:
 476                         logging.error(line.replace('sdkVersion:', '')
 477                                       + ' is not a valid minSdkVersion!')
 478                     else:
 479                         thisinfo['sdkversion'] = m.group(1)
 480                 elif line.startswith("maxSdkVersion:"):
 481                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 482                 elif line.startswith("native-code:"):
 483                     thisinfo['nativecode'] = []
 484                     for arch in line[13:].split(' '):
 485                         thisinfo['nativecode'].append(arch[1:-1])
 486                 elif line.startswith("uses-permission:"):
 487                     perm = re.match(string_pat, line).group(1)
 488                     if perm.startswith("android.permission."):
 489                         perm = perm[19:]
 490                     thisinfo['permissions'].add(perm)
 491                 elif line.startswith("uses-feature:"):
 492                     perm = re.match(string_pat, line).group(1)
 493                     # Filter out this, it's only added with the latest SDK tools and
 494                     # causes problems for lots of apps.
 495                     if perm != "android.hardware.screen.portrait" \
 496                             and perm != "android.hardware.screen.landscape":
 497                         if perm.startswith("android.feature."):
 498                             perm = perm[16:]
 499                         thisinfo['features'].add(perm)
 500
 501             if 'sdkversion' not in thisinfo:
 502                 logging.warn("No SDK version information found in {0}".format(apkfile))
 503                 thisinfo['sdkversion'] = 0
 504
 505             # Check for debuggable apks...
 506             if common.isApkDebuggable(apkfile, config):
 507                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 508
 509             # Calculate the sha256...
 510             sha = hashlib.sha256()
 511             with open(apkfile, 'rb') as f:
 512                 while True:
 513                     t = f.read(1024)
 514                     if len(t) == 0:
 515                         break
 516                     sha.update(t)
 517                 thisinfo['sha256'] = sha.hexdigest()
 518
 519             # verify the jar signature is correct
 520             args = ['jarsigner', '-verify']
 521             if options.verbose:
 522                 args += ['-verbose', '-certs']
 523             args += apkfile
 524             p = FDroidPopen(args)
 525             if p.returncode != 0:
 526                 logging.critical(apkfile + " has a bad signature!")
 527                 sys.exit(1)
 528
 529             # Get the signature (or md5 of, to be precise)...
 530             thisinfo['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 531             if not thisinfo['sig']:
 532                 logging.critical("Failed to get apk signature")
 533                 sys.exit(1)
 534
 535             apk = zipfile.ZipFile(apkfile, 'r')
 536
 537             iconfilename = "%s.%s.png" % (
 538                 thisinfo['id'],
 539                 thisinfo['versioncode'])
 540
 541             # Extract the icon file...
 542             densities = get_densities()
 543             empty_densities = []
 544             for density in densities:
 545                 if density not in thisinfo['icons_src']:
 546                     empty_densities.append(density)
 547                     continue
 548                 iconsrc = thisinfo['icons_src'][density]
 549                 icon_dir = get_icon_dir(repodir, density)
 550                 icondest = os.path.join(icon_dir, iconfilename)
 551
 552                 try:
 553                     iconfile = open(icondest, 'wb')
 554                     iconfile.write(apk.read(iconsrc))
 555                     iconfile.close()
 556                     thisinfo['icons'][density] = iconfilename
 557
 558                 except:
 559                     logging.warn("Error retrieving icon file")
 560                     del thisinfo['icons'][density]
 561                     del thisinfo['icons_src'][density]
 562                     empty_densities.append(density)
 563
 564             if '-1' in thisinfo['icons_src']:
 565                 iconsrc = thisinfo['icons_src']['-1']
 566                 iconpath = os.path.join(
 567                     get_icon_dir(repodir, None), iconfilename)
 568                 iconfile = open(iconpath, 'wb')
 569                 iconfile.write(apk.read(iconsrc))
 570                 iconfile.close()
 571                 try:
 572                     im = Image.open(iconpath)
 573                     dpi = px_to_dpi(im.size[0])
 574                     for density in densities:
 575                         if density in thisinfo['icons']:
 576                             break
 577                         if density == densities[-1] or dpi >= int(density):
 578                             thisinfo['icons'][density] = iconfilename
 579                             shutil.move(iconpath,
 580                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 581                             empty_densities.remove(density)
 582                             break
 583                 except Exception, e:
 584                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 585
 586             if thisinfo['icons']:
 587                 thisinfo['icon'] = iconfilename
 588
 589             apk.close()
 590
 591             # First try resizing down to not lose quality
 592             last_density = None
 593             for density in densities:
 594                 if density not in empty_densities:
 595                     last_density = density
 596                     continue
 597                 if last_density is None:
 598                     continue
 599                 logging.debug("Density %s not available, resizing down from %s"
 600                               % (density, last_density))
 601
 602                 last_iconpath = os.path.join(
 603                     get_icon_dir(repodir, last_density), iconfilename)
 604                 iconpath = os.path.join(
 605                     get_icon_dir(repodir, density), iconfilename)
 606                 try:
 607                     im = Image.open(last_iconpath)
 608                 except:
 609                     logging.warn("Invalid image file at %s" % last_iconpath)
 610                     continue
 611
 612                 size = dpi_to_px(density)
 613
 614                 im.thumbnail((size, size), Image.ANTIALIAS)
 615                 im.save(iconpath, "PNG")
 616                 empty_densities.remove(density)
 617
 618             # Then just copy from the highest resolution available
 619             last_density = None
 620             for density in reversed(densities):
 621                 if density not in empty_densities:
 622                     last_density = density
 623                     continue
 624                 if last_density is None:
 625                     continue
 626                 logging.debug("Density %s not available, copying from lower density %s"
 627                               % (density, last_density))
 628
 629                 shutil.copyfile(
 630                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 631                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 632
 633                 empty_densities.remove(density)
 634
 635             for density in densities:
 636                 icon_dir = get_icon_dir(repodir, density)
 637                 icondest = os.path.join(icon_dir, iconfilename)
 638                 resize_icon(icondest, density)
 639
 640             # Copy from icons-mdpi to icons since mdpi is the baseline density
 641             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 642             if os.path.isfile(baseline):
 643                 shutil.copyfile(baseline,
 644                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 645
 646             # Record in known apks, getting the added date at the same time..
 647             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 648             if added:
 649                 thisinfo['added'] = added
 650
 651             apkcache[apkfilename] = thisinfo
 652             cachechanged = True
 653
 654         apks.append(thisinfo)
 655
 656     return apks, cachechanged
 657
 658
 659 repo_pubkey_fingerprint = None
 660
 661
 662 def make_index(apps, sortedids, apks, repodir, archive, categories):
 663     """Make a repo index.
 664
 665     :param apps: fully populated apps list
 666     :param apks: full populated apks list
 667     :param repodir: the repo directory
 668     :param archive: True if this is the archive repo, False if it's the
 669                     main one.
 670     :param categories: list of categories
 671     """
 672
 673     doc = Document()
 674
 675     def addElement(name, value, doc, parent):
 676         el = doc.createElement(name)
 677         el.appendChild(doc.createTextNode(value))
 678         parent.appendChild(el)
 679
 680     def addElementCDATA(name, value, doc, parent):
 681         el = doc.createElement(name)
 682         el.appendChild(doc.createCDATASection(value))
 683         parent.appendChild(el)
 684
 685     root = doc.createElement("fdroid")
 686     doc.appendChild(root)
 687
 688     repoel = doc.createElement("repo")
 689
 690     if archive:
 691         repoel.setAttribute("name", config['archive_name'])
 692         if config['repo_maxage'] != 0:
 693             repoel.setAttribute("maxage", str(config['repo_maxage']))
 694         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 695         repoel.setAttribute("url", config['archive_url'])
 696         addElement('description', config['archive_description'], doc, repoel)
 697
 698     else:
 699         repoel.setAttribute("name", config['repo_name'])
 700         if config['repo_maxage'] != 0:
 701             repoel.setAttribute("maxage", str(config['repo_maxage']))
 702         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 703         repoel.setAttribute("url", config['repo_url'])
 704         addElement('description', config['repo_description'], doc, repoel)
 705
 706     repoel.setAttribute("version", "12")
 707     repoel.setAttribute("timestamp", str(int(time.time())))
 708
 709     if 'repo_keyalias' in config:
 710
 711         # Generate a certificate fingerprint the same way keytool does it
 712         # (but with slightly different formatting)
 713         def cert_fingerprint(data):
 714             digest = hashlib.sha256(data).digest()
 715             ret = []
 716             ret.append(' '.join("%02X" % ord(b) for b in digest))
 717             return " ".join(ret)
 718
 719         def extract_pubkey():
 720             p = FDroidPopen(['keytool', '-exportcert',
 721                              '-alias', config['repo_keyalias'],
 722                              '-keystore', config['keystore'],
 723                              '-storepass:file', config['keystorepassfile']]
 724                             + config['smartcardoptions'], output=False)
 725             if p.returncode != 0:
 726                 msg = "Failed to get repo pubkey!"
 727                 if config['keystore'] == 'NONE':
 728                     msg += ' Is your crypto smartcard plugged in?'
 729                 logging.critical(msg)
 730                 sys.exit(1)
 731             global repo_pubkey_fingerprint
 732             repo_pubkey_fingerprint = cert_fingerprint(p.output)
 733             return "".join("%02x" % ord(b) for b in p.output)
 734
 735         repoel.setAttribute("pubkey", extract_pubkey())
 736
 737     root.appendChild(repoel)
 738
 739     for appid in sortedids:
 740         app = apps[appid]
 741
 742         if app['Disabled'] is not None:
 743             continue
 744
 745         # Get a list of the apks for this app...
 746         apklist = []
 747         for apk in apks:
 748             if apk['id'] == appid:
 749                 apklist.append(apk)
 750
 751         if len(apklist) == 0:
 752             continue
 753
 754         apel = doc.createElement("application")
 755         apel.setAttribute("id", app['id'])
 756         root.appendChild(apel)
 757
 758         addElement('id', app['id'], doc, apel)
 759         if 'added' in app:
 760             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 761         if 'lastupdated' in app:
 762             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 763         addElement('name', app['Name'], doc, apel)
 764         addElement('summary', app['Summary'], doc, apel)
 765         if app['icon']:
 766             addElement('icon', app['icon'], doc, apel)
 767
 768         def linkres(appid):
 769             if appid in apps:
 770                 return ("fdroid.app:" + appid, apps[appid]['Name'])
 771             raise MetaDataException("Cannot resolve app id " + appid)
 772
 773         addElement('desc',
 774                    metadata.description_html(app['Description'], linkres),
 775                    doc, apel)
 776         addElement('license', app['License'], doc, apel)
 777         if 'Categories' in app:
 778             addElement('categories', ','.join(app["Categories"]), doc, apel)
 779             # We put the first (primary) category in LAST, which will have
 780             # the desired effect of making clients that only understand one
 781             # category see that one.
 782             addElement('category', app["Categories"][0], doc, apel)
 783         addElement('web', app['Web Site'], doc, apel)
 784         addElement('source', app['Source Code'], doc, apel)
 785         addElement('tracker', app['Issue Tracker'], doc, apel)
 786         if app['Donate']:
 787             addElement('donate', app['Donate'], doc, apel)
 788         if app['Bitcoin']:
 789             addElement('bitcoin', app['Bitcoin'], doc, apel)
 790         if app['Litecoin']:
 791             addElement('litecoin', app['Litecoin'], doc, apel)
 792         if app['Dogecoin']:
 793             addElement('dogecoin', app['Dogecoin'], doc, apel)
 794         if app['FlattrID']:
 795             addElement('flattr', app['FlattrID'], doc, apel)
 796
 797         # These elements actually refer to the current version (i.e. which
 798         # one is recommended. They are historically mis-named, and need
 799         # changing, but stay like this for now to support existing clients.
 800         addElement('marketversion', app['Current Version'], doc, apel)
 801         addElement('marketvercode', app['Current Version Code'], doc, apel)
 802
 803         if app['AntiFeatures']:
 804             af = app['AntiFeatures'].split(',')
 805             # TODO: Temporarily not including UpstreamNonFree in the index,
 806             # because current F-Droid clients do not understand it, and also
 807             # look ugly when they encounter an unknown antifeature. This
 808             # filtering can be removed in time...
 809             if 'UpstreamNonFree' in af:
 810                 af.remove('UpstreamNonFree')
 811             if af:
 812                 addElement('antifeatures', ','.join(af), doc, apel)
 813         if app['Provides']:
 814             pv = app['Provides'].split(',')
 815             addElement('provides', ','.join(pv), doc, apel)
 816         if app['Requires Root']:
 817             addElement('requirements', 'root', doc, apel)
 818
 819         # Sort the apk list into version order, just so the web site
 820         # doesn't have to do any work by default...
 821         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 822
 823         # Check for duplicates - they will make the client unhappy...
 824         for i in range(len(apklist) - 1):
 825             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 826                 logging.critical("duplicate versions: '%s' - '%s'" % (
 827                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 828                 sys.exit(1)
 829
 830         for apk in apklist:
 831             apkel = doc.createElement("package")
 832             apel.appendChild(apkel)
 833             addElement('version', apk['version'], doc, apkel)
 834             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 835             addElement('apkname', apk['apkname'], doc, apkel)
 836             if 'srcname' in apk:
 837                 addElement('srcname', apk['srcname'], doc, apkel)
 838             for hash_type in ['sha256']:
 839                 if hash_type not in apk:
 840                     continue
 841                 hashel = doc.createElement("hash")
 842                 hashel.setAttribute("type", hash_type)
 843                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 844                 apkel.appendChild(hashel)
 845             addElement('sig', apk['sig'], doc, apkel)
 846             addElement('size', str(apk['size']), doc, apkel)
 847             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 848             if 'maxsdkversion' in apk:
 849                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 850             if 'added' in apk:
 851                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 852             if app['Requires Root']:
 853                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 854                     apk['permissions'].add('ACCESS_SUPERUSER')
 855
 856             if len(apk['permissions']) > 0:
 857                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 858             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 859                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 860             if len(apk['features']) > 0:
 861                 addElement('features', ','.join(apk['features']), doc, apkel)
 862
 863     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 864     if options.pretty:
 865         output = doc.toprettyxml()
 866     else:
 867         output = doc.toxml()
 868     of.write(output)
 869     of.close()
 870
 871     if 'repo_keyalias' in config:
 872
 873         logging.info("Creating signed index with this key (SHA256):")
 874         logging.info("%s" % repo_pubkey_fingerprint)
 875
 876         # Create a jar of the index...
 877         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 878         if p.returncode != 0:
 879             logging.critical("Failed to create jar file")
 880             sys.exit(1)
 881
 882         # Sign the index...
 883         args = ['jarsigner', '-keystore', config['keystore'],
 884                 '-storepass:file', config['keystorepassfile'],
 885                 '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 886                 os.path.join(repodir, 'index.jar'), config['repo_keyalias']]
 887         if config['keystore'] == 'NONE':
 888             args += config['smartcardoptions']
 889         else:  # smardcards never use -keypass
 890             args += ['-keypass:file', config['keypassfile']]
 891         p = FDroidPopen(args)
 892         # TODO keypass should be sent via stdin
 893         if p.returncode != 0:
 894             logging.critical("Failed to sign index")
 895             sys.exit(1)
 896
 897     # Copy the repo icon into the repo directory...
 898     icon_dir = os.path.join(repodir, 'icons')
 899     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 900     shutil.copyfile(config['repo_icon'], iconfilename)
 901
 902     # Write a category list in the repo to allow quick access...
 903     catdata = ''
 904     for cat in categories:
 905         catdata += cat + '\n'
 906     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 907     f.write(catdata)
 908     f.close()
 909
 910
 911 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 912
 913     for appid, app in apps.iteritems():
 914
 915         # Get a list of the apks for this app...
 916         apklist = []
 917         for apk in apks:
 918             if apk['id'] == appid:
 919                 apklist.append(apk)
 920
 921         # Sort the apk list into version order...
 922         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 923
 924         if app['Archive Policy']:
 925             keepversions = int(app['Archive Policy'][:-9])
 926         else:
 927             keepversions = defaultkeepversions
 928
 929         if len(apklist) > keepversions:
 930             for apk in apklist[keepversions:]:
 931                 logging.info("Moving " + apk['apkname'] + " to archive")
 932                 shutil.move(os.path.join(repodir, apk['apkname']),
 933                             os.path.join(archivedir, apk['apkname']))
 934                 if 'srcname' in apk:
 935                     shutil.move(os.path.join(repodir, apk['srcname']),
 936                                 os.path.join(archivedir, apk['srcname']))
 937                     # Move GPG signature too...
 938                     sigfile = apk['srcname'] + '.asc'
 939                     sigsrc = os.path.join(repodir, sigfile)
 940                     if os.path.exists(sigsrc):
 941                         shutil.move(sigsrc, os.path.join(archivedir, sigfile))
 942
 943                 archapks.append(apk)
 944                 apks.remove(apk)
 945
 946
 947 config = None
 948 options = None
 949
 950
 951 def main():
 952
 953     global config, options
 954
 955     # Parse command line...
 956     parser = OptionParser()
 957     parser.add_option("-c", "--create-metadata", action="store_true", default=False,
 958                       help="Create skeleton metadata files that are missing")
 959     parser.add_option("--delete-unknown", action="store_true", default=False,
 960                       help="Delete APKs without metadata from the repo")
 961     parser.add_option("-v", "--verbose", action="store_true", default=False,
 962                       help="Spew out even more information than normal")
 963     parser.add_option("-q", "--quiet", action="store_true", default=False,
 964                       help="Restrict output to warnings and errors")
 965     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 966                       help="Report on build data status")
 967     parser.add_option("-i", "--interactive", default=False, action="store_true",
 968                       help="Interactively ask about things that need updating.")
 969     parser.add_option("-I", "--icons", action="store_true", default=False,
 970                       help="Resize all the icons exceeding the max pixel size and exit")
 971     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 972                       help="Specify editor to use in interactive mode. Default " +
 973                       "is /etc/alternatives/editor")
 974     parser.add_option("-w", "--wiki", default=False, action="store_true",
 975                       help="Update the wiki")
 976     parser.add_option("", "--pretty", action="store_true", default=False,
 977                       help="Produce human-readable index.xml")
 978     parser.add_option("--clean", action="store_true", default=False,
 979                       help="Clean update - don't uses caches, reprocess all apks")
 980     (options, args) = parser.parse_args()
 981
 982     config = common.read_config(options)
 983
 984     repodirs = ['repo']
 985     if config['archive_older'] != 0:
 986         repodirs.append('archive')
 987         if not os.path.exists('archive'):
 988             os.mkdir('archive')
 989
 990     if options.icons:
 991         resize_all_icons(repodirs)
 992         sys.exit(0)
 993
 994     # check that icons exist now, rather than fail at the end of `fdroid update`
 995     for k in ['repo_icon', 'archive_icon']:
 996         if k in config:
 997             if not os.path.exists(config[k]):
 998                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
 999                 sys.exit(1)
1000
1001     # Get all apps...
1002     apps = metadata.read_metadata()
1003
1004     # Generate a list of categories...
1005     categories = set()
1006     for app in apps.itervalues():
1007         categories.update(app['Categories'])
1008
1009     # Read known apks data (will be updated and written back when we've finished)
1010     knownapks = common.KnownApks()
1011
1012     # Gather information about all the apk files in the repo directory, using
1013     # cached data if possible.
1014     apkcachefile = os.path.join('tmp', 'apkcache')
1015     if not options.clean and os.path.exists(apkcachefile):
1016         with open(apkcachefile, 'rb') as cf:
1017             apkcache = pickle.load(cf)
1018     else:
1019         apkcache = {}
1020     cachechanged = False
1021
1022     delete_disabled_builds(apps, apkcache, repodirs)
1023
1024     # Scan all apks in the main repo
1025     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
1026     if cc:
1027         cachechanged = True
1028
1029     # Generate warnings for apk's with no metadata (or create skeleton
1030     # metadata files, if requested on the command line)
1031     newmetadata = False
1032     for apk in apks:
1033         if apk['id'] not in apps:
1034             if options.create_metadata:
1035                 if 'name' not in apk:
1036                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1037                     continue
1038                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1039                 f.write("License:Unknown\n")
1040                 f.write("Web Site:\n")
1041                 f.write("Source Code:\n")
1042                 f.write("Issue Tracker:\n")
1043                 f.write("Summary:" + apk['name'] + "\n")
1044                 f.write("Description:\n")
1045                 f.write(apk['name'] + "\n")
1046                 f.write(".\n")
1047                 f.close()
1048                 logging.info("Generated skeleton metadata for " + apk['id'])
1049                 newmetadata = True
1050             else:
1051                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1052                 if options.delete_unknown:
1053                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1054                     rmf = os.path.join(repodirs[0], apk['apkname'])
1055                     if not os.path.exists(rmf):
1056                         logging.error("Could not find {0} to remove it".format(rmf))
1057                     else:
1058                         os.remove(rmf)
1059                 else:
1060                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1061
1062     # update the metadata with the newly created ones included
1063     if newmetadata:
1064         apps = metadata.read_metadata()
1065
1066     # Scan the archive repo for apks as well
1067     if len(repodirs) > 1:
1068         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1069         if cc:
1070             cachechanged = True
1071     else:
1072         archapks = []
1073
1074     # Some information from the apks needs to be applied up to the application
1075     # level. When doing this, we use the info from the most recent version's apk.
1076     # We deal with figuring out when the app was added and last updated at the
1077     # same time.
1078     for appid, app in apps.iteritems():
1079         bestver = 0
1080         added = None
1081         lastupdated = None
1082         for apk in apks + archapks:
1083             if apk['id'] == appid:
1084                 if apk['versioncode'] > bestver:
1085                     bestver = apk['versioncode']
1086                     bestapk = apk
1087
1088                 if 'added' in apk:
1089                     if not added or apk['added'] < added:
1090                         added = apk['added']
1091                     if not lastupdated or apk['added'] > lastupdated:
1092                         lastupdated = apk['added']
1093
1094         if added:
1095             app['added'] = added
1096         else:
1097             logging.warn("Don't know when " + appid + " was added")
1098         if lastupdated:
1099             app['lastupdated'] = lastupdated
1100         else:
1101             logging.warn("Don't know when " + appid + " was last updated")
1102
1103         if bestver == 0:
1104             if app['Name'] is None:
1105                 app['Name'] = app['Auto Name'] or appid
1106             app['icon'] = None
1107             logging.warn("Application " + appid + " has no packages")
1108         else:
1109             if app['Name'] is None:
1110                 app['Name'] = bestapk['name']
1111             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1112
1113     # Sort the app list by name, then the web site doesn't have to by default.
1114     # (we had to wait until we'd scanned the apks to do this, because mostly the
1115     # name comes from there!)
1116     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid]['Name'].upper())
1117
1118     if len(repodirs) > 1:
1119         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1120
1121     # Make the index for the main repo...
1122     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1123
1124     # If there's an archive repo,  make the index for it. We already scanned it
1125     # earlier on.
1126     if len(repodirs) > 1:
1127         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1128
1129     if config['update_stats']:
1130
1131         # Update known apks info...
1132         knownapks.writeifchanged()
1133
1134         # Generate latest apps data for widget
1135         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1136             data = ''
1137             for line in file(os.path.join('stats', 'latestapps.txt')):
1138                 appid = line.rstrip()
1139                 data += appid + "\t"
1140                 app = apps[appid]
1141                 data += app['Name'] + "\t"
1142                 if app['icon'] is not None:
1143                     data += app['icon'] + "\t"
1144                 data += app['License'] + "\n"
1145             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1146             f.write(data)
1147             f.close()
1148
1149     if cachechanged:
1150         with open(apkcachefile, 'wb') as cf:
1151             pickle.dump(apkcache, cf)
1152
1153     # Update the wiki...
1154     if options.wiki:
1155         update_wiki(apps, sortedids, apks + archapks)
1156
1157     logging.info("Finished.")
1158
1159 if __name__ == "__main__":
1160     main()