fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from pyasn1.error import PyAsn1Error
  33 from pyasn1.codec.der import decoder, encoder
  34 from pyasn1_modules import rfc2315
  35 from hashlib import md5
  36
  37 from PIL import Image
  38 import logging
  39
  40 import common
  41 import metadata
  42 from common import FDroidPopen, SilentPopen
  43 from metadata import MetaDataException
  44
  45
  46 def get_densities():
  47     return ['640', '480', '320', '240', '160', '120']
  48
  49
  50 def dpi_to_px(density):
  51     return (int(density) * 48) / 160
  52
  53
  54 def px_to_dpi(px):
  55     return (int(px) * 160) / 48
  56
  57
  58 def get_icon_dir(repodir, density):
  59     if density is None:
  60         return os.path.join(repodir, "icons")
  61     return os.path.join(repodir, "icons-%s" % density)
  62
  63
  64 def get_icon_dirs(repodir):
  65     for density in get_densities():
  66         yield get_icon_dir(repodir, density)
  67     yield os.path.join(repodir, "icons")
  68
  69
  70 def update_wiki(apps, sortedids, apks):
  71     """Update the wiki
  72
  73     :param apps: fully populated list of all applications
  74     :param apks: all apks, except...
  75     """
  76     logging.info("Updating wiki")
  77     wikicat = 'Apps'
  78     wikiredircat = 'App Redirects'
  79     import mwclient
  80     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  81                          path=config['wiki_path'])
  82     site.login(config['wiki_user'], config['wiki_password'])
  83     generated_pages = {}
  84     generated_redirects = {}
  85
  86     for appid in sortedids:
  87         app = apps[appid]
  88
  89         wikidata = ''
  90         if app['Disabled']:
  91             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  92         if app['AntiFeatures']:
  93             for af in app['AntiFeatures'].split(','):
  94                 wikidata += '{{AntiFeature|' + af + '}}\n'
  95         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
  96             appid,
  97             app['Name'],
  98             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  99             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
 100             app['Source Code'],
 101             app['Issue Tracker'],
 102             app['Web Site'],
 103             app['Donate'],
 104             app['FlattrID'],
 105             app['Bitcoin'],
 106             app['Litecoin'],
 107             app['Dogecoin'],
 108             app['License'],
 109             app.get('Requires Root', 'No'))
 110
 111         if app['Provides']:
 112             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 113
 114         wikidata += app['Summary']
 115         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 116
 117         wikidata += "=Description=\n"
 118         wikidata += metadata.description_wiki(app['Description']) + "\n"
 119
 120         wikidata += "=Maintainer Notes=\n"
 121         if 'Maintainer Notes' in app:
 122             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 123         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 124
 125         # Get a list of all packages for this application...
 126         apklist = []
 127         gotcurrentver = False
 128         cantupdate = False
 129         buildfails = False
 130         for apk in apks:
 131             if apk['id'] == appid:
 132                 if str(apk['versioncode']) == app['Current Version Code']:
 133                     gotcurrentver = True
 134                 apklist.append(apk)
 135         # Include ones we can't build, as a special case...
 136         for thisbuild in app['builds']:
 137             if thisbuild['disable']:
 138                 if thisbuild['vercode'] == app['Current Version Code']:
 139                     cantupdate = True
 140                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 141                 apklist.append({'versioncode': int(thisbuild['vercode']),
 142                                 'version': thisbuild['version'],
 143                                 'buildproblem': thisbuild['disable']
 144                                 })
 145             else:
 146                 builtit = False
 147                 for apk in apklist:
 148                     if apk['versioncode'] == int(thisbuild['vercode']):
 149                         builtit = True
 150                         break
 151                 if not builtit:
 152                     buildfails = True
 153                     apklist.append({'versioncode': int(thisbuild['vercode']),
 154                                     'version': thisbuild['version'],
 155                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, thisbuild['vercode'])
 156                                     })
 157         if app['Current Version Code'] == '0':
 158             cantupdate = True
 159         # Sort with most recent first...
 160         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 161
 162         wikidata += "=Versions=\n"
 163         if len(apklist) == 0:
 164             wikidata += "We currently have no versions of this app available."
 165         elif not gotcurrentver:
 166             wikidata += "We don't have the current version of this app."
 167         else:
 168             wikidata += "We have the current version of this app."
 169         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 170         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 171         if len(app['No Source Since']) > 0:
 172             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 173         if len(app['Current Version']) > 0:
 174             wikidata += "The current (recommended) version is " + app['Current Version']
 175             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 176         validapks = 0
 177         for apk in apklist:
 178             wikidata += "==" + apk['version'] + "==\n"
 179
 180             if 'buildproblem' in apk:
 181                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 182             else:
 183                 validapks += 1
 184                 wikidata += "This version is built and signed by "
 185                 if 'srcname' in apk:
 186                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 187                 else:
 188                     wikidata += "the original developer.\n\n"
 189             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 190
 191         wikidata += '\n[[Category:' + wikicat + ']]\n'
 192         if len(app['No Source Since']) > 0:
 193             wikidata += '\n[[Category:Apps missing source code]]\n'
 194         if validapks == 0 and not app['Disabled']:
 195             wikidata += '\n[[Category:Apps with no packages]]\n'
 196         if cantupdate and not app['Disabled']:
 197             wikidata += "\n[[Category:Apps we can't update]]\n"
 198         if buildfails and not app['Disabled']:
 199             wikidata += "\n[[Category:Apps with failing builds]]\n"
 200         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 201             wikidata += '\n[[Category:Apps to Update]]\n'
 202         if app['Disabled']:
 203             wikidata += '\n[[Category:Apps that are disabled]]\n'
 204         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 205             wikidata += '\n[[Category:Apps with no update check]]\n'
 206         for appcat in app['Categories']:
 207             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 208
 209         # We can't have underscores in the page name, even if they're in
 210         # the package ID, because MediaWiki messes with them...
 211         pagename = appid.replace('_', ' ')
 212
 213         # Drop a trailing newline, because mediawiki is going to drop it anyway
 214         # and it we don't we'll think the page has changed when it hasn't...
 215         if wikidata.endswith('\n'):
 216             wikidata = wikidata[:-1]
 217
 218         generated_pages[pagename] = wikidata
 219
 220         # Make a redirect from the name to the ID too, unless there's
 221         # already an existing page with the name and it isn't a redirect.
 222         noclobber = False
 223         apppagename = app['Name'].replace('_', ' ')
 224         apppagename = apppagename.replace('{', '')
 225         apppagename = apppagename.replace('}', ' ')
 226         apppagename = apppagename.replace(':', ' ')
 227         # Drop double spaces caused mostly by replacing ':' above
 228         apppagename = apppagename.replace('  ', ' ')
 229         for expagename in site.allpages(prefix=apppagename,
 230                                         filterredir='nonredirects',
 231                                         generator=False):
 232             if expagename == apppagename:
 233                 noclobber = True
 234         # Another reason not to make the redirect page is if the app name
 235         # is the same as it's ID, because that will overwrite the real page
 236         # with an redirect to itself! (Although it seems like an odd
 237         # scenario this happens a lot, e.g. where there is metadata but no
 238         # builds or binaries to extract a name from.
 239         if apppagename == pagename:
 240             noclobber = True
 241         if not noclobber:
 242             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 243
 244     for tcat, genp in [(wikicat, generated_pages),
 245                        (wikiredircat, generated_redirects)]:
 246         catpages = site.Pages['Category:' + tcat]
 247         existingpages = []
 248         for page in catpages:
 249             existingpages.append(page.name)
 250             if page.name in genp:
 251                 pagetxt = page.edit()
 252                 if pagetxt != genp[page.name]:
 253                     logging.debug("Updating modified page " + page.name)
 254                     page.save(genp[page.name], summary='Auto-updated')
 255                 else:
 256                     logging.debug("Page " + page.name + " is unchanged")
 257             else:
 258                 logging.warn("Deleting page " + page.name)
 259                 page.delete('No longer published')
 260         for pagename, text in genp.items():
 261             logging.debug("Checking " + pagename)
 262             if pagename not in existingpages:
 263                 logging.debug("Creating page " + pagename)
 264                 try:
 265                     newpage = site.Pages[pagename]
 266                     newpage.save(text, summary='Auto-created')
 267                 except:
 268                     logging.error("...FAILED to create page")
 269
 270     # Purge server cache to ensure counts are up to date
 271     site.pages['Repository Maintenance'].purge()
 272
 273
 274 def delete_disabled_builds(apps, apkcache, repodirs):
 275     """Delete disabled build outputs.
 276
 277     :param apps: list of all applications, as per metadata.read_metadata
 278     :param apkcache: current apk cache information
 279     :param repodirs: the repo directories to process
 280     """
 281     for appid, app in apps.iteritems():
 282         for build in app['builds']:
 283             if build['disable']:
 284                 apkfilename = appid + '_' + str(build['vercode']) + '.apk'
 285                 for repodir in repodirs:
 286                     apkpath = os.path.join(repodir, apkfilename)
 287                     ascpath = apkpath + ".asc"
 288                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 289                     for name in [apkpath, srcpath, ascpath]:
 290                         if os.path.exists(name):
 291                             logging.warn("Deleting disabled build output " + apkfilename)
 292                             os.remove(name)
 293                 if apkfilename in apkcache:
 294                     del apkcache[apkfilename]
 295
 296
 297 def resize_icon(iconpath, density):
 298
 299     if not os.path.isfile(iconpath):
 300         return
 301
 302     try:
 303         im = Image.open(iconpath)
 304         size = dpi_to_px(density)
 305
 306         if any(length > size for length in im.size):
 307             oldsize = im.size
 308             im.thumbnail((size, size), Image.ANTIALIAS)
 309             logging.debug("%s was too large at %s - new size is %s" % (
 310                 iconpath, oldsize, im.size))
 311             im.save(iconpath, "PNG")
 312
 313     except Exception, e:
 314         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 315
 316
 317 def resize_all_icons(repodirs):
 318     """Resize all icons that exceed the max size
 319
 320     :param repodirs: the repo directories to process
 321     """
 322     for repodir in repodirs:
 323         for density in get_densities():
 324             icon_dir = get_icon_dir(repodir, density)
 325             icon_glob = os.path.join(icon_dir, '*.png')
 326             for iconpath in glob.glob(icon_glob):
 327                 resize_icon(iconpath, density)
 328
 329
 330 cert_path_regex = re.compile(r'^META-INF/.*\.RSA$')
 331
 332
 333 def getsig(apkpath):
 334     """ Get the signing certificate of an apk. To get the same md5 has that
 335     Android gets, we encode the .RSA certificate in a specific format and pass
 336     it hex-encoded to the md5 digest algorithm.
 337
 338     :param apkpath: path to the apk
 339     :returns: A string containing the md5 of the signature of the apk or None
 340               if an error occurred.
 341     """
 342
 343     cert = None
 344
 345     # verify the jar signature is correct
 346     args = ['jarsigner', '-verify', apkpath]
 347     p = FDroidPopen(args)
 348     if p.returncode != 0:
 349         logging.critical(apkpath + " has a bad signature!")
 350         return None
 351
 352     with zipfile.ZipFile(apkpath, 'r') as apk:
 353
 354         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 355
 356         if len(certs) < 1:
 357             logging.error("Found no signing certificates on %s" % apkpath)
 358             return None
 359         if len(certs) > 1:
 360             logging.error("Found multiple signing certificates on %s" % apkpath)
 361             return None
 362
 363         cert = apk.read(certs[0])
 364
 365     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 366     if content.getComponentByName('contentType') != rfc2315.signedData:
 367         logging.error("Unexpected format.")
 368         return None
 369
 370     content = decoder.decode(content.getComponentByName('content'),
 371                              asn1Spec=rfc2315.SignedData())[0]
 372     try:
 373         certificates = content.getComponentByName('certificates')
 374     except PyAsn1Error:
 375         logging.error("Certificates not found.")
 376         return None
 377
 378     cert_encoded = encoder.encode(certificates)[4:]
 379
 380     return md5(cert_encoded.encode('hex')).hexdigest()
 381
 382
 383 def scan_apks(apps, apkcache, repodir, knownapks):
 384     """Scan the apks in the given repo directory.
 385
 386     This also extracts the icons.
 387
 388     :param apps: list of all applications, as per metadata.read_metadata
 389     :param apkcache: current apk cache information
 390     :param repodir: repo directory to scan
 391     :param knownapks: known apks info
 392     :returns: (apks, cachechanged) where apks is a list of apk information,
 393               and cachechanged is True if the apkcache got changed.
 394     """
 395
 396     cachechanged = False
 397
 398     icon_dirs = get_icon_dirs(repodir)
 399     for icon_dir in icon_dirs:
 400         if os.path.exists(icon_dir):
 401             if options.clean:
 402                 shutil.rmtree(icon_dir)
 403                 os.makedirs(icon_dir)
 404         else:
 405             os.makedirs(icon_dir)
 406
 407     apks = []
 408     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 409     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 410     vername_pat = re.compile(".*versionName='([^']*)'.*")
 411     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 412     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 413     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 414     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 415     string_pat = re.compile(".*'([^']*)'.*")
 416     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 417
 418         apkfilename = apkfile[len(repodir) + 1:]
 419         if ' ' in apkfilename:
 420             logging.critical("Spaces in filenames are not allowed.")
 421             sys.exit(1)
 422
 423         if apkfilename in apkcache:
 424             logging.debug("Reading " + apkfilename + " from cache")
 425             thisinfo = apkcache[apkfilename]
 426
 427         else:
 428             logging.debug("Processing " + apkfilename)
 429             thisinfo = {}
 430             thisinfo['apkname'] = apkfilename
 431             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 432             if os.path.exists(os.path.join(repodir, srcfilename)):
 433                 thisinfo['srcname'] = srcfilename
 434             thisinfo['size'] = os.path.getsize(apkfile)
 435             thisinfo['permissions'] = set()
 436             thisinfo['features'] = set()
 437             thisinfo['icons_src'] = {}
 438             thisinfo['icons'] = {}
 439             p = SilentPopen([config['aapt'], 'dump', 'badging', apkfile])
 440             if p.returncode != 0:
 441                 if options.delete_unknown:
 442                     if os.path.exists(apkfile):
 443                         logging.error("Failed to get apk information, deleting " + apkfile)
 444                         os.remove(apkfile)
 445                     else:
 446                         logging.error("Could not find {0} to remove it".format(apkfile))
 447                 else:
 448                     logging.error("Failed to get apk information, skipping " + apkfile)
 449                 continue
 450             for line in p.output.splitlines():
 451                 if line.startswith("package:"):
 452                     try:
 453                         thisinfo['id'] = re.match(name_pat, line).group(1)
 454                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 455                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 456                     except Exception, e:
 457                         logging.error("Package matching failed: " + str(e))
 458                         logging.info("Line was: " + line)
 459                         sys.exit(1)
 460                 elif line.startswith("application:"):
 461                     thisinfo['name'] = re.match(label_pat, line).group(1)
 462                     # Keep path to non-dpi icon in case we need it
 463                     match = re.match(icon_pat_nodpi, line)
 464                     if match:
 465                         thisinfo['icons_src']['-1'] = match.group(1)
 466                 elif line.startswith("launchable-activity:"):
 467                     # Only use launchable-activity as fallback to application
 468                     if not thisinfo['name']:
 469                         thisinfo['name'] = re.match(label_pat, line).group(1)
 470                     if '-1' not in thisinfo['icons_src']:
 471                         match = re.match(icon_pat_nodpi, line)
 472                         if match:
 473                             thisinfo['icons_src']['-1'] = match.group(1)
 474                 elif line.startswith("application-icon-"):
 475                     match = re.match(icon_pat, line)
 476                     if match:
 477                         density = match.group(1)
 478                         path = match.group(2)
 479                         thisinfo['icons_src'][density] = path
 480                 elif line.startswith("sdkVersion:"):
 481                     m = re.match(sdkversion_pat, line)
 482                     if m is None:
 483                         logging.error(line.replace('sdkVersion:', '')
 484                                       + ' is not a valid minSdkVersion!')
 485                     else:
 486                         thisinfo['sdkversion'] = m.group(1)
 487                 elif line.startswith("maxSdkVersion:"):
 488                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 489                 elif line.startswith("native-code:"):
 490                     thisinfo['nativecode'] = []
 491                     for arch in line[13:].split(' '):
 492                         thisinfo['nativecode'].append(arch[1:-1])
 493                 elif line.startswith("uses-permission:"):
 494                     perm = re.match(string_pat, line).group(1)
 495                     if perm.startswith("android.permission."):
 496                         perm = perm[19:]
 497                     thisinfo['permissions'].add(perm)
 498                 elif line.startswith("uses-feature:"):
 499                     perm = re.match(string_pat, line).group(1)
 500                     # Filter out this, it's only added with the latest SDK tools and
 501                     # causes problems for lots of apps.
 502                     if perm != "android.hardware.screen.portrait" \
 503                             and perm != "android.hardware.screen.landscape":
 504                         if perm.startswith("android.feature."):
 505                             perm = perm[16:]
 506                         thisinfo['features'].add(perm)
 507
 508             if 'sdkversion' not in thisinfo:
 509                 logging.warn("No SDK version information found in {0}".format(apkfile))
 510                 thisinfo['sdkversion'] = 0
 511
 512             # Check for debuggable apks...
 513             if common.isApkDebuggable(apkfile, config):
 514                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 515
 516             # Calculate the sha256...
 517             sha = hashlib.sha256()
 518             with open(apkfile, 'rb') as f:
 519                 while True:
 520                     t = f.read(1024)
 521                     if len(t) == 0:
 522                         break
 523                     sha.update(t)
 524                 thisinfo['sha256'] = sha.hexdigest()
 525
 526             # Get the signature (or md5 of, to be precise)...
 527             thisinfo['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 528             if not thisinfo['sig']:
 529                 logging.critical("Failed to get apk signature")
 530                 sys.exit(1)
 531
 532             apk = zipfile.ZipFile(apkfile, 'r')
 533
 534             iconfilename = "%s.%s.png" % (
 535                 thisinfo['id'],
 536                 thisinfo['versioncode'])
 537
 538             # Extract the icon file...
 539             densities = get_densities()
 540             empty_densities = []
 541             for density in densities:
 542                 if density not in thisinfo['icons_src']:
 543                     empty_densities.append(density)
 544                     continue
 545                 iconsrc = thisinfo['icons_src'][density]
 546                 icon_dir = get_icon_dir(repodir, density)
 547                 icondest = os.path.join(icon_dir, iconfilename)
 548
 549                 try:
 550                     iconfile = open(icondest, 'wb')
 551                     iconfile.write(apk.read(iconsrc))
 552                     iconfile.close()
 553                     thisinfo['icons'][density] = iconfilename
 554
 555                 except:
 556                     logging.warn("Error retrieving icon file")
 557                     del thisinfo['icons'][density]
 558                     del thisinfo['icons_src'][density]
 559                     empty_densities.append(density)
 560
 561             if '-1' in thisinfo['icons_src']:
 562                 iconsrc = thisinfo['icons_src']['-1']
 563                 iconpath = os.path.join(
 564                     get_icon_dir(repodir, None), iconfilename)
 565                 iconfile = open(iconpath, 'wb')
 566                 iconfile.write(apk.read(iconsrc))
 567                 iconfile.close()
 568                 try:
 569                     im = Image.open(iconpath)
 570                     dpi = px_to_dpi(im.size[0])
 571                     for density in densities:
 572                         if density in thisinfo['icons']:
 573                             break
 574                         if density == densities[-1] or dpi >= int(density):
 575                             thisinfo['icons'][density] = iconfilename
 576                             shutil.move(iconpath,
 577                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 578                             empty_densities.remove(density)
 579                             break
 580                 except Exception, e:
 581                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 582
 583             if thisinfo['icons']:
 584                 thisinfo['icon'] = iconfilename
 585
 586             apk.close()
 587
 588             # First try resizing down to not lose quality
 589             last_density = None
 590             for density in densities:
 591                 if density not in empty_densities:
 592                     last_density = density
 593                     continue
 594                 if last_density is None:
 595                     continue
 596                 logging.debug("Density %s not available, resizing down from %s"
 597                               % (density, last_density))
 598
 599                 last_iconpath = os.path.join(
 600                     get_icon_dir(repodir, last_density), iconfilename)
 601                 iconpath = os.path.join(
 602                     get_icon_dir(repodir, density), iconfilename)
 603                 try:
 604                     im = Image.open(last_iconpath)
 605                 except:
 606                     logging.warn("Invalid image file at %s" % last_iconpath)
 607                     continue
 608
 609                 size = dpi_to_px(density)
 610
 611                 im.thumbnail((size, size), Image.ANTIALIAS)
 612                 im.save(iconpath, "PNG")
 613                 empty_densities.remove(density)
 614
 615             # Then just copy from the highest resolution available
 616             last_density = None
 617             for density in reversed(densities):
 618                 if density not in empty_densities:
 619                     last_density = density
 620                     continue
 621                 if last_density is None:
 622                     continue
 623                 logging.debug("Density %s not available, copying from lower density %s"
 624                               % (density, last_density))
 625
 626                 shutil.copyfile(
 627                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 628                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 629
 630                 empty_densities.remove(density)
 631
 632             for density in densities:
 633                 icon_dir = get_icon_dir(repodir, density)
 634                 icondest = os.path.join(icon_dir, iconfilename)
 635                 resize_icon(icondest, density)
 636
 637             # Copy from icons-mdpi to icons since mdpi is the baseline density
 638             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 639             if os.path.isfile(baseline):
 640                 shutil.copyfile(baseline,
 641                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 642
 643             # Record in known apks, getting the added date at the same time..
 644             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 645             if added:
 646                 thisinfo['added'] = added
 647
 648             apkcache[apkfilename] = thisinfo
 649             cachechanged = True
 650
 651         apks.append(thisinfo)
 652
 653     return apks, cachechanged
 654
 655
 656 repo_pubkey_fingerprint = None
 657
 658
 659 def make_index(apps, sortedids, apks, repodir, archive, categories):
 660     """Make a repo index.
 661
 662     :param apps: fully populated apps list
 663     :param apks: full populated apks list
 664     :param repodir: the repo directory
 665     :param archive: True if this is the archive repo, False if it's the
 666                     main one.
 667     :param categories: list of categories
 668     """
 669
 670     doc = Document()
 671
 672     def addElement(name, value, doc, parent):
 673         el = doc.createElement(name)
 674         el.appendChild(doc.createTextNode(value))
 675         parent.appendChild(el)
 676
 677     def addElementCDATA(name, value, doc, parent):
 678         el = doc.createElement(name)
 679         el.appendChild(doc.createCDATASection(value))
 680         parent.appendChild(el)
 681
 682     root = doc.createElement("fdroid")
 683     doc.appendChild(root)
 684
 685     repoel = doc.createElement("repo")
 686
 687     if archive:
 688         repoel.setAttribute("name", config['archive_name'])
 689         if config['repo_maxage'] != 0:
 690             repoel.setAttribute("maxage", str(config['repo_maxage']))
 691         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 692         repoel.setAttribute("url", config['archive_url'])
 693         addElement('description', config['archive_description'], doc, repoel)
 694
 695     else:
 696         repoel.setAttribute("name", config['repo_name'])
 697         if config['repo_maxage'] != 0:
 698             repoel.setAttribute("maxage", str(config['repo_maxage']))
 699         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 700         repoel.setAttribute("url", config['repo_url'])
 701         addElement('description', config['repo_description'], doc, repoel)
 702
 703     repoel.setAttribute("version", "12")
 704     repoel.setAttribute("timestamp", str(int(time.time())))
 705
 706     if 'repo_keyalias' in config:
 707
 708         # Generate a certificate fingerprint the same way keytool does it
 709         # (but with slightly different formatting)
 710         def cert_fingerprint(data):
 711             digest = hashlib.sha256(data).digest()
 712             ret = []
 713             ret.append(' '.join("%02X" % ord(b) for b in digest))
 714             return " ".join(ret)
 715
 716         def extract_pubkey():
 717             p = FDroidPopen(['keytool', '-exportcert',
 718                              '-alias', config['repo_keyalias'],
 719                              '-keystore', config['keystore'],
 720                              '-storepass:file', config['keystorepassfile']]
 721                             + config['smartcardoptions'], output=False)
 722             if p.returncode != 0:
 723                 msg = "Failed to get repo pubkey!"
 724                 if config['keystore'] == 'NONE':
 725                     msg += ' Is your crypto smartcard plugged in?'
 726                 logging.critical(msg)
 727                 sys.exit(1)
 728             global repo_pubkey_fingerprint
 729             repo_pubkey_fingerprint = cert_fingerprint(p.output)
 730             return "".join("%02x" % ord(b) for b in p.output)
 731
 732         repoel.setAttribute("pubkey", extract_pubkey())
 733
 734     root.appendChild(repoel)
 735
 736     for appid in sortedids:
 737         app = apps[appid]
 738
 739         if app['Disabled'] is not None:
 740             continue
 741
 742         # Get a list of the apks for this app...
 743         apklist = []
 744         for apk in apks:
 745             if apk['id'] == appid:
 746                 apklist.append(apk)
 747
 748         if len(apklist) == 0:
 749             continue
 750
 751         apel = doc.createElement("application")
 752         apel.setAttribute("id", app['id'])
 753         root.appendChild(apel)
 754
 755         addElement('id', app['id'], doc, apel)
 756         if 'added' in app:
 757             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 758         if 'lastupdated' in app:
 759             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 760         addElement('name', app['Name'], doc, apel)
 761         addElement('summary', app['Summary'], doc, apel)
 762         if app['icon']:
 763             addElement('icon', app['icon'], doc, apel)
 764
 765         def linkres(appid):
 766             if appid in apps:
 767                 return ("fdroid.app:" + appid, apps[appid]['Name'])
 768             raise MetaDataException("Cannot resolve app id " + appid)
 769
 770         addElement('desc',
 771                    metadata.description_html(app['Description'], linkres),
 772                    doc, apel)
 773         addElement('license', app['License'], doc, apel)
 774         if 'Categories' in app:
 775             addElement('categories', ','.join(app["Categories"]), doc, apel)
 776             # We put the first (primary) category in LAST, which will have
 777             # the desired effect of making clients that only understand one
 778             # category see that one.
 779             addElement('category', app["Categories"][0], doc, apel)
 780         addElement('web', app['Web Site'], doc, apel)
 781         addElement('source', app['Source Code'], doc, apel)
 782         addElement('tracker', app['Issue Tracker'], doc, apel)
 783         if app['Donate']:
 784             addElement('donate', app['Donate'], doc, apel)
 785         if app['Bitcoin']:
 786             addElement('bitcoin', app['Bitcoin'], doc, apel)
 787         if app['Litecoin']:
 788             addElement('litecoin', app['Litecoin'], doc, apel)
 789         if app['Dogecoin']:
 790             addElement('dogecoin', app['Dogecoin'], doc, apel)
 791         if app['FlattrID']:
 792             addElement('flattr', app['FlattrID'], doc, apel)
 793
 794         # These elements actually refer to the current version (i.e. which
 795         # one is recommended. They are historically mis-named, and need
 796         # changing, but stay like this for now to support existing clients.
 797         addElement('marketversion', app['Current Version'], doc, apel)
 798         addElement('marketvercode', app['Current Version Code'], doc, apel)
 799
 800         if app['AntiFeatures']:
 801             af = app['AntiFeatures'].split(',')
 802             # TODO: Temporarily not including UpstreamNonFree in the index,
 803             # because current F-Droid clients do not understand it, and also
 804             # look ugly when they encounter an unknown antifeature. This
 805             # filtering can be removed in time...
 806             if 'UpstreamNonFree' in af:
 807                 af.remove('UpstreamNonFree')
 808             if af:
 809                 addElement('antifeatures', ','.join(af), doc, apel)
 810         if app['Provides']:
 811             pv = app['Provides'].split(',')
 812             addElement('provides', ','.join(pv), doc, apel)
 813         if app['Requires Root']:
 814             addElement('requirements', 'root', doc, apel)
 815
 816         # Sort the apk list into version order, just so the web site
 817         # doesn't have to do any work by default...
 818         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 819
 820         # Check for duplicates - they will make the client unhappy...
 821         for i in range(len(apklist) - 1):
 822             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 823                 logging.critical("duplicate versions: '%s' - '%s'" % (
 824                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 825                 sys.exit(1)
 826
 827         current_version_code = 0
 828         current_version_file = None
 829         for apk in apklist:
 830             # find the APK for the "Current Version"
 831             if current_version_code < apk['versioncode']:
 832                 current_version_code = apk['versioncode']
 833             if current_version_code < int(app['Current Version Code']):
 834                 current_version_file = apk['apkname']
 835
 836             apkel = doc.createElement("package")
 837             apel.appendChild(apkel)
 838             addElement('version', apk['version'], doc, apkel)
 839             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 840             addElement('apkname', apk['apkname'], doc, apkel)
 841             if 'srcname' in apk:
 842                 addElement('srcname', apk['srcname'], doc, apkel)
 843             for hash_type in ['sha256']:
 844                 if hash_type not in apk:
 845                     continue
 846                 hashel = doc.createElement("hash")
 847                 hashel.setAttribute("type", hash_type)
 848                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 849                 apkel.appendChild(hashel)
 850             addElement('sig', apk['sig'], doc, apkel)
 851             addElement('size', str(apk['size']), doc, apkel)
 852             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 853             if 'maxsdkversion' in apk:
 854                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 855             if 'added' in apk:
 856                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 857             if app['Requires Root']:
 858                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 859                     apk['permissions'].add('ACCESS_SUPERUSER')
 860
 861             if len(apk['permissions']) > 0:
 862                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 863             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 864                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 865             if len(apk['features']) > 0:
 866                 addElement('features', ','.join(apk['features']), doc, apkel)
 867
 868         if current_version_file is not None \
 869                 and config['make_current_version_link'] \
 870                 and repodir == 'repo':  # only create these
 871             sanitized_name = re.sub('''[ '"&%?+=]''', '',
 872                                     app[config['current_version_name_source']])
 873             apklinkname = sanitized_name + '.apk'
 874             current_version_path = os.path.join(repodir, current_version_file)
 875             if os.path.exists(apklinkname):
 876                 os.remove(apklinkname)
 877             os.symlink(current_version_path, apklinkname)
 878             # also symlink gpg signature, if it exists
 879             for extension in ('.asc', '.sig'):
 880                 sigfile_path = current_version_path + extension
 881                 if os.path.exists(sigfile_path):
 882                     siglinkname = apklinkname + extension
 883                     if os.path.exists(siglinkname):
 884                         os.remove(siglinkname)
 885                     os.symlink(sigfile_path, siglinkname)
 886
 887     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 888     if options.pretty:
 889         output = doc.toprettyxml()
 890     else:
 891         output = doc.toxml()
 892     of.write(output)
 893     of.close()
 894
 895     if 'repo_keyalias' in config:
 896
 897         logging.info("Creating signed index with this key (SHA256):")
 898         logging.info("%s" % repo_pubkey_fingerprint)
 899
 900         # Create a jar of the index...
 901         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 902         if p.returncode != 0:
 903             logging.critical("Failed to create jar file")
 904             sys.exit(1)
 905
 906         # Sign the index...
 907         args = ['jarsigner', '-keystore', config['keystore'],
 908                 '-storepass:file', config['keystorepassfile'],
 909                 '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 910                 os.path.join(repodir, 'index.jar'), config['repo_keyalias']]
 911         if config['keystore'] == 'NONE':
 912             args += config['smartcardoptions']
 913         else:  # smardcards never use -keypass
 914             args += ['-keypass:file', config['keypassfile']]
 915         p = FDroidPopen(args)
 916         # TODO keypass should be sent via stdin
 917         if p.returncode != 0:
 918             logging.critical("Failed to sign index")
 919             sys.exit(1)
 920
 921     # Copy the repo icon into the repo directory...
 922     icon_dir = os.path.join(repodir, 'icons')
 923     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 924     shutil.copyfile(config['repo_icon'], iconfilename)
 925
 926     # Write a category list in the repo to allow quick access...
 927     catdata = ''
 928     for cat in categories:
 929         catdata += cat + '\n'
 930     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 931     f.write(catdata)
 932     f.close()
 933
 934
 935 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 936
 937     for appid, app in apps.iteritems():
 938
 939         # Get a list of the apks for this app...
 940         apklist = []
 941         for apk in apks:
 942             if apk['id'] == appid:
 943                 apklist.append(apk)
 944
 945         # Sort the apk list into version order...
 946         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 947
 948         if app['Archive Policy']:
 949             keepversions = int(app['Archive Policy'][:-9])
 950         else:
 951             keepversions = defaultkeepversions
 952
 953         if len(apklist) > keepversions:
 954             for apk in apklist[keepversions:]:
 955                 logging.info("Moving " + apk['apkname'] + " to archive")
 956                 shutil.move(os.path.join(repodir, apk['apkname']),
 957                             os.path.join(archivedir, apk['apkname']))
 958                 if 'srcname' in apk:
 959                     shutil.move(os.path.join(repodir, apk['srcname']),
 960                                 os.path.join(archivedir, apk['srcname']))
 961                     # Move GPG signature too...
 962                     sigfile = apk['srcname'] + '.asc'
 963                     sigsrc = os.path.join(repodir, sigfile)
 964                     if os.path.exists(sigsrc):
 965                         shutil.move(sigsrc, os.path.join(archivedir, sigfile))
 966
 967                 archapks.append(apk)
 968                 apks.remove(apk)
 969
 970
 971 config = None
 972 options = None
 973
 974
 975 def main():
 976
 977     global config, options
 978
 979     # Parse command line...
 980     parser = OptionParser()
 981     parser.add_option("-c", "--create-metadata", action="store_true", default=False,
 982                       help="Create skeleton metadata files that are missing")
 983     parser.add_option("--delete-unknown", action="store_true", default=False,
 984                       help="Delete APKs without metadata from the repo")
 985     parser.add_option("-v", "--verbose", action="store_true", default=False,
 986                       help="Spew out even more information than normal")
 987     parser.add_option("-q", "--quiet", action="store_true", default=False,
 988                       help="Restrict output to warnings and errors")
 989     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 990                       help="Report on build data status")
 991     parser.add_option("-i", "--interactive", default=False, action="store_true",
 992                       help="Interactively ask about things that need updating.")
 993     parser.add_option("-I", "--icons", action="store_true", default=False,
 994                       help="Resize all the icons exceeding the max pixel size and exit")
 995     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 996                       help="Specify editor to use in interactive mode. Default " +
 997                       "is /etc/alternatives/editor")
 998     parser.add_option("-w", "--wiki", default=False, action="store_true",
 999                       help="Update the wiki")
1000     parser.add_option("", "--pretty", action="store_true", default=False,
1001                       help="Produce human-readable index.xml")
1002     parser.add_option("--clean", action="store_true", default=False,
1003                       help="Clean update - don't uses caches, reprocess all apks")
1004     (options, args) = parser.parse_args()
1005
1006     config = common.read_config(options)
1007
1008     repodirs = ['repo']
1009     if config['archive_older'] != 0:
1010         repodirs.append('archive')
1011         if not os.path.exists('archive'):
1012             os.mkdir('archive')
1013
1014     if options.icons:
1015         resize_all_icons(repodirs)
1016         sys.exit(0)
1017
1018     # check that icons exist now, rather than fail at the end of `fdroid update`
1019     for k in ['repo_icon', 'archive_icon']:
1020         if k in config:
1021             if not os.path.exists(config[k]):
1022                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1023                 sys.exit(1)
1024
1025     # Get all apps...
1026     apps = metadata.read_metadata()
1027
1028     # Generate a list of categories...
1029     categories = set()
1030     for app in apps.itervalues():
1031         categories.update(app['Categories'])
1032
1033     # Read known apks data (will be updated and written back when we've finished)
1034     knownapks = common.KnownApks()
1035
1036     # Gather information about all the apk files in the repo directory, using
1037     # cached data if possible.
1038     apkcachefile = os.path.join('tmp', 'apkcache')
1039     if not options.clean and os.path.exists(apkcachefile):
1040         with open(apkcachefile, 'rb') as cf:
1041             apkcache = pickle.load(cf)
1042     else:
1043         apkcache = {}
1044     cachechanged = False
1045
1046     delete_disabled_builds(apps, apkcache, repodirs)
1047
1048     # Scan all apks in the main repo
1049     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
1050     if cc:
1051         cachechanged = True
1052
1053     # Generate warnings for apk's with no metadata (or create skeleton
1054     # metadata files, if requested on the command line)
1055     newmetadata = False
1056     for apk in apks:
1057         if apk['id'] not in apps:
1058             if options.create_metadata:
1059                 if 'name' not in apk:
1060                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1061                     continue
1062                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1063                 f.write("License:Unknown\n")
1064                 f.write("Web Site:\n")
1065                 f.write("Source Code:\n")
1066                 f.write("Issue Tracker:\n")
1067                 f.write("Summary:" + apk['name'] + "\n")
1068                 f.write("Description:\n")
1069                 f.write(apk['name'] + "\n")
1070                 f.write(".\n")
1071                 f.close()
1072                 logging.info("Generated skeleton metadata for " + apk['id'])
1073                 newmetadata = True
1074             else:
1075                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1076                 if options.delete_unknown:
1077                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1078                     rmf = os.path.join(repodirs[0], apk['apkname'])
1079                     if not os.path.exists(rmf):
1080                         logging.error("Could not find {0} to remove it".format(rmf))
1081                     else:
1082                         os.remove(rmf)
1083                 else:
1084                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1085
1086     # update the metadata with the newly created ones included
1087     if newmetadata:
1088         apps = metadata.read_metadata()
1089
1090     # Scan the archive repo for apks as well
1091     if len(repodirs) > 1:
1092         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1093         if cc:
1094             cachechanged = True
1095     else:
1096         archapks = []
1097
1098     # Some information from the apks needs to be applied up to the application
1099     # level. When doing this, we use the info from the most recent version's apk.
1100     # We deal with figuring out when the app was added and last updated at the
1101     # same time.
1102     for appid, app in apps.iteritems():
1103         bestver = 0
1104         added = None
1105         lastupdated = None
1106         for apk in apks + archapks:
1107             if apk['id'] == appid:
1108                 if apk['versioncode'] > bestver:
1109                     bestver = apk['versioncode']
1110                     bestapk = apk
1111
1112                 if 'added' in apk:
1113                     if not added or apk['added'] < added:
1114                         added = apk['added']
1115                     if not lastupdated or apk['added'] > lastupdated:
1116                         lastupdated = apk['added']
1117
1118         if added:
1119             app['added'] = added
1120         else:
1121             logging.warn("Don't know when " + appid + " was added")
1122         if lastupdated:
1123             app['lastupdated'] = lastupdated
1124         else:
1125             logging.warn("Don't know when " + appid + " was last updated")
1126
1127         if bestver == 0:
1128             if app['Name'] is None:
1129                 app['Name'] = app['Auto Name'] or appid
1130             app['icon'] = None
1131             logging.warn("Application " + appid + " has no packages")
1132         else:
1133             if app['Name'] is None:
1134                 app['Name'] = bestapk['name']
1135             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1136
1137     # Sort the app list by name, then the web site doesn't have to by default.
1138     # (we had to wait until we'd scanned the apks to do this, because mostly the
1139     # name comes from there!)
1140     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid]['Name'].upper())
1141
1142     if len(repodirs) > 1:
1143         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1144
1145     # Make the index for the main repo...
1146     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1147
1148     # If there's an archive repo,  make the index for it. We already scanned it
1149     # earlier on.
1150     if len(repodirs) > 1:
1151         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1152
1153     if config['update_stats']:
1154
1155         # Update known apks info...
1156         knownapks.writeifchanged()
1157
1158         # Generate latest apps data for widget
1159         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1160             data = ''
1161             for line in file(os.path.join('stats', 'latestapps.txt')):
1162                 appid = line.rstrip()
1163                 data += appid + "\t"
1164                 app = apps[appid]
1165                 data += app['Name'] + "\t"
1166                 if app['icon'] is not None:
1167                     data += app['icon'] + "\t"
1168                 data += app['License'] + "\n"
1169             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1170             f.write(data)
1171             f.close()
1172
1173     if cachechanged:
1174         with open(apkcachefile, 'wb') as cf:
1175             pickle.dump(apkcache, cf)
1176
1177     # Update the wiki...
1178     if options.wiki:
1179         update_wiki(apps, sortedids, apks + archapks)
1180
1181     logging.info("Finished.")
1182
1183 if __name__ == "__main__":
1184     main()