fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from pyasn1.error import PyAsn1Error
  33 from pyasn1.codec.der import decoder, encoder
  34 from pyasn1_modules import rfc2315
  35 from hashlib import md5
  36
  37 from PIL import Image
  38 import logging
  39
  40 import common
  41 import metadata
  42 from common import FDroidPopen, SilentPopen
  43 from metadata import MetaDataException
  44
  45
  46 def get_densities():
  47     return ['640', '480', '320', '240', '160', '120']
  48
  49
  50 def dpi_to_px(density):
  51     return (int(density) * 48) / 160
  52
  53
  54 def px_to_dpi(px):
  55     return (int(px) * 160) / 48
  56
  57
  58 def get_icon_dir(repodir, density):
  59     if density is None:
  60         return os.path.join(repodir, "icons")
  61     return os.path.join(repodir, "icons-%s" % density)
  62
  63
  64 def get_icon_dirs(repodir):
  65     for density in get_densities():
  66         yield get_icon_dir(repodir, density)
  67     yield os.path.join(repodir, "icons")
  68
  69
  70 def update_wiki(apps, sortedids, apks):
  71     """Update the wiki
  72
  73     :param apps: fully populated list of all applications
  74     :param apks: all apks, except...
  75     """
  76     logging.info("Updating wiki")
  77     wikicat = 'Apps'
  78     wikiredircat = 'App Redirects'
  79     import mwclient
  80     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  81                          path=config['wiki_path'])
  82     site.login(config['wiki_user'], config['wiki_password'])
  83     generated_pages = {}
  84     generated_redirects = {}
  85
  86     for appid in sortedids:
  87         app = apps[appid]
  88
  89         wikidata = ''
  90         if app['Disabled']:
  91             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  92         if app['AntiFeatures']:
  93             for af in app['AntiFeatures'].split(','):
  94                 wikidata += '{{AntiFeature|' + af + '}}\n'
  95         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
  96             appid,
  97             app['Name'],
  98             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  99             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
 100             app['Source Code'],
 101             app['Issue Tracker'],
 102             app['Web Site'],
 103             app['Donate'],
 104             app['FlattrID'],
 105             app['Bitcoin'],
 106             app['Litecoin'],
 107             app['Dogecoin'],
 108             app['License'],
 109             app.get('Requires Root', 'No'))
 110
 111         if app['Provides']:
 112             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 113
 114         wikidata += app['Summary']
 115         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 116
 117         wikidata += "=Description=\n"
 118         wikidata += metadata.description_wiki(app['Description']) + "\n"
 119
 120         wikidata += "=Maintainer Notes=\n"
 121         if 'Maintainer Notes' in app:
 122             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 123         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 124
 125         # Get a list of all packages for this application...
 126         apklist = []
 127         gotcurrentver = False
 128         cantupdate = False
 129         buildfails = False
 130         for apk in apks:
 131             if apk['id'] == appid:
 132                 if str(apk['versioncode']) == app['Current Version Code']:
 133                     gotcurrentver = True
 134                 apklist.append(apk)
 135         # Include ones we can't build, as a special case...
 136         for thisbuild in app['builds']:
 137             if thisbuild['disable']:
 138                 if thisbuild['vercode'] == app['Current Version Code']:
 139                     cantupdate = True
 140                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 141                 apklist.append({'versioncode': int(thisbuild['vercode']),
 142                                 'version': thisbuild['version'],
 143                                 'buildproblem': thisbuild['disable']
 144                                 })
 145             else:
 146                 builtit = False
 147                 for apk in apklist:
 148                     if apk['versioncode'] == int(thisbuild['vercode']):
 149                         builtit = True
 150                         break
 151                 if not builtit:
 152                     buildfails = True
 153                     apklist.append({'versioncode': int(thisbuild['vercode']),
 154                                     'version': thisbuild['version'],
 155                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, thisbuild['vercode'])
 156                                     })
 157         if app['Current Version Code'] == '0':
 158             cantupdate = True
 159         # Sort with most recent first...
 160         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 161
 162         wikidata += "=Versions=\n"
 163         if len(apklist) == 0:
 164             wikidata += "We currently have no versions of this app available."
 165         elif not gotcurrentver:
 166             wikidata += "We don't have the current version of this app."
 167         else:
 168             wikidata += "We have the current version of this app."
 169         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 170         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 171         if len(app['No Source Since']) > 0:
 172             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 173         if len(app['Current Version']) > 0:
 174             wikidata += "The current (recommended) version is " + app['Current Version']
 175             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 176         validapks = 0
 177         for apk in apklist:
 178             wikidata += "==" + apk['version'] + "==\n"
 179
 180             if 'buildproblem' in apk:
 181                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 182             else:
 183                 validapks += 1
 184                 wikidata += "This version is built and signed by "
 185                 if 'srcname' in apk:
 186                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 187                 else:
 188                     wikidata += "the original developer.\n\n"
 189             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 190
 191         wikidata += '\n[[Category:' + wikicat + ']]\n'
 192         if len(app['No Source Since']) > 0:
 193             wikidata += '\n[[Category:Apps missing source code]]\n'
 194         if validapks == 0 and not app['Disabled']:
 195             wikidata += '\n[[Category:Apps with no packages]]\n'
 196         if cantupdate and not app['Disabled']:
 197             wikidata += "\n[[Category:Apps we can't update]]\n"
 198         if buildfails and not app['Disabled']:
 199             wikidata += "\n[[Category:Apps with failing builds]]\n"
 200         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 201             wikidata += '\n[[Category:Apps to Update]]\n'
 202         if app['Disabled']:
 203             wikidata += '\n[[Category:Apps that are disabled]]\n'
 204         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 205             wikidata += '\n[[Category:Apps with no update check]]\n'
 206         for appcat in app['Categories']:
 207             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 208
 209         # We can't have underscores in the page name, even if they're in
 210         # the package ID, because MediaWiki messes with them...
 211         pagename = appid.replace('_', ' ')
 212
 213         # Drop a trailing newline, because mediawiki is going to drop it anyway
 214         # and it we don't we'll think the page has changed when it hasn't...
 215         if wikidata.endswith('\n'):
 216             wikidata = wikidata[:-1]
 217
 218         generated_pages[pagename] = wikidata
 219
 220         # Make a redirect from the name to the ID too, unless there's
 221         # already an existing page with the name and it isn't a redirect.
 222         noclobber = False
 223         apppagename = app['Name'].replace('_', ' ')
 224         apppagename = apppagename.replace('{', '')
 225         apppagename = apppagename.replace('}', ' ')
 226         apppagename = apppagename.replace(':', ' ')
 227         # Drop double spaces caused mostly by replacing ':' above
 228         apppagename = apppagename.replace('  ', ' ')
 229         for expagename in site.allpages(prefix=apppagename,
 230                                         filterredir='nonredirects',
 231                                         generator=False):
 232             if expagename == apppagename:
 233                 noclobber = True
 234         # Another reason not to make the redirect page is if the app name
 235         # is the same as it's ID, because that will overwrite the real page
 236         # with an redirect to itself! (Although it seems like an odd
 237         # scenario this happens a lot, e.g. where there is metadata but no
 238         # builds or binaries to extract a name from.
 239         if apppagename == pagename:
 240             noclobber = True
 241         if not noclobber:
 242             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 243
 244     for tcat, genp in [(wikicat, generated_pages),
 245                        (wikiredircat, generated_redirects)]:
 246         catpages = site.Pages['Category:' + tcat]
 247         existingpages = []
 248         for page in catpages:
 249             existingpages.append(page.name)
 250             if page.name in genp:
 251                 pagetxt = page.edit()
 252                 if pagetxt != genp[page.name]:
 253                     logging.debug("Updating modified page " + page.name)
 254                     page.save(genp[page.name], summary='Auto-updated')
 255                 else:
 256                     logging.debug("Page " + page.name + " is unchanged")
 257             else:
 258                 logging.warn("Deleting page " + page.name)
 259                 page.delete('No longer published')
 260         for pagename, text in genp.items():
 261             logging.debug("Checking " + pagename)
 262             if pagename not in existingpages:
 263                 logging.debug("Creating page " + pagename)
 264                 try:
 265                     newpage = site.Pages[pagename]
 266                     newpage.save(text, summary='Auto-created')
 267                 except:
 268                     logging.error("...FAILED to create page")
 269
 270     # Purge server cache to ensure counts are up to date
 271     site.pages['Repository Maintenance'].purge()
 272
 273
 274 def delete_disabled_builds(apps, apkcache, repodirs):
 275     """Delete disabled build outputs.
 276
 277     :param apps: list of all applications, as per metadata.read_metadata
 278     :param apkcache: current apk cache information
 279     :param repodirs: the repo directories to process
 280     """
 281     for appid, app in apps.iteritems():
 282         for build in app['builds']:
 283             if build['disable']:
 284                 apkfilename = appid + '_' + str(build['vercode']) + '.apk'
 285                 for repodir in repodirs:
 286                     apkpath = os.path.join(repodir, apkfilename)
 287                     ascpath = apkpath + ".asc"
 288                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 289                     for name in [apkpath, srcpath, ascpath]:
 290                         if os.path.exists(name):
 291                             logging.warn("Deleting disabled build output " + apkfilename)
 292                             os.remove(name)
 293                 if apkfilename in apkcache:
 294                     del apkcache[apkfilename]
 295
 296
 297 def resize_icon(iconpath, density):
 298
 299     if not os.path.isfile(iconpath):
 300         return
 301
 302     try:
 303         im = Image.open(iconpath)
 304         size = dpi_to_px(density)
 305
 306         if any(length > size for length in im.size):
 307             oldsize = im.size
 308             im.thumbnail((size, size), Image.ANTIALIAS)
 309             logging.debug("%s was too large at %s - new size is %s" % (
 310                 iconpath, oldsize, im.size))
 311             im.save(iconpath, "PNG")
 312
 313     except Exception, e:
 314         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 315
 316
 317 def resize_all_icons(repodirs):
 318     """Resize all icons that exceed the max size
 319
 320     :param repodirs: the repo directories to process
 321     """
 322     for repodir in repodirs:
 323         for density in get_densities():
 324             icon_dir = get_icon_dir(repodir, density)
 325             icon_glob = os.path.join(icon_dir, '*.png')
 326             for iconpath in glob.glob(icon_glob):
 327                 resize_icon(iconpath, density)
 328
 329
 330 cert_path_regex = re.compile(r'^META-INF/.*\.RSA$')
 331
 332
 333 def getsig(apkpath):
 334     """ Get the signing certificate of an apk. To get the same md5 has that
 335     Android gets, we encode the .RSA certificate in a specific format and pass
 336     it hex-encoded to the md5 digest algorithm.
 337
 338     :param apkpath: path to the apk
 339     :returns: A string containing the md5 of the signature of the apk or None
 340               if an error occurred.
 341     """
 342
 343     cert = None
 344
 345     # verify the jar signature is correct
 346     args = ['jarsigner', '-verify', apkpath]
 347     p = FDroidPopen(args)
 348     if p.returncode != 0:
 349         logging.critical(apkpath + " has a bad signature!")
 350         return None
 351
 352     with zipfile.ZipFile(apkpath, 'r') as apk:
 353
 354         certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
 355
 356         if len(certs) < 1:
 357             logging.error("Found no signing certificates on %s" % apkpath)
 358             return None
 359         if len(certs) > 1:
 360             logging.error("Found multiple signing certificates on %s" % apkpath)
 361             return None
 362
 363         cert = apk.read(certs[0])
 364
 365     content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
 366     if content.getComponentByName('contentType') != rfc2315.signedData:
 367         logging.error("Unexpected format.")
 368         return None
 369
 370     content = decoder.decode(content.getComponentByName('content'),
 371                              asn1Spec=rfc2315.SignedData())[0]
 372     try:
 373         certificates = content.getComponentByName('certificates')
 374     except PyAsn1Error:
 375         logging.error("Certificates not found.")
 376         return None
 377
 378     cert_encoded = encoder.encode(certificates)[4:]
 379
 380     return md5(cert_encoded.encode('hex')).hexdigest()
 381
 382
 383 def scan_apks(apps, apkcache, repodir, knownapks):
 384     """Scan the apks in the given repo directory.
 385
 386     This also extracts the icons.
 387
 388     :param apps: list of all applications, as per metadata.read_metadata
 389     :param apkcache: current apk cache information
 390     :param repodir: repo directory to scan
 391     :param knownapks: known apks info
 392     :returns: (apks, cachechanged) where apks is a list of apk information,
 393               and cachechanged is True if the apkcache got changed.
 394     """
 395
 396     cachechanged = False
 397
 398     icon_dirs = get_icon_dirs(repodir)
 399     for icon_dir in icon_dirs:
 400         if os.path.exists(icon_dir):
 401             if options.clean:
 402                 shutil.rmtree(icon_dir)
 403                 os.makedirs(icon_dir)
 404         else:
 405             os.makedirs(icon_dir)
 406
 407     apks = []
 408     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 409     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 410     vername_pat = re.compile(".*versionName='([^']*)'.*")
 411     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 412     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 413     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 414     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 415     string_pat = re.compile(".*'([^']*)'.*")
 416     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 417
 418         apkfilename = apkfile[len(repodir) + 1:]
 419         if ' ' in apkfilename:
 420             logging.critical("Spaces in filenames are not allowed.")
 421             sys.exit(1)
 422
 423         if apkfilename in apkcache:
 424             logging.debug("Reading " + apkfilename + " from cache")
 425             thisinfo = apkcache[apkfilename]
 426
 427         else:
 428             logging.debug("Processing " + apkfilename)
 429             thisinfo = {}
 430             thisinfo['apkname'] = apkfilename
 431             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 432             if os.path.exists(os.path.join(repodir, srcfilename)):
 433                 thisinfo['srcname'] = srcfilename
 434             thisinfo['size'] = os.path.getsize(apkfile)
 435             thisinfo['permissions'] = set()
 436             thisinfo['features'] = set()
 437             thisinfo['icons_src'] = {}
 438             thisinfo['icons'] = {}
 439             p = SilentPopen([config['aapt'], 'dump', 'badging', apkfile])
 440             if p.returncode != 0:
 441                 if options.delete_unknown:
 442                     if os.path.exists(apkfile):
 443                         logging.error("Failed to get apk information, deleting " + apkfile)
 444                         os.remove(apkfile)
 445                     else:
 446                         logging.error("Could not find {0} to remove it".format(apkfile))
 447                 else:
 448                     logging.error("Failed to get apk information, skipping " + apkfile)
 449                 continue
 450             for line in p.output.splitlines():
 451                 if line.startswith("package:"):
 452                     try:
 453                         thisinfo['id'] = re.match(name_pat, line).group(1)
 454                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 455                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 456                     except Exception, e:
 457                         logging.error("Package matching failed: " + str(e))
 458                         logging.info("Line was: " + line)
 459                         sys.exit(1)
 460                 elif line.startswith("application:"):
 461                     thisinfo['name'] = re.match(label_pat, line).group(1)
 462                     # Keep path to non-dpi icon in case we need it
 463                     match = re.match(icon_pat_nodpi, line)
 464                     if match:
 465                         thisinfo['icons_src']['-1'] = match.group(1)
 466                 elif line.startswith("launchable-activity:"):
 467                     # Only use launchable-activity as fallback to application
 468                     if not thisinfo['name']:
 469                         thisinfo['name'] = re.match(label_pat, line).group(1)
 470                     if '-1' not in thisinfo['icons_src']:
 471                         match = re.match(icon_pat_nodpi, line)
 472                         if match:
 473                             thisinfo['icons_src']['-1'] = match.group(1)
 474                 elif line.startswith("application-icon-"):
 475                     match = re.match(icon_pat, line)
 476                     if match:
 477                         density = match.group(1)
 478                         path = match.group(2)
 479                         thisinfo['icons_src'][density] = path
 480                 elif line.startswith("sdkVersion:"):
 481                     m = re.match(sdkversion_pat, line)
 482                     if m is None:
 483                         logging.error(line.replace('sdkVersion:', '')
 484                                       + ' is not a valid minSdkVersion!')
 485                     else:
 486                         thisinfo['sdkversion'] = m.group(1)
 487                 elif line.startswith("maxSdkVersion:"):
 488                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 489                 elif line.startswith("native-code:"):
 490                     thisinfo['nativecode'] = []
 491                     for arch in line[13:].split(' '):
 492                         thisinfo['nativecode'].append(arch[1:-1])
 493                 elif line.startswith("uses-permission:"):
 494                     perm = re.match(string_pat, line).group(1)
 495                     if perm.startswith("android.permission."):
 496                         perm = perm[19:]
 497                     thisinfo['permissions'].add(perm)
 498                 elif line.startswith("uses-feature:"):
 499                     perm = re.match(string_pat, line).group(1)
 500                     # Filter out this, it's only added with the latest SDK tools and
 501                     # causes problems for lots of apps.
 502                     if perm != "android.hardware.screen.portrait" \
 503                             and perm != "android.hardware.screen.landscape":
 504                         if perm.startswith("android.feature."):
 505                             perm = perm[16:]
 506                         thisinfo['features'].add(perm)
 507
 508             if 'sdkversion' not in thisinfo:
 509                 logging.warn("No SDK version information found in {0}".format(apkfile))
 510                 thisinfo['sdkversion'] = 0
 511
 512             # Check for debuggable apks...
 513             if common.isApkDebuggable(apkfile, config):
 514                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 515
 516             # Calculate the sha256...
 517             sha = hashlib.sha256()
 518             with open(apkfile, 'rb') as f:
 519                 while True:
 520                     t = f.read(1024)
 521                     if len(t) == 0:
 522                         break
 523                     sha.update(t)
 524                 thisinfo['sha256'] = sha.hexdigest()
 525
 526             # Get the signature (or md5 of, to be precise)...
 527             thisinfo['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
 528             if not thisinfo['sig']:
 529                 logging.critical("Failed to get apk signature")
 530                 sys.exit(1)
 531
 532             apk = zipfile.ZipFile(apkfile, 'r')
 533
 534             iconfilename = "%s.%s.png" % (
 535                 thisinfo['id'],
 536                 thisinfo['versioncode'])
 537
 538             # Extract the icon file...
 539             densities = get_densities()
 540             empty_densities = []
 541             for density in densities:
 542                 if density not in thisinfo['icons_src']:
 543                     empty_densities.append(density)
 544                     continue
 545                 iconsrc = thisinfo['icons_src'][density]
 546                 icon_dir = get_icon_dir(repodir, density)
 547                 icondest = os.path.join(icon_dir, iconfilename)
 548
 549                 try:
 550                     iconfile = open(icondest, 'wb')
 551                     iconfile.write(apk.read(iconsrc))
 552                     iconfile.close()
 553                     thisinfo['icons'][density] = iconfilename
 554
 555                 except:
 556                     logging.warn("Error retrieving icon file")
 557                     del thisinfo['icons'][density]
 558                     del thisinfo['icons_src'][density]
 559                     empty_densities.append(density)
 560
 561             if '-1' in thisinfo['icons_src']:
 562                 iconsrc = thisinfo['icons_src']['-1']
 563                 iconpath = os.path.join(
 564                     get_icon_dir(repodir, None), iconfilename)
 565                 iconfile = open(iconpath, 'wb')
 566                 iconfile.write(apk.read(iconsrc))
 567                 iconfile.close()
 568                 try:
 569                     im = Image.open(iconpath)
 570                     dpi = px_to_dpi(im.size[0])
 571                     for density in densities:
 572                         if density in thisinfo['icons']:
 573                             break
 574                         if density == densities[-1] or dpi >= int(density):
 575                             thisinfo['icons'][density] = iconfilename
 576                             shutil.move(iconpath,
 577                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 578                             empty_densities.remove(density)
 579                             break
 580                 except Exception, e:
 581                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 582
 583             if thisinfo['icons']:
 584                 thisinfo['icon'] = iconfilename
 585
 586             apk.close()
 587
 588             # First try resizing down to not lose quality
 589             last_density = None
 590             for density in densities:
 591                 if density not in empty_densities:
 592                     last_density = density
 593                     continue
 594                 if last_density is None:
 595                     continue
 596                 logging.debug("Density %s not available, resizing down from %s"
 597                               % (density, last_density))
 598
 599                 last_iconpath = os.path.join(
 600                     get_icon_dir(repodir, last_density), iconfilename)
 601                 iconpath = os.path.join(
 602                     get_icon_dir(repodir, density), iconfilename)
 603                 try:
 604                     im = Image.open(last_iconpath)
 605                 except:
 606                     logging.warn("Invalid image file at %s" % last_iconpath)
 607                     continue
 608
 609                 size = dpi_to_px(density)
 610
 611                 im.thumbnail((size, size), Image.ANTIALIAS)
 612                 im.save(iconpath, "PNG")
 613                 empty_densities.remove(density)
 614
 615             # Then just copy from the highest resolution available
 616             last_density = None
 617             for density in reversed(densities):
 618                 if density not in empty_densities:
 619                     last_density = density
 620                     continue
 621                 if last_density is None:
 622                     continue
 623                 logging.debug("Density %s not available, copying from lower density %s"
 624                               % (density, last_density))
 625
 626                 shutil.copyfile(
 627                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 628                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 629
 630                 empty_densities.remove(density)
 631
 632             for density in densities:
 633                 icon_dir = get_icon_dir(repodir, density)
 634                 icondest = os.path.join(icon_dir, iconfilename)
 635                 resize_icon(icondest, density)
 636
 637             # Copy from icons-mdpi to icons since mdpi is the baseline density
 638             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 639             if os.path.isfile(baseline):
 640                 shutil.copyfile(baseline,
 641                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 642
 643             # Record in known apks, getting the added date at the same time..
 644             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 645             if added:
 646                 thisinfo['added'] = added
 647
 648             apkcache[apkfilename] = thisinfo
 649             cachechanged = True
 650
 651         apks.append(thisinfo)
 652
 653     return apks, cachechanged
 654
 655
 656 repo_pubkey_fingerprint = None
 657
 658
 659 def make_index(apps, sortedids, apks, repodir, archive, categories):
 660     """Make a repo index.
 661
 662     :param apps: fully populated apps list
 663     :param apks: full populated apks list
 664     :param repodir: the repo directory
 665     :param archive: True if this is the archive repo, False if it's the
 666                     main one.
 667     :param categories: list of categories
 668     """
 669
 670     doc = Document()
 671
 672     def addElement(name, value, doc, parent):
 673         el = doc.createElement(name)
 674         el.appendChild(doc.createTextNode(value))
 675         parent.appendChild(el)
 676
 677     def addElementCDATA(name, value, doc, parent):
 678         el = doc.createElement(name)
 679         el.appendChild(doc.createCDATASection(value))
 680         parent.appendChild(el)
 681
 682     root = doc.createElement("fdroid")
 683     doc.appendChild(root)
 684
 685     repoel = doc.createElement("repo")
 686
 687     if archive:
 688         repoel.setAttribute("name", config['archive_name'])
 689         if config['repo_maxage'] != 0:
 690             repoel.setAttribute("maxage", str(config['repo_maxage']))
 691         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 692         repoel.setAttribute("url", config['archive_url'])
 693         addElement('description', config['archive_description'], doc, repoel)
 694
 695     else:
 696         repoel.setAttribute("name", config['repo_name'])
 697         if config['repo_maxage'] != 0:
 698             repoel.setAttribute("maxage", str(config['repo_maxage']))
 699         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 700         repoel.setAttribute("url", config['repo_url'])
 701         addElement('description', config['repo_description'], doc, repoel)
 702
 703     repoel.setAttribute("version", "12")
 704     repoel.setAttribute("timestamp", str(int(time.time())))
 705
 706     if 'repo_keyalias' in config:
 707
 708         # Generate a certificate fingerprint the same way keytool does it
 709         # (but with slightly different formatting)
 710         def cert_fingerprint(data):
 711             digest = hashlib.sha256(data).digest()
 712             ret = []
 713             ret.append(' '.join("%02X" % ord(b) for b in digest))
 714             return " ".join(ret)
 715
 716         def extract_pubkey():
 717             p = FDroidPopen(['keytool', '-exportcert',
 718                              '-alias', config['repo_keyalias'],
 719                              '-keystore', config['keystore'],
 720                              '-storepass:file', config['keystorepassfile']]
 721                             + config['smartcardoptions'], output=False)
 722             if p.returncode != 0:
 723                 msg = "Failed to get repo pubkey!"
 724                 if config['keystore'] == 'NONE':
 725                     msg += ' Is your crypto smartcard plugged in?'
 726                 logging.critical(msg)
 727                 sys.exit(1)
 728             global repo_pubkey_fingerprint
 729             repo_pubkey_fingerprint = cert_fingerprint(p.output)
 730             return "".join("%02x" % ord(b) for b in p.output)
 731
 732         repoel.setAttribute("pubkey", extract_pubkey())
 733
 734     root.appendChild(repoel)
 735
 736     for appid in sortedids:
 737         app = apps[appid]
 738
 739         if app['Disabled'] is not None:
 740             continue
 741
 742         # Get a list of the apks for this app...
 743         apklist = []
 744         for apk in apks:
 745             if apk['id'] == appid:
 746                 apklist.append(apk)
 747
 748         if len(apklist) == 0:
 749             continue
 750
 751         apel = doc.createElement("application")
 752         apel.setAttribute("id", app['id'])
 753         root.appendChild(apel)
 754
 755         addElement('id', app['id'], doc, apel)
 756         if 'added' in app:
 757             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 758         if 'lastupdated' in app:
 759             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 760         addElement('name', app['Name'], doc, apel)
 761         addElement('summary', app['Summary'], doc, apel)
 762         if app['icon']:
 763             addElement('icon', app['icon'], doc, apel)
 764
 765         def linkres(appid):
 766             if appid in apps:
 767                 return ("fdroid.app:" + appid, apps[appid]['Name'])
 768             raise MetaDataException("Cannot resolve app id " + appid)
 769
 770         addElement('desc',
 771                    metadata.description_html(app['Description'], linkres),
 772                    doc, apel)
 773         addElement('license', app['License'], doc, apel)
 774         if 'Categories' in app:
 775             addElement('categories', ','.join(app["Categories"]), doc, apel)
 776             # We put the first (primary) category in LAST, which will have
 777             # the desired effect of making clients that only understand one
 778             # category see that one.
 779             addElement('category', app["Categories"][0], doc, apel)
 780         addElement('web', app['Web Site'], doc, apel)
 781         addElement('source', app['Source Code'], doc, apel)
 782         addElement('tracker', app['Issue Tracker'], doc, apel)
 783         if app['Donate']:
 784             addElement('donate', app['Donate'], doc, apel)
 785         if app['Bitcoin']:
 786             addElement('bitcoin', app['Bitcoin'], doc, apel)
 787         if app['Litecoin']:
 788             addElement('litecoin', app['Litecoin'], doc, apel)
 789         if app['Dogecoin']:
 790             addElement('dogecoin', app['Dogecoin'], doc, apel)
 791         if app['FlattrID']:
 792             addElement('flattr', app['FlattrID'], doc, apel)
 793
 794         # These elements actually refer to the current version (i.e. which
 795         # one is recommended. They are historically mis-named, and need
 796         # changing, but stay like this for now to support existing clients.
 797         addElement('marketversion', app['Current Version'], doc, apel)
 798         addElement('marketvercode', app['Current Version Code'], doc, apel)
 799
 800         if app['AntiFeatures']:
 801             af = app['AntiFeatures'].split(',')
 802             # TODO: Temporarily not including UpstreamNonFree in the index,
 803             # because current F-Droid clients do not understand it, and also
 804             # look ugly when they encounter an unknown antifeature. This
 805             # filtering can be removed in time...
 806             if 'UpstreamNonFree' in af:
 807                 af.remove('UpstreamNonFree')
 808             if af:
 809                 addElement('antifeatures', ','.join(af), doc, apel)
 810         if app['Provides']:
 811             pv = app['Provides'].split(',')
 812             addElement('provides', ','.join(pv), doc, apel)
 813         if app['Requires Root']:
 814             addElement('requirements', 'root', doc, apel)
 815
 816         # Sort the apk list into version order, just so the web site
 817         # doesn't have to do any work by default...
 818         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 819
 820         # Check for duplicates - they will make the client unhappy...
 821         for i in range(len(apklist) - 1):
 822             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 823                 logging.critical("duplicate versions: '%s' - '%s'" % (
 824                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 825                 sys.exit(1)
 826
 827         for apk in apklist:
 828             apkel = doc.createElement("package")
 829             apel.appendChild(apkel)
 830             addElement('version', apk['version'], doc, apkel)
 831             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 832             addElement('apkname', apk['apkname'], doc, apkel)
 833             if 'srcname' in apk:
 834                 addElement('srcname', apk['srcname'], doc, apkel)
 835             for hash_type in ['sha256']:
 836                 if hash_type not in apk:
 837                     continue
 838                 hashel = doc.createElement("hash")
 839                 hashel.setAttribute("type", hash_type)
 840                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 841                 apkel.appendChild(hashel)
 842             addElement('sig', apk['sig'], doc, apkel)
 843             addElement('size', str(apk['size']), doc, apkel)
 844             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 845             if 'maxsdkversion' in apk:
 846                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 847             if 'added' in apk:
 848                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 849             if app['Requires Root']:
 850                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 851                     apk['permissions'].add('ACCESS_SUPERUSER')
 852
 853             if len(apk['permissions']) > 0:
 854                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 855             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 856                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 857             if len(apk['features']) > 0:
 858                 addElement('features', ','.join(apk['features']), doc, apkel)
 859
 860     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 861     if options.pretty:
 862         output = doc.toprettyxml()
 863     else:
 864         output = doc.toxml()
 865     of.write(output)
 866     of.close()
 867
 868     if 'repo_keyalias' in config:
 869
 870         logging.info("Creating signed index with this key (SHA256):")
 871         logging.info("%s" % repo_pubkey_fingerprint)
 872
 873         # Create a jar of the index...
 874         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 875         if p.returncode != 0:
 876             logging.critical("Failed to create jar file")
 877             sys.exit(1)
 878
 879         # Sign the index...
 880         args = ['jarsigner', '-keystore', config['keystore'],
 881                 '-storepass:file', config['keystorepassfile'],
 882                 '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 883                 os.path.join(repodir, 'index.jar'), config['repo_keyalias']]
 884         if config['keystore'] == 'NONE':
 885             args += config['smartcardoptions']
 886         else:  # smardcards never use -keypass
 887             args += ['-keypass:file', config['keypassfile']]
 888         p = FDroidPopen(args)
 889         # TODO keypass should be sent via stdin
 890         if p.returncode != 0:
 891             logging.critical("Failed to sign index")
 892             sys.exit(1)
 893
 894     # Copy the repo icon into the repo directory...
 895     icon_dir = os.path.join(repodir, 'icons')
 896     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 897     shutil.copyfile(config['repo_icon'], iconfilename)
 898
 899     # Write a category list in the repo to allow quick access...
 900     catdata = ''
 901     for cat in categories:
 902         catdata += cat + '\n'
 903     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 904     f.write(catdata)
 905     f.close()
 906
 907
 908 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 909
 910     for appid, app in apps.iteritems():
 911
 912         # Get a list of the apks for this app...
 913         apklist = []
 914         for apk in apks:
 915             if apk['id'] == appid:
 916                 apklist.append(apk)
 917
 918         # Sort the apk list into version order...
 919         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 920
 921         if app['Archive Policy']:
 922             keepversions = int(app['Archive Policy'][:-9])
 923         else:
 924             keepversions = defaultkeepversions
 925
 926         if len(apklist) > keepversions:
 927             for apk in apklist[keepversions:]:
 928                 logging.info("Moving " + apk['apkname'] + " to archive")
 929                 shutil.move(os.path.join(repodir, apk['apkname']),
 930                             os.path.join(archivedir, apk['apkname']))
 931                 if 'srcname' in apk:
 932                     shutil.move(os.path.join(repodir, apk['srcname']),
 933                                 os.path.join(archivedir, apk['srcname']))
 934                     # Move GPG signature too...
 935                     sigfile = apk['srcname'] + '.asc'
 936                     sigsrc = os.path.join(repodir, sigfile)
 937                     if os.path.exists(sigsrc):
 938                         shutil.move(sigsrc, os.path.join(archivedir, sigfile))
 939
 940                 archapks.append(apk)
 941                 apks.remove(apk)
 942
 943
 944 config = None
 945 options = None
 946
 947
 948 def main():
 949
 950     global config, options
 951
 952     # Parse command line...
 953     parser = OptionParser()
 954     parser.add_option("-c", "--create-metadata", action="store_true", default=False,
 955                       help="Create skeleton metadata files that are missing")
 956     parser.add_option("--delete-unknown", action="store_true", default=False,
 957                       help="Delete APKs without metadata from the repo")
 958     parser.add_option("-v", "--verbose", action="store_true", default=False,
 959                       help="Spew out even more information than normal")
 960     parser.add_option("-q", "--quiet", action="store_true", default=False,
 961                       help="Restrict output to warnings and errors")
 962     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 963                       help="Report on build data status")
 964     parser.add_option("-i", "--interactive", default=False, action="store_true",
 965                       help="Interactively ask about things that need updating.")
 966     parser.add_option("-I", "--icons", action="store_true", default=False,
 967                       help="Resize all the icons exceeding the max pixel size and exit")
 968     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 969                       help="Specify editor to use in interactive mode. Default " +
 970                       "is /etc/alternatives/editor")
 971     parser.add_option("-w", "--wiki", default=False, action="store_true",
 972                       help="Update the wiki")
 973     parser.add_option("", "--pretty", action="store_true", default=False,
 974                       help="Produce human-readable index.xml")
 975     parser.add_option("--clean", action="store_true", default=False,
 976                       help="Clean update - don't uses caches, reprocess all apks")
 977     (options, args) = parser.parse_args()
 978
 979     config = common.read_config(options)
 980
 981     repodirs = ['repo']
 982     if config['archive_older'] != 0:
 983         repodirs.append('archive')
 984         if not os.path.exists('archive'):
 985             os.mkdir('archive')
 986
 987     if options.icons:
 988         resize_all_icons(repodirs)
 989         sys.exit(0)
 990
 991     # check that icons exist now, rather than fail at the end of `fdroid update`
 992     for k in ['repo_icon', 'archive_icon']:
 993         if k in config:
 994             if not os.path.exists(config[k]):
 995                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
 996                 sys.exit(1)
 997
 998     # Get all apps...
 999     apps = metadata.read_metadata()
1000
1001     # Generate a list of categories...
1002     categories = set()
1003     for app in apps.itervalues():
1004         categories.update(app['Categories'])
1005
1006     # Read known apks data (will be updated and written back when we've finished)
1007     knownapks = common.KnownApks()
1008
1009     # Gather information about all the apk files in the repo directory, using
1010     # cached data if possible.
1011     apkcachefile = os.path.join('tmp', 'apkcache')
1012     if not options.clean and os.path.exists(apkcachefile):
1013         with open(apkcachefile, 'rb') as cf:
1014             apkcache = pickle.load(cf)
1015     else:
1016         apkcache = {}
1017     cachechanged = False
1018
1019     delete_disabled_builds(apps, apkcache, repodirs)
1020
1021     # Scan all apks in the main repo
1022     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
1023     if cc:
1024         cachechanged = True
1025
1026     # Generate warnings for apk's with no metadata (or create skeleton
1027     # metadata files, if requested on the command line)
1028     newmetadata = False
1029     for apk in apks:
1030         if apk['id'] not in apps:
1031             if options.create_metadata:
1032                 if 'name' not in apk:
1033                     logging.error(apk['id'] + ' does not have a name! Skipping...')
1034                     continue
1035                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1036                 f.write("License:Unknown\n")
1037                 f.write("Web Site:\n")
1038                 f.write("Source Code:\n")
1039                 f.write("Issue Tracker:\n")
1040                 f.write("Summary:" + apk['name'] + "\n")
1041                 f.write("Description:\n")
1042                 f.write(apk['name'] + "\n")
1043                 f.write(".\n")
1044                 f.close()
1045                 logging.info("Generated skeleton metadata for " + apk['id'])
1046                 newmetadata = True
1047             else:
1048                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1049                 if options.delete_unknown:
1050                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1051                     rmf = os.path.join(repodirs[0], apk['apkname'])
1052                     if not os.path.exists(rmf):
1053                         logging.error("Could not find {0} to remove it".format(rmf))
1054                     else:
1055                         os.remove(rmf)
1056                 else:
1057                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1058
1059     # update the metadata with the newly created ones included
1060     if newmetadata:
1061         apps = metadata.read_metadata()
1062
1063     # Scan the archive repo for apks as well
1064     if len(repodirs) > 1:
1065         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1066         if cc:
1067             cachechanged = True
1068     else:
1069         archapks = []
1070
1071     # Some information from the apks needs to be applied up to the application
1072     # level. When doing this, we use the info from the most recent version's apk.
1073     # We deal with figuring out when the app was added and last updated at the
1074     # same time.
1075     for appid, app in apps.iteritems():
1076         bestver = 0
1077         added = None
1078         lastupdated = None
1079         for apk in apks + archapks:
1080             if apk['id'] == appid:
1081                 if apk['versioncode'] > bestver:
1082                     bestver = apk['versioncode']
1083                     bestapk = apk
1084
1085                 if 'added' in apk:
1086                     if not added or apk['added'] < added:
1087                         added = apk['added']
1088                     if not lastupdated or apk['added'] > lastupdated:
1089                         lastupdated = apk['added']
1090
1091         if added:
1092             app['added'] = added
1093         else:
1094             logging.warn("Don't know when " + appid + " was added")
1095         if lastupdated:
1096             app['lastupdated'] = lastupdated
1097         else:
1098             logging.warn("Don't know when " + appid + " was last updated")
1099
1100         if bestver == 0:
1101             if app['Name'] is None:
1102                 app['Name'] = app['Auto Name'] or appid
1103             app['icon'] = None
1104             logging.warn("Application " + appid + " has no packages")
1105         else:
1106             if app['Name'] is None:
1107                 app['Name'] = bestapk['name']
1108             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1109
1110     # Sort the app list by name, then the web site doesn't have to by default.
1111     # (we had to wait until we'd scanned the apks to do this, because mostly the
1112     # name comes from there!)
1113     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid]['Name'].upper())
1114
1115     if len(repodirs) > 1:
1116         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1117
1118     # Make the index for the main repo...
1119     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1120
1121     # If there's an archive repo,  make the index for it. We already scanned it
1122     # earlier on.
1123     if len(repodirs) > 1:
1124         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1125
1126     if config['update_stats']:
1127
1128         # Update known apks info...
1129         knownapks.writeifchanged()
1130
1131         # Generate latest apps data for widget
1132         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1133             data = ''
1134             for line in file(os.path.join('stats', 'latestapps.txt')):
1135                 appid = line.rstrip()
1136                 data += appid + "\t"
1137                 app = apps[appid]
1138                 data += app['Name'] + "\t"
1139                 if app['icon'] is not None:
1140                     data += app['icon'] + "\t"
1141                 data += app['License'] + "\n"
1142             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1143             f.write(data)
1144             f.close()
1145
1146     if cachechanged:
1147         with open(apkcachefile, 'wb') as cf:
1148             pickle.dump(apkcache, cf)
1149
1150     # Update the wiki...
1151     if options.wiki:
1152         update_wiki(apps, sortedids, apks + archapks)
1153
1154     logging.info("Finished.")
1155
1156 if __name__ == "__main__":
1157     main()