fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from PIL import Image
  33 import logging
  34
  35 import common
  36 import metadata
  37 from common import FDroidPopen, SilentPopen
  38 from metadata import MetaDataException
  39
  40
  41 def get_densities():
  42     return ['640', '480', '320', '240', '160', '120']
  43
  44
  45 def dpi_to_px(density):
  46     return (int(density) * 48) / 160
  47
  48
  49 def px_to_dpi(px):
  50     return (int(px) * 160) / 48
  51
  52
  53 def get_icon_dir(repodir, density):
  54     if density is None:
  55         return os.path.join(repodir, "icons")
  56     return os.path.join(repodir, "icons-%s" % density)
  57
  58
  59 def get_icon_dirs(repodir):
  60     for density in get_densities():
  61         yield get_icon_dir(repodir, density)
  62     yield os.path.join(repodir, "icons")
  63
  64
  65 def update_wiki(apps, sortedids, apks):
  66     """Update the wiki
  67
  68     :param apps: fully populated list of all applications
  69     :param apks: all apks, except...
  70     """
  71     logging.info("Updating wiki")
  72     wikicat = 'Apps'
  73     wikiredircat = 'App Redirects'
  74     import mwclient
  75     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  76                          path=config['wiki_path'])
  77     site.login(config['wiki_user'], config['wiki_password'])
  78     generated_pages = {}
  79     generated_redirects = {}
  80
  81     for appid in sortedids:
  82         app = apps[appid]
  83
  84         wikidata = ''
  85         if app['Disabled']:
  86             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  87         if app['AntiFeatures']:
  88             for af in app['AntiFeatures'].split(','):
  89                 wikidata += '{{AntiFeature|' + af + '}}\n'
  90         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
  91             appid,
  92             app['Name'],
  93             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  94             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  95             app['Source Code'],
  96             app['Issue Tracker'],
  97             app['Web Site'],
  98             app['Donate'],
  99             app['FlattrID'],
 100             app['Bitcoin'],
 101             app['Litecoin'],
 102             app['Dogecoin'],
 103             app['License'],
 104             app.get('Requires Root', 'No'))
 105
 106         if app['Provides']:
 107             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 108
 109         wikidata += app['Summary']
 110         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
 111
 112         wikidata += "=Description=\n"
 113         wikidata += metadata.description_wiki(app['Description']) + "\n"
 114
 115         wikidata += "=Maintainer Notes=\n"
 116         if 'Maintainer Notes' in app:
 117             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 118         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
 119
 120         # Get a list of all packages for this application...
 121         apklist = []
 122         gotcurrentver = False
 123         cantupdate = False
 124         buildfails = False
 125         for apk in apks:
 126             if apk['id'] == appid:
 127                 if str(apk['versioncode']) == app['Current Version Code']:
 128                     gotcurrentver = True
 129                 apklist.append(apk)
 130         # Include ones we can't build, as a special case...
 131         for thisbuild in app['builds']:
 132             if thisbuild['disable']:
 133                 if thisbuild['vercode'] == app['Current Version Code']:
 134                     cantupdate = True
 135                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 136                 apklist.append({'versioncode': int(thisbuild['vercode']),
 137                                 'version': thisbuild['version'],
 138                                 'buildproblem': thisbuild['disable']
 139                                 })
 140             else:
 141                 builtit = False
 142                 for apk in apklist:
 143                     if apk['versioncode'] == int(thisbuild['vercode']):
 144                         builtit = True
 145                         break
 146                 if not builtit:
 147                     buildfails = True
 148                     apklist.append({'versioncode': int(thisbuild['vercode']),
 149                                     'version': thisbuild['version'],
 150                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, thisbuild['vercode'])
 151                                     })
 152         if app['Current Version Code'] == '0':
 153             cantupdate = True
 154         # Sort with most recent first...
 155         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 156
 157         wikidata += "=Versions=\n"
 158         if len(apklist) == 0:
 159             wikidata += "We currently have no versions of this app available."
 160         elif not gotcurrentver:
 161             wikidata += "We don't have the current version of this app."
 162         else:
 163             wikidata += "We have the current version of this app."
 164         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 165         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 166         if len(app['No Source Since']) > 0:
 167             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 168         if len(app['Current Version']) > 0:
 169             wikidata += "The current (recommended) version is " + app['Current Version']
 170             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 171         validapks = 0
 172         for apk in apklist:
 173             wikidata += "==" + apk['version'] + "==\n"
 174
 175             if 'buildproblem' in apk:
 176                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 177             else:
 178                 validapks += 1
 179                 wikidata += "This version is built and signed by "
 180                 if 'srcname' in apk:
 181                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 182                 else:
 183                     wikidata += "the original developer.\n\n"
 184             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 185
 186         wikidata += '\n[[Category:' + wikicat + ']]\n'
 187         if len(app['No Source Since']) > 0:
 188             wikidata += '\n[[Category:Apps missing source code]]\n'
 189         if validapks == 0 and not app['Disabled']:
 190             wikidata += '\n[[Category:Apps with no packages]]\n'
 191         if cantupdate and not app['Disabled']:
 192             wikidata += "\n[[Category:Apps we can't update]]\n"
 193         if buildfails and not app['Disabled']:
 194             wikidata += "\n[[Category:Apps with failing builds]]\n"
 195         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 196             wikidata += '\n[[Category:Apps to Update]]\n'
 197         if app['Disabled']:
 198             wikidata += '\n[[Category:Apps that are disabled]]\n'
 199         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 200             wikidata += '\n[[Category:Apps with no update check]]\n'
 201         for appcat in app['Categories']:
 202             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 203
 204         # We can't have underscores in the page name, even if they're in
 205         # the package ID, because MediaWiki messes with them...
 206         pagename = appid.replace('_', ' ')
 207
 208         # Drop a trailing newline, because mediawiki is going to drop it anyway
 209         # and it we don't we'll think the page has changed when it hasn't...
 210         if wikidata.endswith('\n'):
 211             wikidata = wikidata[:-1]
 212
 213         generated_pages[pagename] = wikidata
 214
 215         # Make a redirect from the name to the ID too, unless there's
 216         # already an existing page with the name and it isn't a redirect.
 217         noclobber = False
 218         apppagename = app['Name'].replace('_', ' ')
 219         apppagename = apppagename.replace('{', '')
 220         apppagename = apppagename.replace('}', ' ')
 221         apppagename = apppagename.replace(':', ' ')
 222         # Drop double spaces caused mostly by replacing ':' above
 223         apppagename = apppagename.replace('  ', ' ')
 224         for expagename in site.allpages(prefix=apppagename,
 225                                         filterredir='nonredirects',
 226                                         generator=False):
 227             if expagename == apppagename:
 228                 noclobber = True
 229         # Another reason not to make the redirect page is if the app name
 230         # is the same as it's ID, because that will overwrite the real page
 231         # with an redirect to itself! (Although it seems like an odd
 232         # scenario this happens a lot, e.g. where there is metadata but no
 233         # builds or binaries to extract a name from.
 234         if apppagename == pagename:
 235             noclobber = True
 236         if not noclobber:
 237             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 238
 239     for tcat, genp in [(wikicat, generated_pages),
 240                        (wikiredircat, generated_redirects)]:
 241         catpages = site.Pages['Category:' + tcat]
 242         existingpages = []
 243         for page in catpages:
 244             existingpages.append(page.name)
 245             if page.name in genp:
 246                 pagetxt = page.edit()
 247                 if pagetxt != genp[page.name]:
 248                     logging.debug("Updating modified page " + page.name)
 249                     page.save(genp[page.name], summary='Auto-updated')
 250                 else:
 251                     logging.debug("Page " + page.name + " is unchanged")
 252             else:
 253                 logging.warn("Deleting page " + page.name)
 254                 page.delete('No longer published')
 255         for pagename, text in genp.items():
 256             logging.debug("Checking " + pagename)
 257             if pagename not in existingpages:
 258                 logging.debug("Creating page " + pagename)
 259                 try:
 260                     newpage = site.Pages[pagename]
 261                     newpage.save(text, summary='Auto-created')
 262                 except:
 263                     logging.error("...FAILED to create page")
 264
 265     # Purge server cache to ensure counts are up to date
 266     site.pages['Repository Maintenance'].purge()
 267
 268
 269 def delete_disabled_builds(apps, apkcache, repodirs):
 270     """Delete disabled build outputs.
 271
 272     :param apps: list of all applications, as per metadata.read_metadata
 273     :param apkcache: current apk cache information
 274     :param repodirs: the repo directories to process
 275     """
 276     for appid, app in apps.iteritems():
 277         for build in app['builds']:
 278             if build['disable']:
 279                 apkfilename = appid + '_' + str(build['vercode']) + '.apk'
 280                 for repodir in repodirs:
 281                     apkpath = os.path.join(repodir, apkfilename)
 282                     ascpath = apkpath + ".asc"
 283                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 284                     for name in [apkpath, srcpath, ascpath]:
 285                         if os.path.exists(name):
 286                             logging.warn("Deleting disabled build output " + apkfilename)
 287                             os.remove(name)
 288                 if apkfilename in apkcache:
 289                     del apkcache[apkfilename]
 290
 291
 292 def resize_icon(iconpath, density):
 293
 294     if not os.path.isfile(iconpath):
 295         return
 296
 297     try:
 298         im = Image.open(iconpath)
 299         size = dpi_to_px(density)
 300
 301         if any(length > size for length in im.size):
 302             oldsize = im.size
 303             im.thumbnail((size, size), Image.ANTIALIAS)
 304             logging.debug("%s was too large at %s - new size is %s" % (
 305                 iconpath, oldsize, im.size))
 306             im.save(iconpath, "PNG")
 307
 308     except Exception, e:
 309         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 310
 311
 312 def resize_all_icons(repodirs):
 313     """Resize all icons that exceed the max size
 314
 315     :param repodirs: the repo directories to process
 316     """
 317     for repodir in repodirs:
 318         for density in get_densities():
 319             icon_dir = get_icon_dir(repodir, density)
 320             icon_glob = os.path.join(icon_dir, '*.png')
 321             for iconpath in glob.glob(icon_glob):
 322                 resize_icon(iconpath, density)
 323
 324
 325 def scan_apks(apps, apkcache, repodir, knownapks):
 326     """Scan the apks in the given repo directory.
 327
 328     This also extracts the icons.
 329
 330     :param apps: list of all applications, as per metadata.read_metadata
 331     :param apkcache: current apk cache information
 332     :param repodir: repo directory to scan
 333     :param knownapks: known apks info
 334     :returns: (apks, cachechanged) where apks is a list of apk information,
 335               and cachechanged is True if the apkcache got changed.
 336     """
 337
 338     cachechanged = False
 339
 340     icon_dirs = get_icon_dirs(repodir)
 341     for icon_dir in icon_dirs:
 342         if os.path.exists(icon_dir):
 343             if options.clean:
 344                 shutil.rmtree(icon_dir)
 345                 os.makedirs(icon_dir)
 346         else:
 347             os.makedirs(icon_dir)
 348
 349     apks = []
 350     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 351     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 352     vername_pat = re.compile(".*versionName='([^']*)'.*")
 353     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 354     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 355     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 356     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 357     string_pat = re.compile(".*'([^']*)'.*")
 358     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 359
 360         apkfilename = apkfile[len(repodir) + 1:]
 361         if ' ' in apkfilename:
 362             logging.critical("Spaces in filenames are not allowed.")
 363             sys.exit(1)
 364
 365         if apkfilename in apkcache:
 366             logging.debug("Reading " + apkfilename + " from cache")
 367             thisinfo = apkcache[apkfilename]
 368
 369         else:
 370             logging.debug("Processing " + apkfilename)
 371             thisinfo = {}
 372             thisinfo['apkname'] = apkfilename
 373             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 374             if os.path.exists(os.path.join(repodir, srcfilename)):
 375                 thisinfo['srcname'] = srcfilename
 376             thisinfo['size'] = os.path.getsize(apkfile)
 377             thisinfo['permissions'] = set()
 378             thisinfo['features'] = set()
 379             thisinfo['icons_src'] = {}
 380             thisinfo['icons'] = {}
 381             p = SilentPopen([config['aapt'], 'dump', 'badging', apkfile])
 382             if p.returncode != 0:
 383                 if options.delete_unknown:
 384                     if os.path.exists(apkfile):
 385                         logging.error("Failed to get apk information, deleting " + apkfile)
 386                         os.remove(apkfile)
 387                     else:
 388                         logging.error("Could not find {0} to remove it".format(apkfile))
 389                 else:
 390                     logging.error("Failed to get apk information, skipping " + apkfile)
 391                 continue
 392             for line in p.output.splitlines():
 393                 if line.startswith("package:"):
 394                     try:
 395                         thisinfo['id'] = re.match(name_pat, line).group(1)
 396                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 397                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 398                     except Exception, e:
 399                         logging.error("Package matching failed: " + str(e))
 400                         logging.info("Line was: " + line)
 401                         sys.exit(1)
 402                 elif line.startswith("application:"):
 403                     thisinfo['name'] = re.match(label_pat, line).group(1)
 404                     # Keep path to non-dpi icon in case we need it
 405                     match = re.match(icon_pat_nodpi, line)
 406                     if match:
 407                         thisinfo['icons_src']['-1'] = match.group(1)
 408                 elif line.startswith("launchable-activity:"):
 409                     # Only use launchable-activity as fallback to application
 410                     if not thisinfo['name']:
 411                         thisinfo['name'] = re.match(label_pat, line).group(1)
 412                     if '-1' not in thisinfo['icons_src']:
 413                         match = re.match(icon_pat_nodpi, line)
 414                         if match:
 415                             thisinfo['icons_src']['-1'] = match.group(1)
 416                 elif line.startswith("application-icon-"):
 417                     match = re.match(icon_pat, line)
 418                     if match:
 419                         density = match.group(1)
 420                         path = match.group(2)
 421                         thisinfo['icons_src'][density] = path
 422                 elif line.startswith("sdkVersion:"):
 423                     m = re.match(sdkversion_pat, line)
 424                     if m is None:
 425                         logging.error(line.replace('sdkVersion:', '')
 426                                       + ' is not a valid minSdkVersion!')
 427                     else:
 428                         thisinfo['sdkversion'] = m.group(1)
 429                 elif line.startswith("maxSdkVersion:"):
 430                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 431                 elif line.startswith("native-code:"):
 432                     thisinfo['nativecode'] = []
 433                     for arch in line[13:].split(' '):
 434                         thisinfo['nativecode'].append(arch[1:-1])
 435                 elif line.startswith("uses-permission:"):
 436                     perm = re.match(string_pat, line).group(1)
 437                     if perm.startswith("android.permission."):
 438                         perm = perm[19:]
 439                     thisinfo['permissions'].add(perm)
 440                 elif line.startswith("uses-feature:"):
 441                     perm = re.match(string_pat, line).group(1)
 442                     # Filter out this, it's only added with the latest SDK tools and
 443                     # causes problems for lots of apps.
 444                     if perm != "android.hardware.screen.portrait" \
 445                             and perm != "android.hardware.screen.landscape":
 446                         if perm.startswith("android.feature."):
 447                             perm = perm[16:]
 448                         thisinfo['features'].add(perm)
 449
 450             if 'sdkversion' not in thisinfo:
 451                 logging.warn("No SDK version information found in {0}".format(apkfile))
 452                 thisinfo['sdkversion'] = 0
 453
 454             # Check for debuggable apks...
 455             if common.isApkDebuggable(apkfile, config):
 456                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 457
 458             # Calculate the sha256...
 459             sha = hashlib.sha256()
 460             with open(apkfile, 'rb') as f:
 461                 while True:
 462                     t = f.read(1024)
 463                     if len(t) == 0:
 464                         break
 465                     sha.update(t)
 466                 thisinfo['sha256'] = sha.hexdigest()
 467
 468             # verify the jar signature is correct
 469             args = ['jarsigner', '-verify']
 470             if options.verbose:
 471                 args += ['-verbose', '-certs']
 472             args += apkfile
 473             p = FDroidPopen(args)
 474             if p.returncode != 0:
 475                 logging.critical(apkfile + " has a bad signature!")
 476                 sys.exit(1)
 477
 478             # Get the signature (or md5 of, to be precise)...
 479             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 480             if not os.path.exists(getsig_dir + "/getsig.class"):
 481                 logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
 482                 sys.exit(1)
 483             p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 484                              'getsig', os.path.join(os.getcwd(), apkfile)])
 485             thisinfo['sig'] = None
 486             for line in p.output.splitlines():
 487                 if line.startswith('Result:'):
 488                     thisinfo['sig'] = line[7:].strip()
 489                     break
 490             if p.returncode != 0 or not thisinfo['sig']:
 491                 logging.critical("Failed to get apk signature")
 492                 sys.exit(1)
 493
 494             apk = zipfile.ZipFile(apkfile, 'r')
 495
 496             iconfilename = "%s.%s.png" % (
 497                 thisinfo['id'],
 498                 thisinfo['versioncode'])
 499
 500             # Extract the icon file...
 501             densities = get_densities()
 502             empty_densities = []
 503             for density in densities:
 504                 if density not in thisinfo['icons_src']:
 505                     empty_densities.append(density)
 506                     continue
 507                 iconsrc = thisinfo['icons_src'][density]
 508                 icon_dir = get_icon_dir(repodir, density)
 509                 icondest = os.path.join(icon_dir, iconfilename)
 510
 511                 try:
 512                     iconfile = open(icondest, 'wb')
 513                     iconfile.write(apk.read(iconsrc))
 514                     iconfile.close()
 515                     thisinfo['icons'][density] = iconfilename
 516
 517                 except:
 518                     logging.warn("Error retrieving icon file")
 519                     del thisinfo['icons'][density]
 520                     del thisinfo['icons_src'][density]
 521                     empty_densities.append(density)
 522
 523             if '-1' in thisinfo['icons_src']:
 524                 iconsrc = thisinfo['icons_src']['-1']
 525                 iconpath = os.path.join(
 526                     get_icon_dir(repodir, None), iconfilename)
 527                 iconfile = open(iconpath, 'wb')
 528                 iconfile.write(apk.read(iconsrc))
 529                 iconfile.close()
 530                 try:
 531                     im = Image.open(iconpath)
 532                     dpi = px_to_dpi(im.size[0])
 533                     for density in densities:
 534                         if density in thisinfo['icons']:
 535                             break
 536                         if density == densities[-1] or dpi >= int(density):
 537                             thisinfo['icons'][density] = iconfilename
 538                             shutil.move(iconpath,
 539                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 540                             empty_densities.remove(density)
 541                             break
 542                 except Exception, e:
 543                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 544
 545             if thisinfo['icons']:
 546                 thisinfo['icon'] = iconfilename
 547
 548             apk.close()
 549
 550             # First try resizing down to not lose quality
 551             last_density = None
 552             for density in densities:
 553                 if density not in empty_densities:
 554                     last_density = density
 555                     continue
 556                 if last_density is None:
 557                     continue
 558                 logging.debug("Density %s not available, resizing down from %s"
 559                               % (density, last_density))
 560
 561                 last_iconpath = os.path.join(
 562                     get_icon_dir(repodir, last_density), iconfilename)
 563                 iconpath = os.path.join(
 564                     get_icon_dir(repodir, density), iconfilename)
 565                 try:
 566                     im = Image.open(last_iconpath)
 567                 except:
 568                     logging.warn("Invalid image file at %s" % last_iconpath)
 569                     continue
 570
 571                 size = dpi_to_px(density)
 572
 573                 im.thumbnail((size, size), Image.ANTIALIAS)
 574                 im.save(iconpath, "PNG")
 575                 empty_densities.remove(density)
 576
 577             # Then just copy from the highest resolution available
 578             last_density = None
 579             for density in reversed(densities):
 580                 if density not in empty_densities:
 581                     last_density = density
 582                     continue
 583                 if last_density is None:
 584                     continue
 585                 logging.debug("Density %s not available, copying from lower density %s"
 586                               % (density, last_density))
 587
 588                 shutil.copyfile(
 589                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 590                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 591
 592                 empty_densities.remove(density)
 593
 594             for density in densities:
 595                 icon_dir = get_icon_dir(repodir, density)
 596                 icondest = os.path.join(icon_dir, iconfilename)
 597                 resize_icon(icondest, density)
 598
 599             # Copy from icons-mdpi to icons since mdpi is the baseline density
 600             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 601             if os.path.isfile(baseline):
 602                 shutil.copyfile(baseline,
 603                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 604
 605             # Record in known apks, getting the added date at the same time..
 606             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 607             if added:
 608                 thisinfo['added'] = added
 609
 610             apkcache[apkfilename] = thisinfo
 611             cachechanged = True
 612
 613         apks.append(thisinfo)
 614
 615     return apks, cachechanged
 616
 617
 618 repo_pubkey_fingerprint = None
 619
 620
 621 def make_index(apps, sortedids, apks, repodir, archive, categories):
 622     """Make a repo index.
 623
 624     :param apps: fully populated apps list
 625     :param apks: full populated apks list
 626     :param repodir: the repo directory
 627     :param archive: True if this is the archive repo, False if it's the
 628                     main one.
 629     :param categories: list of categories
 630     """
 631
 632     doc = Document()
 633
 634     def addElement(name, value, doc, parent):
 635         el = doc.createElement(name)
 636         el.appendChild(doc.createTextNode(value))
 637         parent.appendChild(el)
 638
 639     def addElementCDATA(name, value, doc, parent):
 640         el = doc.createElement(name)
 641         el.appendChild(doc.createCDATASection(value))
 642         parent.appendChild(el)
 643
 644     root = doc.createElement("fdroid")
 645     doc.appendChild(root)
 646
 647     repoel = doc.createElement("repo")
 648
 649     if archive:
 650         repoel.setAttribute("name", config['archive_name'])
 651         if config['repo_maxage'] != 0:
 652             repoel.setAttribute("maxage", str(config['repo_maxage']))
 653         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 654         repoel.setAttribute("url", config['archive_url'])
 655         addElement('description', config['archive_description'], doc, repoel)
 656
 657     else:
 658         repoel.setAttribute("name", config['repo_name'])
 659         if config['repo_maxage'] != 0:
 660             repoel.setAttribute("maxage", str(config['repo_maxage']))
 661         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 662         repoel.setAttribute("url", config['repo_url'])
 663         addElement('description', config['repo_description'], doc, repoel)
 664
 665     repoel.setAttribute("version", "12")
 666     repoel.setAttribute("timestamp", str(int(time.time())))
 667
 668     if 'repo_keyalias' in config:
 669
 670         # Generate a certificate fingerprint the same way keytool does it
 671         # (but with slightly different formatting)
 672         def cert_fingerprint(data):
 673             digest = hashlib.sha256(data).digest()
 674             ret = []
 675             ret.append(' '.join("%02X" % ord(b) for b in digest))
 676             return " ".join(ret)
 677
 678         def extract_pubkey():
 679             p = FDroidPopen(['keytool', '-exportcert',
 680                              '-alias', config['repo_keyalias'],
 681                              '-keystore', config['keystore'],
 682                              '-storepass:file', config['keystorepassfile']]
 683                             + config['smartcardoptions'], output=False)
 684             if p.returncode != 0:
 685                 msg = "Failed to get repo pubkey!"
 686                 if config['keystore'] == 'NONE':
 687                     msg += ' Is your crypto smartcard plugged in?'
 688                 logging.critical(msg)
 689                 sys.exit(1)
 690             global repo_pubkey_fingerprint
 691             repo_pubkey_fingerprint = cert_fingerprint(p.output)
 692             return "".join("%02x" % ord(b) for b in p.output)
 693
 694         repoel.setAttribute("pubkey", extract_pubkey())
 695
 696     root.appendChild(repoel)
 697
 698     for appid in sortedids:
 699         app = apps[appid]
 700
 701         if app['Disabled'] is not None:
 702             continue
 703
 704         # Get a list of the apks for this app...
 705         apklist = []
 706         for apk in apks:
 707             if apk['id'] == appid:
 708                 apklist.append(apk)
 709
 710         if len(apklist) == 0:
 711             continue
 712
 713         apel = doc.createElement("application")
 714         apel.setAttribute("id", app['id'])
 715         root.appendChild(apel)
 716
 717         addElement('id', app['id'], doc, apel)
 718         if 'added' in app:
 719             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 720         if 'lastupdated' in app:
 721             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 722         addElement('name', app['Name'], doc, apel)
 723         addElement('summary', app['Summary'], doc, apel)
 724         if app['icon']:
 725             addElement('icon', app['icon'], doc, apel)
 726
 727         def linkres(appid):
 728             if appid in apps:
 729                 return ("fdroid.app:" + appid, apps[appid]['Name'])
 730             raise MetaDataException("Cannot resolve app id " + appid)
 731
 732         addElement('desc',
 733                    metadata.description_html(app['Description'], linkres),
 734                    doc, apel)
 735         addElement('license', app['License'], doc, apel)
 736         if 'Categories' in app:
 737             addElement('categories', ','.join(app["Categories"]), doc, apel)
 738             # We put the first (primary) category in LAST, which will have
 739             # the desired effect of making clients that only understand one
 740             # category see that one.
 741             addElement('category', app["Categories"][0], doc, apel)
 742         addElement('web', app['Web Site'], doc, apel)
 743         addElement('source', app['Source Code'], doc, apel)
 744         addElement('tracker', app['Issue Tracker'], doc, apel)
 745         if app['Donate']:
 746             addElement('donate', app['Donate'], doc, apel)
 747         if app['Bitcoin']:
 748             addElement('bitcoin', app['Bitcoin'], doc, apel)
 749         if app['Litecoin']:
 750             addElement('litecoin', app['Litecoin'], doc, apel)
 751         if app['Dogecoin']:
 752             addElement('dogecoin', app['Dogecoin'], doc, apel)
 753         if app['FlattrID']:
 754             addElement('flattr', app['FlattrID'], doc, apel)
 755
 756         # These elements actually refer to the current version (i.e. which
 757         # one is recommended. They are historically mis-named, and need
 758         # changing, but stay like this for now to support existing clients.
 759         addElement('marketversion', app['Current Version'], doc, apel)
 760         addElement('marketvercode', app['Current Version Code'], doc, apel)
 761
 762         if app['AntiFeatures']:
 763             af = app['AntiFeatures'].split(',')
 764             # TODO: Temporarily not including UpstreamNonFree in the index,
 765             # because current F-Droid clients do not understand it, and also
 766             # look ugly when they encounter an unknown antifeature. This
 767             # filtering can be removed in time...
 768             if 'UpstreamNonFree' in af:
 769                 af.remove('UpstreamNonFree')
 770             if af:
 771                 addElement('antifeatures', ','.join(af), doc, apel)
 772         if app['Provides']:
 773             pv = app['Provides'].split(',')
 774             addElement('provides', ','.join(pv), doc, apel)
 775         if app['Requires Root']:
 776             addElement('requirements', 'root', doc, apel)
 777
 778         # Sort the apk list into version order, just so the web site
 779         # doesn't have to do any work by default...
 780         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 781
 782         # Check for duplicates - they will make the client unhappy...
 783         for i in range(len(apklist) - 1):
 784             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 785                 logging.critical("duplicate versions: '%s' - '%s'" % (
 786                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 787                 sys.exit(1)
 788
 789         for apk in apklist:
 790             apkel = doc.createElement("package")
 791             apel.appendChild(apkel)
 792             addElement('version', apk['version'], doc, apkel)
 793             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 794             addElement('apkname', apk['apkname'], doc, apkel)
 795             if 'srcname' in apk:
 796                 addElement('srcname', apk['srcname'], doc, apkel)
 797             for hash_type in ['sha256']:
 798                 if hash_type not in apk:
 799                     continue
 800                 hashel = doc.createElement("hash")
 801                 hashel.setAttribute("type", hash_type)
 802                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 803                 apkel.appendChild(hashel)
 804             addElement('sig', apk['sig'], doc, apkel)
 805             addElement('size', str(apk['size']), doc, apkel)
 806             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 807             if 'maxsdkversion' in apk:
 808                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 809             if 'added' in apk:
 810                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 811             if app['Requires Root']:
 812                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 813                     apk['permissions'].add('ACCESS_SUPERUSER')
 814
 815             if len(apk['permissions']) > 0:
 816                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 817             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 818                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 819             if len(apk['features']) > 0:
 820                 addElement('features', ','.join(apk['features']), doc, apkel)
 821
 822     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 823     if options.pretty:
 824         output = doc.toprettyxml()
 825     else:
 826         output = doc.toxml()
 827     of.write(output)
 828     of.close()
 829
 830     if 'repo_keyalias' in config:
 831
 832         logging.info("Creating signed index with this key (SHA256):")
 833         logging.info("%s" % repo_pubkey_fingerprint)
 834
 835         # Create a jar of the index...
 836         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 837         if p.returncode != 0:
 838             logging.critical("Failed to create jar file")
 839             sys.exit(1)
 840
 841         # Sign the index...
 842         args = ['jarsigner', '-keystore', config['keystore'],
 843                 '-storepass:file', config['keystorepassfile'],
 844                 '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 845                 os.path.join(repodir, 'index.jar'), config['repo_keyalias']]
 846         if config['keystore'] == 'NONE':
 847             args += config['smartcardoptions']
 848         else:  # smardcards never use -keypass
 849             args += ['-keypass:file', config['keypassfile']]
 850         p = FDroidPopen(args)
 851         # TODO keypass should be sent via stdin
 852         if p.returncode != 0:
 853             logging.critical("Failed to sign index")
 854             sys.exit(1)
 855
 856     # Copy the repo icon into the repo directory...
 857     icon_dir = os.path.join(repodir, 'icons')
 858     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 859     shutil.copyfile(config['repo_icon'], iconfilename)
 860
 861     # Write a category list in the repo to allow quick access...
 862     catdata = ''
 863     for cat in categories:
 864         catdata += cat + '\n'
 865     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 866     f.write(catdata)
 867     f.close()
 868
 869
 870 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 871
 872     for appid, app in apps.iteritems():
 873
 874         # Get a list of the apks for this app...
 875         apklist = []
 876         for apk in apks:
 877             if apk['id'] == appid:
 878                 apklist.append(apk)
 879
 880         # Sort the apk list into version order...
 881         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 882
 883         if app['Archive Policy']:
 884             keepversions = int(app['Archive Policy'][:-9])
 885         else:
 886             keepversions = defaultkeepversions
 887
 888         if len(apklist) > keepversions:
 889             for apk in apklist[keepversions:]:
 890                 logging.info("Moving " + apk['apkname'] + " to archive")
 891                 shutil.move(os.path.join(repodir, apk['apkname']),
 892                             os.path.join(archivedir, apk['apkname']))
 893                 if 'srcname' in apk:
 894                     shutil.move(os.path.join(repodir, apk['srcname']),
 895                                 os.path.join(archivedir, apk['srcname']))
 896                     # Move GPG signature too...
 897                     sigfile = apk['srcname'] + '.asc'
 898                     sigsrc = os.path.join(repodir, sigfile)
 899                     if os.path.exists(sigsrc):
 900                         shutil.move(sigsrc, os.path.join(archivedir, sigfile))
 901
 902                 archapks.append(apk)
 903                 apks.remove(apk)
 904
 905
 906 config = None
 907 options = None
 908
 909
 910 def main():
 911
 912     global config, options
 913
 914     # Parse command line...
 915     parser = OptionParser()
 916     parser.add_option("-c", "--create-metadata", action="store_true", default=False,
 917                       help="Create skeleton metadata files that are missing")
 918     parser.add_option("--delete-unknown", action="store_true", default=False,
 919                       help="Delete APKs without metadata from the repo")
 920     parser.add_option("-v", "--verbose", action="store_true", default=False,
 921                       help="Spew out even more information than normal")
 922     parser.add_option("-q", "--quiet", action="store_true", default=False,
 923                       help="Restrict output to warnings and errors")
 924     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 925                       help="Report on build data status")
 926     parser.add_option("-i", "--interactive", default=False, action="store_true",
 927                       help="Interactively ask about things that need updating.")
 928     parser.add_option("-I", "--icons", action="store_true", default=False,
 929                       help="Resize all the icons exceeding the max pixel size and exit")
 930     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 931                       help="Specify editor to use in interactive mode. Default " +
 932                       "is /etc/alternatives/editor")
 933     parser.add_option("-w", "--wiki", default=False, action="store_true",
 934                       help="Update the wiki")
 935     parser.add_option("", "--pretty", action="store_true", default=False,
 936                       help="Produce human-readable index.xml")
 937     parser.add_option("--clean", action="store_true", default=False,
 938                       help="Clean update - don't uses caches, reprocess all apks")
 939     (options, args) = parser.parse_args()
 940
 941     config = common.read_config(options)
 942
 943     repodirs = ['repo']
 944     if config['archive_older'] != 0:
 945         repodirs.append('archive')
 946         if not os.path.exists('archive'):
 947             os.mkdir('archive')
 948
 949     if options.icons:
 950         resize_all_icons(repodirs)
 951         sys.exit(0)
 952
 953     # check that icons exist now, rather than fail at the end of `fdroid update`
 954     for k in ['repo_icon', 'archive_icon']:
 955         if k in config:
 956             if not os.path.exists(config[k]):
 957                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
 958                 sys.exit(1)
 959
 960     # Get all apps...
 961     apps = metadata.read_metadata()
 962
 963     # Generate a list of categories...
 964     categories = set()
 965     for app in apps.itervalues():
 966         categories.update(app['Categories'])
 967
 968     # Read known apks data (will be updated and written back when we've finished)
 969     knownapks = common.KnownApks()
 970
 971     # Gather information about all the apk files in the repo directory, using
 972     # cached data if possible.
 973     apkcachefile = os.path.join('tmp', 'apkcache')
 974     if not options.clean and os.path.exists(apkcachefile):
 975         with open(apkcachefile, 'rb') as cf:
 976             apkcache = pickle.load(cf)
 977     else:
 978         apkcache = {}
 979     cachechanged = False
 980
 981     delete_disabled_builds(apps, apkcache, repodirs)
 982
 983     # Scan all apks in the main repo
 984     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 985     if cc:
 986         cachechanged = True
 987
 988     # Generate warnings for apk's with no metadata (or create skeleton
 989     # metadata files, if requested on the command line)
 990     newmetadata = False
 991     for apk in apks:
 992         if apk['id'] not in apps:
 993             if options.create_metadata:
 994                 if 'name' not in apk:
 995                     logging.error(apk['id'] + ' does not have a name! Skipping...')
 996                     continue
 997                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 998                 f.write("License:Unknown\n")
 999                 f.write("Web Site:\n")
1000                 f.write("Source Code:\n")
1001                 f.write("Issue Tracker:\n")
1002                 f.write("Summary:" + apk['name'] + "\n")
1003                 f.write("Description:\n")
1004                 f.write(apk['name'] + "\n")
1005                 f.write(".\n")
1006                 f.close()
1007                 logging.info("Generated skeleton metadata for " + apk['id'])
1008                 newmetadata = True
1009             else:
1010                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1011                 if options.delete_unknown:
1012                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1013                     rmf = os.path.join(repodirs[0], apk['apkname'])
1014                     if not os.path.exists(rmf):
1015                         logging.error("Could not find {0} to remove it".format(rmf))
1016                     else:
1017                         os.remove(rmf)
1018                 else:
1019                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1020
1021     # update the metadata with the newly created ones included
1022     if newmetadata:
1023         apps = metadata.read_metadata()
1024
1025     # Scan the archive repo for apks as well
1026     if len(repodirs) > 1:
1027         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1028         if cc:
1029             cachechanged = True
1030     else:
1031         archapks = []
1032
1033     # Some information from the apks needs to be applied up to the application
1034     # level. When doing this, we use the info from the most recent version's apk.
1035     # We deal with figuring out when the app was added and last updated at the
1036     # same time.
1037     for appid, app in apps.iteritems():
1038         bestver = 0
1039         added = None
1040         lastupdated = None
1041         for apk in apks + archapks:
1042             if apk['id'] == appid:
1043                 if apk['versioncode'] > bestver:
1044                     bestver = apk['versioncode']
1045                     bestapk = apk
1046
1047                 if 'added' in apk:
1048                     if not added or apk['added'] < added:
1049                         added = apk['added']
1050                     if not lastupdated or apk['added'] > lastupdated:
1051                         lastupdated = apk['added']
1052
1053         if added:
1054             app['added'] = added
1055         else:
1056             logging.warn("Don't know when " + appid + " was added")
1057         if lastupdated:
1058             app['lastupdated'] = lastupdated
1059         else:
1060             logging.warn("Don't know when " + appid + " was last updated")
1061
1062         if bestver == 0:
1063             if app['Name'] is None:
1064                 app['Name'] = app['Auto Name'] or appid
1065             app['icon'] = None
1066             logging.warn("Application " + appid + " has no packages")
1067         else:
1068             if app['Name'] is None:
1069                 app['Name'] = bestapk['name']
1070             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1071
1072     # Sort the app list by name, then the web site doesn't have to by default.
1073     # (we had to wait until we'd scanned the apks to do this, because mostly the
1074     # name comes from there!)
1075     sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid]['Name'].upper())
1076
1077     if len(repodirs) > 1:
1078         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1079
1080     # Make the index for the main repo...
1081     make_index(apps, sortedids, apks, repodirs[0], False, categories)
1082
1083     # If there's an archive repo,  make the index for it. We already scanned it
1084     # earlier on.
1085     if len(repodirs) > 1:
1086         make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1087
1088     if config['update_stats']:
1089
1090         # Update known apks info...
1091         knownapks.writeifchanged()
1092
1093         # Generate latest apps data for widget
1094         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1095             data = ''
1096             for line in file(os.path.join('stats', 'latestapps.txt')):
1097                 appid = line.rstrip()
1098                 data += appid + "\t"
1099                 app = apps[appid]
1100                 data += app['Name'] + "\t"
1101                 if app['icon'] is not None:
1102                     data += app['icon'] + "\t"
1103                 data += app['License'] + "\n"
1104             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1105             f.write(data)
1106             f.close()
1107
1108     if cachechanged:
1109         with open(apkcachefile, 'wb') as cf:
1110             pickle.dump(apkcache, cf)
1111
1112     # Update the wiki...
1113     if options.wiki:
1114         update_wiki(apps, sortedids, apks + archapks)
1115
1116     logging.info("Finished.")
1117
1118 if __name__ == "__main__":
1119     main()