fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from PIL import Image
  33 import logging
  34
  35 import common
  36 import metadata
  37 from common import FDroidPopen, SilentPopen
  38 from metadata import MetaDataException
  39
  40
  41 def get_densities():
  42     return ['640', '480', '320', '240', '160', '120']
  43
  44
  45 def dpi_to_px(density):
  46     return (int(density) * 48) / 160
  47
  48
  49 def px_to_dpi(px):
  50     return (int(px) * 160) / 48
  51
  52
  53 def get_icon_dir(repodir, density):
  54     if density is None:
  55         return os.path.join(repodir, "icons")
  56     return os.path.join(repodir, "icons-%s" % density)
  57
  58
  59 def get_icon_dirs(repodir):
  60     for density in get_densities():
  61         yield get_icon_dir(repodir, density)
  62     yield os.path.join(repodir, "icons")
  63
  64
  65 def update_wiki(apps, apks):
  66     """Update the wiki
  67
  68     :param apps: fully populated list of all applications
  69     :param apks: all apks, except...
  70     """
  71     logging.info("Updating wiki")
  72     wikicat = 'Apps'
  73     wikiredircat = 'App Redirects'
  74     import mwclient
  75     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  76                          path=config['wiki_path'])
  77     site.login(config['wiki_user'], config['wiki_password'])
  78     generated_pages = {}
  79     generated_redirects = {}
  80     for app in apps:
  81         wikidata = ''
  82         if app['Disabled']:
  83             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  84         if app['AntiFeatures']:
  85             for af in app['AntiFeatures'].split(','):
  86                 wikidata += '{{AntiFeature|' + af + '}}\n'
  87         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
  88             app['id'],
  89             app['Name'],
  90             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  91             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  92             app['Source Code'],
  93             app['Issue Tracker'],
  94             app['Web Site'],
  95             app['Donate'],
  96             app['FlattrID'],
  97             app['Bitcoin'],
  98             app['Litecoin'],
  99             app['Dogecoin'],
 100             app['License'],
 101             app.get('Requires Root', 'No'))
 102
 103         if app['Provides']:
 104             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 105
 106         wikidata += app['Summary']
 107         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 108
 109         wikidata += "=Description=\n"
 110         wikidata += metadata.description_wiki(app['Description']) + "\n"
 111
 112         wikidata += "=Maintainer Notes=\n"
 113         if 'Maintainer Notes' in app:
 114             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 115         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(app['id'])
 116
 117         # Get a list of all packages for this application...
 118         apklist = []
 119         gotcurrentver = False
 120         cantupdate = False
 121         buildfails = False
 122         for apk in apks:
 123             if apk['id'] == app['id']:
 124                 if str(apk['versioncode']) == app['Current Version Code']:
 125                     gotcurrentver = True
 126                 apklist.append(apk)
 127         # Include ones we can't build, as a special case...
 128         for thisbuild in app['builds']:
 129             if thisbuild['disable']:
 130                 if thisbuild['vercode'] == app['Current Version Code']:
 131                     cantupdate = True
 132                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 133                 apklist.append({'versioncode': int(thisbuild['vercode']),
 134                                 'version': thisbuild['version'],
 135                                 'buildproblem': thisbuild['disable']
 136                                 })
 137             else:
 138                 builtit = False
 139                 for apk in apklist:
 140                     if apk['versioncode'] == int(thisbuild['vercode']):
 141                         builtit = True
 142                         break
 143                 if not builtit:
 144                     buildfails = True
 145                     apklist.append({'versioncode': int(thisbuild['vercode']),
 146                                     'version': thisbuild['version'],
 147                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(app['id'], thisbuild['vercode'])
 148                                     })
 149         if app['Current Version Code'] == '0':
 150             cantupdate = True
 151         # Sort with most recent first...
 152         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 153
 154         wikidata += "=Versions=\n"
 155         if len(apklist) == 0:
 156             wikidata += "We currently have no versions of this app available."
 157         elif not gotcurrentver:
 158             wikidata += "We don't have the current version of this app."
 159         else:
 160             wikidata += "We have the current version of this app."
 161         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 162         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 163         if len(app['No Source Since']) > 0:
 164             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 165         if len(app['Current Version']) > 0:
 166             wikidata += "The current (recommended) version is " + app['Current Version']
 167             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 168         validapks = 0
 169         for apk in apklist:
 170             wikidata += "==" + apk['version'] + "==\n"
 171
 172             if 'buildproblem' in apk:
 173                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 174             else:
 175                 validapks += 1
 176                 wikidata += "This version is built and signed by "
 177                 if 'srcname' in apk:
 178                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 179                 else:
 180                     wikidata += "the original developer.\n\n"
 181             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 182
 183         wikidata += '\n[[Category:' + wikicat + ']]\n'
 184         if len(app['No Source Since']) > 0:
 185             wikidata += '\n[[Category:Apps missing source code]]\n'
 186         if validapks == 0 and not app['Disabled']:
 187             wikidata += '\n[[Category:Apps with no packages]]\n'
 188         if cantupdate and not app['Disabled']:
 189             wikidata += "\n[[Category:Apps we can't update]]\n"
 190         if buildfails and not app['Disabled']:
 191             wikidata += "\n[[Category:Apps with failing builds]]\n"
 192         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 193             wikidata += '\n[[Category:Apps to Update]]\n'
 194         if app['Disabled']:
 195             wikidata += '\n[[Category:Apps that are disabled]]\n'
 196         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 197             wikidata += '\n[[Category:Apps with no update check]]\n'
 198         for appcat in app['Categories']:
 199             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 200
 201         # We can't have underscores in the page name, even if they're in
 202         # the package ID, because MediaWiki messes with them...
 203         pagename = app['id'].replace('_', ' ')
 204
 205         # Drop a trailing newline, because mediawiki is going to drop it anyway
 206         # and it we don't we'll think the page has changed when it hasn't...
 207         if wikidata.endswith('\n'):
 208             wikidata = wikidata[:-1]
 209
 210         generated_pages[pagename] = wikidata
 211
 212         # Make a redirect from the name to the ID too, unless there's
 213         # already an existing page with the name and it isn't a redirect.
 214         noclobber = False
 215         apppagename = app['Name'].replace('_', ' ')
 216         apppagename = apppagename.replace('{', '')
 217         apppagename = apppagename.replace('}', ' ')
 218         apppagename = apppagename.replace(':', ' ')
 219         # Drop double spaces caused mostly by replacing ':' above
 220         apppagename = apppagename.replace('  ', ' ')
 221         for expagename in site.allpages(prefix=apppagename,
 222                                         filterredir='nonredirects',
 223                                         generator=False):
 224             if expagename == apppagename:
 225                 noclobber = True
 226         # Another reason not to make the redirect page is if the app name
 227         # is the same as it's ID, because that will overwrite the real page
 228         # with an redirect to itself! (Although it seems like an odd
 229         # scenario this happens a lot, e.g. where there is metadata but no
 230         # builds or binaries to extract a name from.
 231         if apppagename == pagename:
 232             noclobber = True
 233         if not noclobber:
 234             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 235
 236     for tcat, genp in [(wikicat, generated_pages),
 237                        (wikiredircat, generated_redirects)]:
 238         catpages = site.Pages['Category:' + tcat]
 239         existingpages = []
 240         for page in catpages:
 241             existingpages.append(page.name)
 242             if page.name in genp:
 243                 pagetxt = page.edit()
 244                 if pagetxt != genp[page.name]:
 245                     logging.debug("Updating modified page " + page.name)
 246                     page.save(genp[page.name], summary='Auto-updated')
 247                 else:
 248                     logging.debug("Page " + page.name + " is unchanged")
 249             else:
 250                 logging.warn("Deleting page " + page.name)
 251                 page.delete('No longer published')
 252         for pagename, text in genp.items():
 253             logging.debug("Checking " + pagename)
 254             if pagename not in existingpages:
 255                 logging.debug("Creating page " + pagename)
 256                 try:
 257                     newpage = site.Pages[pagename]
 258                     newpage.save(text, summary='Auto-created')
 259                 except:
 260                     logging.error("...FAILED to create page")
 261
 262     # Purge server cache to ensure counts are up to date
 263     site.pages['Repository Maintenance'].purge()
 264
 265
 266 def delete_disabled_builds(apps, apkcache, repodirs):
 267     """Delete disabled build outputs.
 268
 269     :param apps: list of all applications, as per metadata.read_metadata
 270     :param apkcache: current apk cache information
 271     :param repodirs: the repo directories to process
 272     """
 273     for app in apps:
 274         for build in app['builds']:
 275             if build['disable']:
 276                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 277                 for repodir in repodirs:
 278                     apkpath = os.path.join(repodir, apkfilename)
 279                     ascpath = apkpath + ".asc"
 280                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 281                     for name in [apkpath, srcpath, ascpath]:
 282                         if os.path.exists(name):
 283                             logging.warn("Deleting disabled build output " + apkfilename)
 284                             os.remove(name)
 285                 if apkfilename in apkcache:
 286                     del apkcache[apkfilename]
 287
 288
 289 def resize_icon(iconpath, density):
 290
 291     if not os.path.isfile(iconpath):
 292         return
 293
 294     try:
 295         im = Image.open(iconpath)
 296         size = dpi_to_px(density)
 297
 298         if any(length > size for length in im.size):
 299             oldsize = im.size
 300             im.thumbnail((size, size), Image.ANTIALIAS)
 301             logging.debug("%s was too large at %s - new size is %s" % (
 302                 iconpath, oldsize, im.size))
 303             im.save(iconpath, "PNG")
 304
 305     except Exception, e:
 306         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 307
 308
 309 def resize_all_icons(repodirs):
 310     """Resize all icons that exceed the max size
 311
 312     :param repodirs: the repo directories to process
 313     """
 314     for repodir in repodirs:
 315         for density in get_densities():
 316             icon_dir = get_icon_dir(repodir, density)
 317             icon_glob = os.path.join(icon_dir, '*.png')
 318             for iconpath in glob.glob(icon_glob):
 319                 resize_icon(iconpath, density)
 320
 321
 322 def scan_apks(apps, apkcache, repodir, knownapks):
 323     """Scan the apks in the given repo directory.
 324
 325     This also extracts the icons.
 326
 327     :param apps: list of all applications, as per metadata.read_metadata
 328     :param apkcache: current apk cache information
 329     :param repodir: repo directory to scan
 330     :param knownapks: known apks info
 331     :returns: (apks, cachechanged) where apks is a list of apk information,
 332               and cachechanged is True if the apkcache got changed.
 333     """
 334
 335     cachechanged = False
 336
 337     icon_dirs = get_icon_dirs(repodir)
 338     for icon_dir in icon_dirs:
 339         if os.path.exists(icon_dir):
 340             if options.clean:
 341                 shutil.rmtree(icon_dir)
 342                 os.makedirs(icon_dir)
 343         else:
 344             os.makedirs(icon_dir)
 345
 346     apks = []
 347     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 348     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 349     vername_pat = re.compile(".*versionName='([^']*)'.*")
 350     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 351     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 352     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 353     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 354     string_pat = re.compile(".*'([^']*)'.*")
 355     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 356
 357         apkfilename = apkfile[len(repodir) + 1:]
 358         if ' ' in apkfilename:
 359             logging.critical("Spaces in filenames are not allowed.")
 360             sys.exit(1)
 361
 362         if apkfilename in apkcache:
 363             logging.debug("Reading " + apkfilename + " from cache")
 364             thisinfo = apkcache[apkfilename]
 365
 366         else:
 367             logging.debug("Processing " + apkfilename)
 368             thisinfo = {}
 369             thisinfo['apkname'] = apkfilename
 370             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 371             if os.path.exists(os.path.join(repodir, srcfilename)):
 372                 thisinfo['srcname'] = srcfilename
 373             thisinfo['size'] = os.path.getsize(apkfile)
 374             thisinfo['permissions'] = set()
 375             thisinfo['features'] = set()
 376             thisinfo['icons_src'] = {}
 377             thisinfo['icons'] = {}
 378             p = SilentPopen([config['aapt'], 'dump', 'badging', apkfile])
 379             if p.returncode != 0:
 380                 if options.delete_unknown:
 381                     if os.path.exists(apkfile):
 382                         logging.error("Failed to get apk information, deleting " + apkfile)
 383                         os.remove(apkfile)
 384                     else:
 385                         logging.error("Could not find {0} to remove it".format(apkfile))
 386                 else:
 387                     logging.error("Failed to get apk information, skipping " + apkfile)
 388                 continue
 389             for line in p.output.splitlines():
 390                 if line.startswith("package:"):
 391                     try:
 392                         thisinfo['id'] = re.match(name_pat, line).group(1)
 393                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 394                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 395                     except Exception, e:
 396                         logging.error("Package matching failed: " + str(e))
 397                         logging.info("Line was: " + line)
 398                         sys.exit(1)
 399                 elif line.startswith("application:"):
 400                     thisinfo['name'] = re.match(label_pat, line).group(1)
 401                     # Keep path to non-dpi icon in case we need it
 402                     match = re.match(icon_pat_nodpi, line)
 403                     if match:
 404                         thisinfo['icons_src']['-1'] = match.group(1)
 405                 elif line.startswith("launchable-activity:"):
 406                     # Only use launchable-activity as fallback to application
 407                     if not thisinfo['name']:
 408                         thisinfo['name'] = re.match(label_pat, line).group(1)
 409                     if '-1' not in thisinfo['icons_src']:
 410                         match = re.match(icon_pat_nodpi, line)
 411                         if match:
 412                             thisinfo['icons_src']['-1'] = match.group(1)
 413                 elif line.startswith("application-icon-"):
 414                     match = re.match(icon_pat, line)
 415                     if match:
 416                         density = match.group(1)
 417                         path = match.group(2)
 418                         thisinfo['icons_src'][density] = path
 419                 elif line.startswith("sdkVersion:"):
 420                     m = re.match(sdkversion_pat, line)
 421                     if m is None:
 422                         logging.error(line.replace('sdkVersion:', '')
 423                                       + ' is not a valid minSdkVersion!')
 424                     else:
 425                         thisinfo['sdkversion'] = m.group(1)
 426                 elif line.startswith("maxSdkVersion:"):
 427                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 428                 elif line.startswith("native-code:"):
 429                     thisinfo['nativecode'] = []
 430                     for arch in line[13:].split(' '):
 431                         thisinfo['nativecode'].append(arch[1:-1])
 432                 elif line.startswith("uses-permission:"):
 433                     perm = re.match(string_pat, line).group(1)
 434                     if perm.startswith("android.permission."):
 435                         perm = perm[19:]
 436                     thisinfo['permissions'].add(perm)
 437                 elif line.startswith("uses-feature:"):
 438                     perm = re.match(string_pat, line).group(1)
 439                     # Filter out this, it's only added with the latest SDK tools and
 440                     # causes problems for lots of apps.
 441                     if perm != "android.hardware.screen.portrait" \
 442                             and perm != "android.hardware.screen.landscape":
 443                         if perm.startswith("android.feature."):
 444                             perm = perm[16:]
 445                         thisinfo['features'].add(perm)
 446
 447             if 'sdkversion' not in thisinfo:
 448                 logging.warn("No SDK version information found in {0}".format(apkfile))
 449                 thisinfo['sdkversion'] = 0
 450
 451             # Check for debuggable apks...
 452             if common.isApkDebuggable(apkfile, config):
 453                 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
 454
 455             # Calculate the sha256...
 456             sha = hashlib.sha256()
 457             with open(apkfile, 'rb') as f:
 458                 while True:
 459                     t = f.read(1024)
 460                     if len(t) == 0:
 461                         break
 462                     sha.update(t)
 463                 thisinfo['sha256'] = sha.hexdigest()
 464
 465             # Get the signature (or md5 of, to be precise)...
 466             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 467             if not os.path.exists(getsig_dir + "/getsig.class"):
 468                 logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
 469                 sys.exit(1)
 470             p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 471                              'getsig', os.path.join(os.getcwd(), apkfile)])
 472             thisinfo['sig'] = None
 473             for line in p.output.splitlines():
 474                 if line.startswith('Result:'):
 475                     thisinfo['sig'] = line[7:].strip()
 476                     break
 477             if p.returncode != 0 or not thisinfo['sig']:
 478                 logging.critical("Failed to get apk signature")
 479                 sys.exit(1)
 480
 481             apk = zipfile.ZipFile(apkfile, 'r')
 482
 483             iconfilename = "%s.%s.png" % (
 484                 thisinfo['id'],
 485                 thisinfo['versioncode'])
 486
 487             # Extract the icon file...
 488             densities = get_densities()
 489             empty_densities = []
 490             for density in densities:
 491                 if density not in thisinfo['icons_src']:
 492                     empty_densities.append(density)
 493                     continue
 494                 iconsrc = thisinfo['icons_src'][density]
 495                 icon_dir = get_icon_dir(repodir, density)
 496                 icondest = os.path.join(icon_dir, iconfilename)
 497
 498                 try:
 499                     iconfile = open(icondest, 'wb')
 500                     iconfile.write(apk.read(iconsrc))
 501                     iconfile.close()
 502                     thisinfo['icons'][density] = iconfilename
 503
 504                 except:
 505                     logging.warn("Error retrieving icon file")
 506                     del thisinfo['icons'][density]
 507                     del thisinfo['icons_src'][density]
 508                     empty_densities.append(density)
 509
 510             if '-1' in thisinfo['icons_src']:
 511                 iconsrc = thisinfo['icons_src']['-1']
 512                 iconpath = os.path.join(
 513                     get_icon_dir(repodir, None), iconfilename)
 514                 iconfile = open(iconpath, 'wb')
 515                 iconfile.write(apk.read(iconsrc))
 516                 iconfile.close()
 517                 try:
 518                     im = Image.open(iconpath)
 519                     dpi = px_to_dpi(im.size[0])
 520                     for density in densities:
 521                         if density in thisinfo['icons']:
 522                             break
 523                         if density == densities[-1] or dpi >= int(density):
 524                             thisinfo['icons'][density] = iconfilename
 525                             shutil.move(iconpath,
 526                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 527                             empty_densities.remove(density)
 528                             break
 529                 except Exception, e:
 530                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 531
 532             if thisinfo['icons']:
 533                 thisinfo['icon'] = iconfilename
 534
 535             apk.close()
 536
 537             # First try resizing down to not lose quality
 538             last_density = None
 539             for density in densities:
 540                 if density not in empty_densities:
 541                     last_density = density
 542                     continue
 543                 if last_density is None:
 544                     continue
 545                 logging.debug("Density %s not available, resizing down from %s"
 546                               % (density, last_density))
 547
 548                 last_iconpath = os.path.join(
 549                     get_icon_dir(repodir, last_density), iconfilename)
 550                 iconpath = os.path.join(
 551                     get_icon_dir(repodir, density), iconfilename)
 552                 try:
 553                     im = Image.open(last_iconpath)
 554                 except:
 555                     logging.warn("Invalid image file at %s" % last_iconpath)
 556                     continue
 557
 558                 size = dpi_to_px(density)
 559
 560                 im.thumbnail((size, size), Image.ANTIALIAS)
 561                 im.save(iconpath, "PNG")
 562                 empty_densities.remove(density)
 563
 564             # Then just copy from the highest resolution available
 565             last_density = None
 566             for density in reversed(densities):
 567                 if density not in empty_densities:
 568                     last_density = density
 569                     continue
 570                 if last_density is None:
 571                     continue
 572                 logging.debug("Density %s not available, copying from lower density %s"
 573                               % (density, last_density))
 574
 575                 shutil.copyfile(
 576                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 577                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 578
 579                 empty_densities.remove(density)
 580
 581             for density in densities:
 582                 icon_dir = get_icon_dir(repodir, density)
 583                 icondest = os.path.join(icon_dir, iconfilename)
 584                 resize_icon(icondest, density)
 585
 586             # Copy from icons-mdpi to icons since mdpi is the baseline density
 587             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 588             if os.path.isfile(baseline):
 589                 shutil.copyfile(baseline,
 590                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 591
 592             # Record in known apks, getting the added date at the same time..
 593             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 594             if added:
 595                 thisinfo['added'] = added
 596
 597             apkcache[apkfilename] = thisinfo
 598             cachechanged = True
 599
 600         apks.append(thisinfo)
 601
 602     return apks, cachechanged
 603
 604
 605 repo_pubkey_fingerprint = None
 606
 607
 608 def make_index(apps, apks, repodir, archive, categories):
 609     """Make a repo index.
 610
 611     :param apps: fully populated apps list
 612     :param apks: full populated apks list
 613     :param repodir: the repo directory
 614     :param archive: True if this is the archive repo, False if it's the
 615                     main one.
 616     :param categories: list of categories
 617     """
 618
 619     doc = Document()
 620
 621     def addElement(name, value, doc, parent):
 622         el = doc.createElement(name)
 623         el.appendChild(doc.createTextNode(value))
 624         parent.appendChild(el)
 625
 626     def addElementCDATA(name, value, doc, parent):
 627         el = doc.createElement(name)
 628         el.appendChild(doc.createCDATASection(value))
 629         parent.appendChild(el)
 630
 631     root = doc.createElement("fdroid")
 632     doc.appendChild(root)
 633
 634     repoel = doc.createElement("repo")
 635
 636     if archive:
 637         repoel.setAttribute("name", config['archive_name'])
 638         if config['repo_maxage'] != 0:
 639             repoel.setAttribute("maxage", str(config['repo_maxage']))
 640         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 641         repoel.setAttribute("url", config['archive_url'])
 642         addElement('description', config['archive_description'], doc, repoel)
 643
 644     else:
 645         repoel.setAttribute("name", config['repo_name'])
 646         if config['repo_maxage'] != 0:
 647             repoel.setAttribute("maxage", str(config['repo_maxage']))
 648         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 649         repoel.setAttribute("url", config['repo_url'])
 650         addElement('description', config['repo_description'], doc, repoel)
 651
 652     repoel.setAttribute("version", "12")
 653     repoel.setAttribute("timestamp", str(int(time.time())))
 654
 655     if 'repo_keyalias' in config:
 656
 657         # Generate a certificate fingerprint the same way keytool does it
 658         # (but with slightly different formatting)
 659         def cert_fingerprint(data):
 660             digest = hashlib.sha256(data).digest()
 661             ret = []
 662             ret.append(' '.join("%02X" % ord(b) for b in digest))
 663             return " ".join(ret)
 664
 665         def extract_pubkey():
 666             p = FDroidPopen(['keytool', '-exportcert',
 667                              '-alias', config['repo_keyalias'],
 668                              '-keystore', config['keystore'],
 669                              '-storepass:file', config['keystorepassfile']]
 670                             + config['smartcardoptions'], output=False)
 671             if p.returncode != 0:
 672                 msg = "Failed to get repo pubkey!"
 673                 if config['keystore'] == 'NONE':
 674                     msg += ' Is your crypto smartcard plugged in?'
 675                 logging.critical(msg)
 676                 sys.exit(1)
 677             global repo_pubkey_fingerprint
 678             repo_pubkey_fingerprint = cert_fingerprint(p.output)
 679             return "".join("%02x" % ord(b) for b in p.output)
 680
 681         repoel.setAttribute("pubkey", extract_pubkey())
 682
 683     root.appendChild(repoel)
 684
 685     for app in apps:
 686
 687         if app['Disabled'] is not None:
 688             continue
 689
 690         # Get a list of the apks for this app...
 691         apklist = []
 692         for apk in apks:
 693             if apk['id'] == app['id']:
 694                 apklist.append(apk)
 695
 696         if len(apklist) == 0:
 697             continue
 698
 699         apel = doc.createElement("application")
 700         apel.setAttribute("id", app['id'])
 701         root.appendChild(apel)
 702
 703         addElement('id', app['id'], doc, apel)
 704         if 'added' in app:
 705             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 706         if 'lastupdated' in app:
 707             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 708         addElement('name', app['Name'], doc, apel)
 709         addElement('summary', app['Summary'], doc, apel)
 710         if app['icon']:
 711             addElement('icon', app['icon'], doc, apel)
 712
 713         def linkres(link):
 714             for app in apps:
 715                 if app['id'] == link:
 716                     return ("fdroid.app:" + link, app['Name'])
 717             raise MetaDataException("Cannot resolve app id " + link)
 718         addElement('desc',
 719                    metadata.description_html(app['Description'], linkres),
 720                    doc, apel)
 721         addElement('license', app['License'], doc, apel)
 722         if 'Categories' in app:
 723             addElement('categories', ','.join(app["Categories"]), doc, apel)
 724             # We put the first (primary) category in LAST, which will have
 725             # the desired effect of making clients that only understand one
 726             # category see that one.
 727             addElement('category', app["Categories"][0], doc, apel)
 728         addElement('web', app['Web Site'], doc, apel)
 729         addElement('source', app['Source Code'], doc, apel)
 730         addElement('tracker', app['Issue Tracker'], doc, apel)
 731         if app['Donate']:
 732             addElement('donate', app['Donate'], doc, apel)
 733         if app['Bitcoin']:
 734             addElement('bitcoin', app['Bitcoin'], doc, apel)
 735         if app['Litecoin']:
 736             addElement('litecoin', app['Litecoin'], doc, apel)
 737         if app['Dogecoin']:
 738             addElement('dogecoin', app['Dogecoin'], doc, apel)
 739         if app['FlattrID']:
 740             addElement('flattr', app['FlattrID'], doc, apel)
 741
 742         # These elements actually refer to the current version (i.e. which
 743         # one is recommended. They are historically mis-named, and need
 744         # changing, but stay like this for now to support existing clients.
 745         addElement('marketversion', app['Current Version'], doc, apel)
 746         addElement('marketvercode', app['Current Version Code'], doc, apel)
 747
 748         if app['AntiFeatures']:
 749             af = app['AntiFeatures'].split(',')
 750             # TODO: Temporarily not including UpstreamNonFree in the index,
 751             # because current F-Droid clients do not understand it, and also
 752             # look ugly when they encounter an unknown antifeature. This
 753             # filtering can be removed in time...
 754             if 'UpstreamNonFree' in af:
 755                 af.remove('UpstreamNonFree')
 756             if af:
 757                 addElement('antifeatures', ','.join(af), doc, apel)
 758         if app['Provides']:
 759             pv = app['Provides'].split(',')
 760             addElement('provides', ','.join(pv), doc, apel)
 761         if app['Requires Root']:
 762             addElement('requirements', 'root', doc, apel)
 763
 764         # Sort the apk list into version order, just so the web site
 765         # doesn't have to do any work by default...
 766         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 767
 768         # Check for duplicates - they will make the client unhappy...
 769         for i in range(len(apklist) - 1):
 770             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 771                 logging.critical("duplicate versions: '%s' - '%s'" % (
 772                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 773                 sys.exit(1)
 774
 775         for apk in apklist:
 776             apkel = doc.createElement("package")
 777             apel.appendChild(apkel)
 778             addElement('version', apk['version'], doc, apkel)
 779             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 780             addElement('apkname', apk['apkname'], doc, apkel)
 781             if 'srcname' in apk:
 782                 addElement('srcname', apk['srcname'], doc, apkel)
 783             for hash_type in ['sha256']:
 784                 if hash_type not in apk:
 785                     continue
 786                 hashel = doc.createElement("hash")
 787                 hashel.setAttribute("type", hash_type)
 788                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 789                 apkel.appendChild(hashel)
 790             addElement('sig', apk['sig'], doc, apkel)
 791             addElement('size', str(apk['size']), doc, apkel)
 792             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 793             if 'maxsdkversion' in apk:
 794                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 795             if 'added' in apk:
 796                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 797             if app['Requires Root']:
 798                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 799                     apk['permissions'].add('ACCESS_SUPERUSER')
 800
 801             if len(apk['permissions']) > 0:
 802                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 803             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 804                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 805             if len(apk['features']) > 0:
 806                 addElement('features', ','.join(apk['features']), doc, apkel)
 807
 808     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 809     if options.pretty:
 810         output = doc.toprettyxml()
 811     else:
 812         output = doc.toxml()
 813     of.write(output)
 814     of.close()
 815
 816     if 'repo_keyalias' in config:
 817
 818         logging.info("Creating signed index with this key (SHA256):")
 819         logging.info("%s" % repo_pubkey_fingerprint)
 820
 821         # Create a jar of the index...
 822         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 823         if p.returncode != 0:
 824             logging.critical("Failed to create jar file")
 825             sys.exit(1)
 826
 827         # Sign the index...
 828         args = ['jarsigner', '-keystore', config['keystore'],
 829                 '-storepass:file', config['keystorepassfile'],
 830                 '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 831                 os.path.join(repodir, 'index.jar'), config['repo_keyalias']]
 832         if config['keystore'] == 'NONE':
 833             args += config['smartcardoptions']
 834         else:  # smardcards never use -keypass
 835             args += ['-keypass:file', config['keypassfile']]
 836         p = FDroidPopen(args)
 837         # TODO keypass should be sent via stdin
 838         if p.returncode != 0:
 839             logging.critical("Failed to sign index")
 840             sys.exit(1)
 841
 842     # Copy the repo icon into the repo directory...
 843     icon_dir = os.path.join(repodir, 'icons')
 844     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 845     shutil.copyfile(config['repo_icon'], iconfilename)
 846
 847     # Write a category list in the repo to allow quick access...
 848     catdata = ''
 849     for cat in categories:
 850         catdata += cat + '\n'
 851     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 852     f.write(catdata)
 853     f.close()
 854
 855
 856 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 857
 858     for app in apps:
 859
 860         # Get a list of the apks for this app...
 861         apklist = []
 862         for apk in apks:
 863             if apk['id'] == app['id']:
 864                 apklist.append(apk)
 865
 866         # Sort the apk list into version order...
 867         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 868
 869         if app['Archive Policy']:
 870             keepversions = int(app['Archive Policy'][:-9])
 871         else:
 872             keepversions = defaultkeepversions
 873
 874         if len(apklist) > keepversions:
 875             for apk in apklist[keepversions:]:
 876                 logging.info("Moving " + apk['apkname'] + " to archive")
 877                 shutil.move(os.path.join(repodir, apk['apkname']),
 878                             os.path.join(archivedir, apk['apkname']))
 879                 if 'srcname' in apk:
 880                     shutil.move(os.path.join(repodir, apk['srcname']),
 881                                 os.path.join(archivedir, apk['srcname']))
 882                     # Move GPG signature too...
 883                     sigfile = apk['srcname'] + '.asc'
 884                     sigsrc = os.path.join(repodir, sigfile)
 885                     if os.path.exists(sigsrc):
 886                         shutil.move(sigsrc, os.path.join(archivedir, sigfile))
 887
 888                 archapks.append(apk)
 889                 apks.remove(apk)
 890
 891
 892 config = None
 893 options = None
 894
 895
 896 def main():
 897
 898     global config, options
 899
 900     # Parse command line...
 901     parser = OptionParser()
 902     parser.add_option("-c", "--create-metadata", action="store_true", default=False,
 903                       help="Create skeleton metadata files that are missing")
 904     parser.add_option("--delete-unknown", action="store_true", default=False,
 905                       help="Delete APKs without metadata from the repo")
 906     parser.add_option("-v", "--verbose", action="store_true", default=False,
 907                       help="Spew out even more information than normal")
 908     parser.add_option("-q", "--quiet", action="store_true", default=False,
 909                       help="Restrict output to warnings and errors")
 910     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 911                       help="Report on build data status")
 912     parser.add_option("-i", "--interactive", default=False, action="store_true",
 913                       help="Interactively ask about things that need updating.")
 914     parser.add_option("-I", "--icons", action="store_true", default=False,
 915                       help="Resize all the icons exceeding the max pixel size and exit")
 916     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 917                       help="Specify editor to use in interactive mode. Default " +
 918                       "is /etc/alternatives/editor")
 919     parser.add_option("-w", "--wiki", default=False, action="store_true",
 920                       help="Update the wiki")
 921     parser.add_option("", "--pretty", action="store_true", default=False,
 922                       help="Produce human-readable index.xml")
 923     parser.add_option("--clean", action="store_true", default=False,
 924                       help="Clean update - don't uses caches, reprocess all apks")
 925     (options, args) = parser.parse_args()
 926
 927     config = common.read_config(options)
 928
 929     repodirs = ['repo']
 930     if config['archive_older'] != 0:
 931         repodirs.append('archive')
 932         if not os.path.exists('archive'):
 933             os.mkdir('archive')
 934
 935     if options.icons:
 936         resize_all_icons(repodirs)
 937         sys.exit(0)
 938
 939     # check that icons exist now, rather than fail at the end of `fdroid update`
 940     for k in ['repo_icon', 'archive_icon']:
 941         if k in config:
 942             if not os.path.exists(config[k]):
 943                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
 944                 sys.exit(1)
 945
 946     # Get all apps...
 947     apps = metadata.read_metadata()
 948
 949     # Generate a list of categories...
 950     categories = set()
 951     for app in apps:
 952         categories.update(app['Categories'])
 953
 954     # Read known apks data (will be updated and written back when we've finished)
 955     knownapks = common.KnownApks()
 956
 957     # Gather information about all the apk files in the repo directory, using
 958     # cached data if possible.
 959     apkcachefile = os.path.join('tmp', 'apkcache')
 960     if not options.clean and os.path.exists(apkcachefile):
 961         with open(apkcachefile, 'rb') as cf:
 962             apkcache = pickle.load(cf)
 963     else:
 964         apkcache = {}
 965     cachechanged = False
 966
 967     delete_disabled_builds(apps, apkcache, repodirs)
 968
 969     # Scan all apks in the main repo
 970     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 971     if cc:
 972         cachechanged = True
 973
 974     # Generate warnings for apk's with no metadata (or create skeleton
 975     # metadata files, if requested on the command line)
 976     newmetadata = False
 977     for apk in apks:
 978         found = False
 979         for app in apps:
 980             if app['id'] == apk['id']:
 981                 found = True
 982                 break
 983         if not found:
 984             if options.create_metadata:
 985                 if 'name' not in apk:
 986                     logging.error(apk['id'] + ' does not have a name! Skipping...')
 987                     continue
 988                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 989                 f.write("License:Unknown\n")
 990                 f.write("Web Site:\n")
 991                 f.write("Source Code:\n")
 992                 f.write("Issue Tracker:\n")
 993                 f.write("Summary:" + apk['name'] + "\n")
 994                 f.write("Description:\n")
 995                 f.write(apk['name'] + "\n")
 996                 f.write(".\n")
 997                 f.close()
 998                 logging.info("Generated skeleton metadata for " + apk['id'])
 999                 newmetadata = True
1000             else:
1001                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1002                 if options.delete_unknown:
1003                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1004                     rmf = os.path.join(repodirs[0], apk['apkname'])
1005                     if not os.path.exists(rmf):
1006                         logging.error("Could not find {0} to remove it".format(rmf))
1007                     else:
1008                         os.remove(rmf)
1009                 else:
1010                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1011
1012     # update the metadata with the newly created ones included
1013     if newmetadata:
1014         apps = metadata.read_metadata()
1015
1016     # Scan the archive repo for apks as well
1017     if len(repodirs) > 1:
1018         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1019         if cc:
1020             cachechanged = True
1021     else:
1022         archapks = []
1023
1024     # Some information from the apks needs to be applied up to the application
1025     # level. When doing this, we use the info from the most recent version's apk.
1026     # We deal with figuring out when the app was added and last updated at the
1027     # same time.
1028     for app in apps:
1029         bestver = 0
1030         added = None
1031         lastupdated = None
1032         for apk in apks + archapks:
1033             if apk['id'] == app['id']:
1034                 if apk['versioncode'] > bestver:
1035                     bestver = apk['versioncode']
1036                     bestapk = apk
1037
1038                 if 'added' in apk:
1039                     if not added or apk['added'] < added:
1040                         added = apk['added']
1041                     if not lastupdated or apk['added'] > lastupdated:
1042                         lastupdated = apk['added']
1043
1044         if added:
1045             app['added'] = added
1046         else:
1047             logging.warn("Don't know when " + app['id'] + " was added")
1048         if lastupdated:
1049             app['lastupdated'] = lastupdated
1050         else:
1051             logging.warn("Don't know when " + app['id'] + " was last updated")
1052
1053         if bestver == 0:
1054             if app['Name'] is None:
1055                 app['Name'] = app['id']
1056             app['icon'] = None
1057             logging.warn("Application " + app['id'] + " has no packages")
1058         else:
1059             if app['Name'] is None:
1060                 app['Name'] = bestapk['name']
1061             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1062
1063     # Sort the app list by name, then the web site doesn't have to by default.
1064     # (we had to wait until we'd scanned the apks to do this, because mostly the
1065     # name comes from there!)
1066     apps = sorted(apps, key=lambda app: app['Name'].upper())
1067
1068     if len(repodirs) > 1:
1069         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1070
1071     # Make the index for the main repo...
1072     make_index(apps, apks, repodirs[0], False, categories)
1073
1074     # If there's an archive repo,  make the index for it. We already scanned it
1075     # earlier on.
1076     if len(repodirs) > 1:
1077         make_index(apps, archapks, repodirs[1], True, categories)
1078
1079     if config['update_stats']:
1080
1081         # Update known apks info...
1082         knownapks.writeifchanged()
1083
1084         # Generate latest apps data for widget
1085         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1086             data = ''
1087             for line in file(os.path.join('stats', 'latestapps.txt')):
1088                 appid = line.rstrip()
1089                 data += appid + "\t"
1090                 for app in apps:
1091                     if app['id'] == appid:
1092                         data += app['Name'] + "\t"
1093                         if app['icon'] is not None:
1094                             data += app['icon'] + "\t"
1095                         data += app['License'] + "\n"
1096                         break
1097             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1098             f.write(data)
1099             f.close()
1100
1101     if cachechanged:
1102         with open(apkcachefile, 'wb') as cf:
1103             pickle.dump(apkcache, cf)
1104
1105     # Update the wiki...
1106     if options.wiki:
1107         update_wiki(apps, apks + archapks)
1108
1109     logging.info("Finished.")
1110
1111 if __name__ == "__main__":
1112     main()