fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from PIL import Image
  33 import logging
  34
  35 import common
  36 import metadata
  37 from common import FDroidPopen, SilentPopen
  38 from metadata import MetaDataException
  39
  40
  41 def get_densities():
  42     return ['640', '480', '320', '240', '160', '120']
  43
  44
  45 def dpi_to_px(density):
  46     return (int(density) * 48) / 160
  47
  48
  49 def px_to_dpi(px):
  50     return (int(px) * 160) / 48
  51
  52
  53 def get_icon_dir(repodir, density):
  54     if density is None:
  55         return os.path.join(repodir, "icons")
  56     return os.path.join(repodir, "icons-%s" % density)
  57
  58
  59 def get_icon_dirs(repodir):
  60     for density in get_densities():
  61         yield get_icon_dir(repodir, density)
  62     yield os.path.join(repodir, "icons")
  63
  64
  65 def update_wiki(apps, apks):
  66     """Update the wiki
  67
  68     :param apps: fully populated list of all applications
  69     :param apks: all apks, except...
  70     """
  71     logging.info("Updating wiki")
  72     wikicat = 'Apps'
  73     wikiredircat = 'App Redirects'
  74     import mwclient
  75     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  76                          path=config['wiki_path'])
  77     site.login(config['wiki_user'], config['wiki_password'])
  78     generated_pages = {}
  79     generated_redirects = {}
  80     for app in apps:
  81         wikidata = ''
  82         if app['Disabled']:
  83             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  84         if app['AntiFeatures']:
  85             for af in app['AntiFeatures'].split(','):
  86                 wikidata += '{{AntiFeature|' + af + '}}\n'
  87         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
  88             app['id'],
  89             app['Name'],
  90             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  91             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  92             app['Source Code'],
  93             app['Issue Tracker'],
  94             app['Web Site'],
  95             app['Donate'],
  96             app['FlattrID'],
  97             app['Bitcoin'],
  98             app['Litecoin'],
  99             app['Dogecoin'],
 100             app['License'],
 101             app.get('Requires Root', 'No'))
 102
 103         if app['Provides']:
 104             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 105
 106         wikidata += app['Summary']
 107         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 108
 109         wikidata += "=Description=\n"
 110         wikidata += metadata.description_wiki(app['Description']) + "\n"
 111
 112         wikidata += "=Maintainer Notes=\n"
 113         if 'Maintainer Notes' in app:
 114             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 115         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(app['id'])
 116
 117         # Get a list of all packages for this application...
 118         apklist = []
 119         gotcurrentver = False
 120         cantupdate = False
 121         buildfails = False
 122         for apk in apks:
 123             if apk['id'] == app['id']:
 124                 if str(apk['versioncode']) == app['Current Version Code']:
 125                     gotcurrentver = True
 126                 apklist.append(apk)
 127         # Include ones we can't build, as a special case...
 128         for thisbuild in app['builds']:
 129             if thisbuild['disable']:
 130                 if thisbuild['vercode'] == app['Current Version Code']:
 131                     cantupdate = True
 132                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 133                 apklist.append({'versioncode': int(thisbuild['vercode']),
 134                                 'version': thisbuild['version'],
 135                                 'buildproblem': thisbuild['disable']
 136                                 })
 137             else:
 138                 builtit = False
 139                 for apk in apklist:
 140                     if apk['versioncode'] == int(thisbuild['vercode']):
 141                         builtit = True
 142                         break
 143                 if not builtit:
 144                     buildfails = True
 145                     apklist.append({'versioncode': int(thisbuild['vercode']),
 146                                     'version': thisbuild['version'],
 147                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 148                                     })
 149         if app['Current Version Code'] == '0':
 150             cantupdate = True
 151         # Sort with most recent first...
 152         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 153
 154         wikidata += "=Versions=\n"
 155         if len(apklist) == 0:
 156             wikidata += "We currently have no versions of this app available."
 157         elif not gotcurrentver:
 158             wikidata += "We don't have the current version of this app."
 159         else:
 160             wikidata += "We have the current version of this app."
 161         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 162         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 163         if len(app['No Source Since']) > 0:
 164             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 165         if len(app['Current Version']) > 0:
 166             wikidata += "The current (recommended) version is " + app['Current Version']
 167             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 168         validapks = 0
 169         for apk in apklist:
 170             wikidata += "==" + apk['version'] + "==\n"
 171
 172             if 'buildproblem' in apk:
 173                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 174             else:
 175                 validapks += 1
 176                 wikidata += "This version is built and signed by "
 177                 if 'srcname' in apk:
 178                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 179                 else:
 180                     wikidata += "the original developer.\n\n"
 181             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 182
 183         wikidata += '\n[[Category:' + wikicat + ']]\n'
 184         if len(app['No Source Since']) > 0:
 185             wikidata += '\n[[Category:Apps missing source code]]\n'
 186         if validapks == 0 and not app['Disabled']:
 187             wikidata += '\n[[Category:Apps with no packages]]\n'
 188         if cantupdate and not app['Disabled']:
 189             wikidata += "\n[[Category:Apps we can't update]]\n"
 190         if buildfails and not app['Disabled']:
 191             wikidata += "\n[[Category:Apps with failing builds]]\n"
 192         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 193             wikidata += '\n[[Category:Apps to Update]]\n'
 194         if app['Disabled']:
 195             wikidata += '\n[[Category:Apps that are disabled]]\n'
 196         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 197             wikidata += '\n[[Category:Apps with no update check]]\n'
 198         for appcat in app['Categories']:
 199             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 200
 201         # We can't have underscores in the page name, even if they're in
 202         # the package ID, because MediaWiki messes with them...
 203         pagename = app['id'].replace('_', ' ')
 204
 205         # Drop a trailing newline, because mediawiki is going to drop it anyway
 206         # and it we don't we'll think the page has changed when it hasn't...
 207         if wikidata.endswith('\n'):
 208             wikidata = wikidata[:-1]
 209
 210         generated_pages[pagename] = wikidata
 211
 212         # Make a redirect from the name to the ID too, unless there's
 213         # already an existing page with the name and it isn't a redirect.
 214         noclobber = False
 215         apppagename = app['Name'].replace('_', ' ')
 216         apppagename = apppagename.replace('{', '')
 217         apppagename = apppagename.replace('}', ' ')
 218         apppagename = apppagename.replace(':', ' ')
 219         # Drop double spaces caused mostly by replacing ':' above
 220         apppagename = apppagename.replace('  ', ' ')
 221         for expagename in site.allpages(prefix=apppagename,
 222                                         filterredir='nonredirects',
 223                                         generator=False):
 224             if expagename == apppagename:
 225                 noclobber = True
 226         # Another reason not to make the redirect page is if the app name
 227         # is the same as it's ID, because that will overwrite the real page
 228         # with an redirect to itself! (Although it seems like an odd
 229         # scenario this happens a lot, e.g. where there is metadata but no
 230         # builds or binaries to extract a name from.
 231         if apppagename == pagename:
 232             noclobber = True
 233         if not noclobber:
 234             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 235
 236     for tcat, genp in [(wikicat, generated_pages),
 237                        (wikiredircat, generated_redirects)]:
 238         catpages = site.Pages['Category:' + tcat]
 239         existingpages = []
 240         for page in catpages:
 241             existingpages.append(page.name)
 242             if page.name in genp:
 243                 pagetxt = page.edit()
 244                 if pagetxt != genp[page.name]:
 245                     logging.debug("Updating modified page " + page.name)
 246                     page.save(genp[page.name], summary='Auto-updated')
 247                 else:
 248                     logging.debug("Page " + page.name + " is unchanged")
 249             else:
 250                 logging.warn("Deleting page " + page.name)
 251                 page.delete('No longer published')
 252         for pagename, text in genp.items():
 253             logging.debug("Checking " + pagename)
 254             if pagename not in existingpages:
 255                 logging.debug("Creating page " + pagename)
 256                 try:
 257                     newpage = site.Pages[pagename]
 258                     newpage.save(text, summary='Auto-created')
 259                 except:
 260                     logging.error("...FAILED to create page")
 261
 262     # Purge server cache to ensure counts are up to date
 263     site.pages['Repository Maintenance'].purge()
 264
 265
 266 def delete_disabled_builds(apps, apkcache, repodirs):
 267     """Delete disabled build outputs.
 268
 269     :param apps: list of all applications, as per metadata.read_metadata
 270     :param apkcache: current apk cache information
 271     :param repodirs: the repo directories to process
 272     """
 273     for app in apps:
 274         for build in app['builds']:
 275             if build['disable']:
 276                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 277                 for repodir in repodirs:
 278                     apkpath = os.path.join(repodir, apkfilename)
 279                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 280                     for name in [apkpath, srcpath]:
 281                         if os.path.exists(name):
 282                             logging.warn("Deleting disabled build output " + apkfilename)
 283                             os.remove(name)
 284                 if apkfilename in apkcache:
 285                     del apkcache[apkfilename]
 286
 287
 288 def resize_icon(iconpath, density):
 289
 290     if not os.path.isfile(iconpath):
 291         return
 292
 293     try:
 294         im = Image.open(iconpath)
 295         size = dpi_to_px(density)
 296
 297         if any(length > size for length in im.size):
 298             oldsize = im.size
 299             im.thumbnail((size, size), Image.ANTIALIAS)
 300             logging.debug("%s was too large at %s - new size is %s" % (
 301                 iconpath, oldsize, im.size))
 302             im.save(iconpath, "PNG")
 303
 304     except Exception, e:
 305         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 306
 307
 308 def resize_all_icons(repodirs):
 309     """Resize all icons that exceed the max size
 310
 311     :param repodirs: the repo directories to process
 312     """
 313     for repodir in repodirs:
 314         for density in get_densities():
 315             icon_dir = get_icon_dir(repodir, density)
 316             icon_glob = os.path.join(icon_dir, '*.png')
 317             for iconpath in glob.glob(icon_glob):
 318                 resize_icon(iconpath, density)
 319
 320
 321 def scan_apks(apps, apkcache, repodir, knownapks):
 322     """Scan the apks in the given repo directory.
 323
 324     This also extracts the icons.
 325
 326     :param apps: list of all applications, as per metadata.read_metadata
 327     :param apkcache: current apk cache information
 328     :param repodir: repo directory to scan
 329     :param knownapks: known apks info
 330     :returns: (apks, cachechanged) where apks is a list of apk information,
 331               and cachechanged is True if the apkcache got changed.
 332     """
 333
 334     cachechanged = False
 335
 336     icon_dirs = get_icon_dirs(repodir)
 337     for icon_dir in icon_dirs:
 338         if os.path.exists(icon_dir):
 339             if options.clean:
 340                 shutil.rmtree(icon_dir)
 341                 os.makedirs(icon_dir)
 342         else:
 343             os.makedirs(icon_dir)
 344
 345     apks = []
 346     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 347     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 348     vername_pat = re.compile(".*versionName='([^']*)'.*")
 349     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 350     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 351     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 352     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 353     string_pat = re.compile(".*'([^']*)'.*")
 354     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 355
 356         apkfilename = apkfile[len(repodir) + 1:]
 357         if ' ' in apkfilename:
 358             logging.error("No spaces in APK filenames!")
 359             sys.exit(1)
 360
 361         if apkfilename in apkcache:
 362             logging.debug("Reading " + apkfilename + " from cache")
 363             thisinfo = apkcache[apkfilename]
 364
 365         else:
 366             logging.debug("Processing " + apkfilename)
 367             thisinfo = {}
 368             thisinfo['apkname'] = apkfilename
 369             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 370             if os.path.exists(os.path.join(repodir, srcfilename)):
 371                 thisinfo['srcname'] = srcfilename
 372             thisinfo['size'] = os.path.getsize(apkfile)
 373             thisinfo['permissions'] = []
 374             thisinfo['features'] = []
 375             thisinfo['icons_src'] = {}
 376             thisinfo['icons'] = {}
 377             p = SilentPopen([os.path.join(config['sdk_path'], 'build-tools',
 378                                           config['build_tools'], 'aapt'),
 379                              'dump', 'badging', apkfile])
 380             if p.returncode != 0:
 381                 if options.delete_unknown:
 382                     if os.path.exists(apkfile):
 383                         logging.error("Failed to get apk information, deleting " + apkfile)
 384                         os.remove(apkfile)
 385                     else:
 386                         logging.error("Could not find {0} to remove it".format(apkfile))
 387                 else:
 388                     logging.error("Failed to get apk information, skipping " + apkfile)
 389                 continue
 390             for line in p.stdout.splitlines():
 391                 if line.startswith("package:"):
 392                     try:
 393                         thisinfo['id'] = re.match(name_pat, line).group(1)
 394                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 395                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 396                     except Exception, e:
 397                         logging.info("Package matching failed: " + str(e))
 398                         logging.info("Line was: " + line)
 399                         sys.exit(1)
 400                 elif line.startswith("application:"):
 401                     thisinfo['name'] = re.match(label_pat, line).group(1)
 402                     # Keep path to non-dpi icon in case we need it
 403                     match = re.match(icon_pat_nodpi, line)
 404                     if match:
 405                         thisinfo['icons_src']['-1'] = match.group(1)
 406                 elif line.startswith("launchable-activity:"):
 407                     # Only use launchable-activity as fallback to application
 408                     if not thisinfo['name']:
 409                         thisinfo['name'] = re.match(label_pat, line).group(1)
 410                     if '-1' not in thisinfo['icons_src']:
 411                         match = re.match(icon_pat_nodpi, line)
 412                         if match:
 413                             thisinfo['icons_src']['-1'] = match.group(1)
 414                 elif line.startswith("application-icon-"):
 415                     match = re.match(icon_pat, line)
 416                     if match:
 417                         density = match.group(1)
 418                         path = match.group(2)
 419                         thisinfo['icons_src'][density] = path
 420                 elif line.startswith("sdkVersion:"):
 421                     m = re.match(sdkversion_pat, line)
 422                     if m is None:
 423                         logging.error(line.replace('sdkVersion:', '')
 424                                       + ' is not a valid minSdkVersion!')
 425                     else:
 426                         thisinfo['sdkversion'] = m.group(1)
 427                 elif line.startswith("maxSdkVersion:"):
 428                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 429                 elif line.startswith("native-code:"):
 430                     thisinfo['nativecode'] = []
 431                     for arch in line[13:].split(' '):
 432                         thisinfo['nativecode'].append(arch[1:-1])
 433                 elif line.startswith("uses-permission:"):
 434                     perm = re.match(string_pat, line).group(1)
 435                     if perm.startswith("android.permission."):
 436                         perm = perm[19:]
 437                     thisinfo['permissions'].append(perm)
 438                 elif line.startswith("uses-feature:"):
 439                     perm = re.match(string_pat, line).group(1)
 440                     # Filter out this, it's only added with the latest SDK tools and
 441                     # causes problems for lots of apps.
 442                     if perm != "android.hardware.screen.portrait" \
 443                             and perm != "android.hardware.screen.landscape":
 444                         if perm.startswith("android.feature."):
 445                             perm = perm[16:]
 446                         thisinfo['features'].append(perm)
 447
 448             if 'sdkversion' not in thisinfo:
 449                 logging.warn("no SDK version information found")
 450                 thisinfo['sdkversion'] = 0
 451
 452             # Check for debuggable apks...
 453             if common.isApkDebuggable(apkfile, config):
 454                 logging.warn('{0} is set to android:debuggable="true"!'.format(apkfile))
 455
 456             # Calculate the sha256...
 457             sha = hashlib.sha256()
 458             with open(apkfile, 'rb') as f:
 459                 while True:
 460                     t = f.read(1024)
 461                     if len(t) == 0:
 462                         break
 463                     sha.update(t)
 464                 thisinfo['sha256'] = sha.hexdigest()
 465
 466             # Get the signature (or md5 of, to be precise)...
 467             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 468             if not os.path.exists(getsig_dir + "/getsig.class"):
 469                 logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
 470                 sys.exit(1)
 471             p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 472                              'getsig', os.path.join(os.getcwd(), apkfile)])
 473             if p.returncode != 0 or not p.stdout.startswith('Result:'):
 474                 logging.critical("Failed to get apk signature")
 475                 sys.exit(1)
 476             thisinfo['sig'] = p.stdout[7:].strip()
 477
 478             apk = zipfile.ZipFile(apkfile, 'r')
 479
 480             iconfilename = "%s.%s.png" % (
 481                 thisinfo['id'],
 482                 thisinfo['versioncode'])
 483
 484             # Extract the icon file...
 485             densities = get_densities()
 486             empty_densities = []
 487             for density in densities:
 488                 if density not in thisinfo['icons_src']:
 489                     empty_densities.append(density)
 490                     continue
 491                 iconsrc = thisinfo['icons_src'][density]
 492                 icon_dir = get_icon_dir(repodir, density)
 493                 icondest = os.path.join(icon_dir, iconfilename)
 494
 495                 try:
 496                     iconfile = open(icondest, 'wb')
 497                     iconfile.write(apk.read(iconsrc))
 498                     iconfile.close()
 499                     thisinfo['icons'][density] = iconfilename
 500
 501                 except:
 502                     logging.warn("Error retrieving icon file")
 503                     del thisinfo['icons'][density]
 504                     del thisinfo['icons_src'][density]
 505                     empty_densities.append(density)
 506
 507             if '-1' in thisinfo['icons_src']:
 508                 iconsrc = thisinfo['icons_src']['-1']
 509                 iconpath = os.path.join(
 510                     get_icon_dir(repodir, None), iconfilename)
 511                 iconfile = open(iconpath, 'wb')
 512                 iconfile.write(apk.read(iconsrc))
 513                 iconfile.close()
 514                 try:
 515                     im = Image.open(iconpath)
 516                     dpi = px_to_dpi(im.size[0])
 517                     for density in densities:
 518                         if density in thisinfo['icons']:
 519                             break
 520                         if density == densities[-1] or dpi >= int(density):
 521                             thisinfo['icons'][density] = iconfilename
 522                             shutil.move(iconpath,
 523                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 524                             empty_densities.remove(density)
 525                             break
 526                 except Exception, e:
 527                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 528
 529             if thisinfo['icons']:
 530                 thisinfo['icon'] = iconfilename
 531
 532             apk.close()
 533
 534             # First try resizing down to not lose quality
 535             last_density = None
 536             for density in densities:
 537                 if density not in empty_densities:
 538                     last_density = density
 539                     continue
 540                 if last_density is None:
 541                     continue
 542                 logging.info("Density %s not available, resizing down from %s"
 543                              % (density, last_density))
 544
 545                 last_iconpath = os.path.join(
 546                     get_icon_dir(repodir, last_density), iconfilename)
 547                 iconpath = os.path.join(
 548                     get_icon_dir(repodir, density), iconfilename)
 549                 try:
 550                     im = Image.open(last_iconpath)
 551                 except:
 552                     logging.warn("Invalid image file at %s" % last_iconpath)
 553                     continue
 554
 555                 size = dpi_to_px(density)
 556
 557                 im.thumbnail((size, size), Image.ANTIALIAS)
 558                 im.save(iconpath, "PNG")
 559                 empty_densities.remove(density)
 560
 561             # Then just copy from the highest resolution available
 562             last_density = None
 563             for density in reversed(densities):
 564                 if density not in empty_densities:
 565                     last_density = density
 566                     continue
 567                 if last_density is None:
 568                     continue
 569                 logging.info("Density %s not available, copying from lower density %s"
 570                              % (density, last_density))
 571
 572                 shutil.copyfile(
 573                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 574                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 575
 576                 empty_densities.remove(density)
 577
 578             for density in densities:
 579                 icon_dir = get_icon_dir(repodir, density)
 580                 icondest = os.path.join(icon_dir, iconfilename)
 581                 resize_icon(icondest, density)
 582
 583             # Copy from icons-mdpi to icons since mdpi is the baseline density
 584             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 585             if os.path.isfile(baseline):
 586                 shutil.copyfile(baseline,
 587                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 588
 589             # Record in known apks, getting the added date at the same time..
 590             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 591             if added:
 592                 thisinfo['added'] = added
 593
 594             apkcache[apkfilename] = thisinfo
 595             cachechanged = True
 596
 597         apks.append(thisinfo)
 598
 599     return apks, cachechanged
 600
 601
 602 repo_pubkey_fingerprint = None
 603
 604
 605 def make_index(apps, apks, repodir, archive, categories):
 606     """Make a repo index.
 607
 608     :param apps: fully populated apps list
 609     :param apks: full populated apks list
 610     :param repodir: the repo directory
 611     :param archive: True if this is the archive repo, False if it's the
 612                     main one.
 613     :param categories: list of categories
 614     """
 615
 616     doc = Document()
 617
 618     def addElement(name, value, doc, parent):
 619         el = doc.createElement(name)
 620         el.appendChild(doc.createTextNode(value))
 621         parent.appendChild(el)
 622
 623     def addElementCDATA(name, value, doc, parent):
 624         el = doc.createElement(name)
 625         el.appendChild(doc.createCDATASection(value))
 626         parent.appendChild(el)
 627
 628     root = doc.createElement("fdroid")
 629     doc.appendChild(root)
 630
 631     repoel = doc.createElement("repo")
 632
 633     if archive:
 634         repoel.setAttribute("name", config['archive_name'])
 635         if config['repo_maxage'] != 0:
 636             repoel.setAttribute("maxage", str(config['repo_maxage']))
 637         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 638         repoel.setAttribute("url", config['archive_url'])
 639         addElement('description', config['archive_description'], doc, repoel)
 640
 641     else:
 642         repoel.setAttribute("name", config['repo_name'])
 643         if config['repo_maxage'] != 0:
 644             repoel.setAttribute("maxage", str(config['repo_maxage']))
 645         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 646         repoel.setAttribute("url", config['repo_url'])
 647         addElement('description', config['repo_description'], doc, repoel)
 648
 649     repoel.setAttribute("version", "12")
 650     repoel.setAttribute("timestamp", str(int(time.time())))
 651
 652     if 'repo_keyalias' in config:
 653
 654         # Generate a certificate fingerprint the same way keytool does it
 655         # (but with slightly different formatting)
 656         def cert_fingerprint(data):
 657             digest = hashlib.sha256(data).digest()
 658             ret = []
 659             ret.append(' '.join("%02X" % ord(b) for b in digest))
 660             return " ".join(ret)
 661
 662         def extract_pubkey():
 663             p = FDroidPopen(['keytool', '-exportcert',
 664                              '-alias', config['repo_keyalias'],
 665                              '-keystore', config['keystore'],
 666                              '-storepass:file', config['keystorepassfile']]
 667                             + config['smartcardoptions'], output=False)
 668             if p.returncode != 0:
 669                 msg = "Failed to get repo pubkey!"
 670                 if config['keystore'] == 'NONE':
 671                     msg += ' Is your crypto smartcard plugged in?'
 672                 logging.critical(msg)
 673                 sys.exit(1)
 674             global repo_pubkey_fingerprint
 675             repo_pubkey_fingerprint = cert_fingerprint(p.stdout)
 676             return "".join("%02x" % ord(b) for b in p.stdout)
 677
 678         repoel.setAttribute("pubkey", extract_pubkey())
 679
 680     root.appendChild(repoel)
 681
 682     for app in apps:
 683
 684         if app['Disabled'] is not None:
 685             continue
 686
 687         # Get a list of the apks for this app...
 688         apklist = []
 689         for apk in apks:
 690             if apk['id'] == app['id']:
 691                 apklist.append(apk)
 692
 693         if len(apklist) == 0:
 694             continue
 695
 696         apel = doc.createElement("application")
 697         apel.setAttribute("id", app['id'])
 698         root.appendChild(apel)
 699
 700         addElement('id', app['id'], doc, apel)
 701         if 'added' in app:
 702             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 703         if 'lastupdated' in app:
 704             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 705         addElement('name', app['Name'], doc, apel)
 706         addElement('summary', app['Summary'], doc, apel)
 707         if app['icon']:
 708             addElement('icon', app['icon'], doc, apel)
 709
 710         def linkres(link):
 711             for app in apps:
 712                 if app['id'] == link:
 713                     return ("fdroid.app:" + link, app['Name'])
 714             raise MetaDataException("Cannot resolve app id " + link)
 715         addElement('desc',
 716                    metadata.description_html(app['Description'], linkres),
 717                    doc, apel)
 718         addElement('license', app['License'], doc, apel)
 719         if 'Categories' in app:
 720             addElement('categories', ','.join(app["Categories"]), doc, apel)
 721             # We put the first (primary) category in LAST, which will have
 722             # the desired effect of making clients that only understand one
 723             # category see that one.
 724             addElement('category', app["Categories"][0], doc, apel)
 725         addElement('web', app['Web Site'], doc, apel)
 726         addElement('source', app['Source Code'], doc, apel)
 727         addElement('tracker', app['Issue Tracker'], doc, apel)
 728         if app['Donate']:
 729             addElement('donate', app['Donate'], doc, apel)
 730         if app['Bitcoin']:
 731             addElement('bitcoin', app['Bitcoin'], doc, apel)
 732         if app['Litecoin']:
 733             addElement('litecoin', app['Litecoin'], doc, apel)
 734         if app['Dogecoin']:
 735             addElement('dogecoin', app['Dogecoin'], doc, apel)
 736         if app['FlattrID']:
 737             addElement('flattr', app['FlattrID'], doc, apel)
 738
 739         # These elements actually refer to the current version (i.e. which
 740         # one is recommended. They are historically mis-named, and need
 741         # changing, but stay like this for now to support existing clients.
 742         addElement('marketversion', app['Current Version'], doc, apel)
 743         addElement('marketvercode', app['Current Version Code'], doc, apel)
 744
 745         if app['AntiFeatures']:
 746             af = app['AntiFeatures'].split(',')
 747             # TODO: Temporarily not including UpstreamNonFree in the index,
 748             # because current F-Droid clients do not understand it, and also
 749             # look ugly when they encounter an unknown antifeature. This
 750             # filtering can be removed in time...
 751             if 'UpstreamNonFree' in af:
 752                 af.remove('UpstreamNonFree')
 753             if af:
 754                 addElement('antifeatures', ','.join(af), doc, apel)
 755         if app['Provides']:
 756             pv = app['Provides'].split(',')
 757             addElement('provides', ','.join(pv), doc, apel)
 758         if app['Requires Root']:
 759             addElement('requirements', 'root', doc, apel)
 760
 761         # Sort the apk list into version order, just so the web site
 762         # doesn't have to do any work by default...
 763         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 764
 765         # Check for duplicates - they will make the client unhappy...
 766         for i in range(len(apklist) - 1):
 767             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 768                 logging.critical("duplicate versions: '%s' - '%s'" % (
 769                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 770                 sys.exit(1)
 771
 772         for apk in apklist:
 773             apkel = doc.createElement("package")
 774             apel.appendChild(apkel)
 775             addElement('version', apk['version'], doc, apkel)
 776             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 777             addElement('apkname', apk['apkname'], doc, apkel)
 778             if 'srcname' in apk:
 779                 addElement('srcname', apk['srcname'], doc, apkel)
 780             for hash_type in ['sha256']:
 781                 if hash_type not in apk:
 782                     continue
 783                 hashel = doc.createElement("hash")
 784                 hashel.setAttribute("type", hash_type)
 785                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 786                 apkel.appendChild(hashel)
 787             addElement('sig', apk['sig'], doc, apkel)
 788             addElement('size', str(apk['size']), doc, apkel)
 789             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 790             if 'maxsdkversion' in apk:
 791                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 792             if 'added' in apk:
 793                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 794             if app['Requires Root']:
 795                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 796                     apk['permissions'].append('ACCESS_SUPERUSER')
 797
 798             if len(apk['permissions']) > 0:
 799                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 800             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 801                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 802             if len(apk['features']) > 0:
 803                 addElement('features', ','.join(apk['features']), doc, apkel)
 804
 805     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 806     if options.pretty:
 807         output = doc.toprettyxml()
 808     else:
 809         output = doc.toxml()
 810     of.write(output)
 811     of.close()
 812
 813     if 'repo_keyalias' in config:
 814
 815         logging.info("Creating signed index with this key (SHA256):")
 816         logging.info("%s" % repo_pubkey_fingerprint)
 817
 818         # Create a jar of the index...
 819         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 820         if p.returncode != 0:
 821             logging.critical("Failed to create jar file")
 822             sys.exit(1)
 823
 824         # Sign the index...
 825         args = ['jarsigner', '-keystore', config['keystore'],
 826                 '-storepass:file', config['keystorepassfile'],
 827                 '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 828                 os.path.join(repodir, 'index.jar'), config['repo_keyalias']]
 829         if config['keystore'] == 'NONE':
 830             args += config['smartcardoptions']
 831         else:  # smardcards never use -keypass
 832             args += ['-keypass:file', config['keypassfile']]
 833         p = FDroidPopen(args)
 834         # TODO keypass should be sent via stdin
 835         if p.returncode != 0:
 836             logging.info("Failed to sign index")
 837             sys.exit(1)
 838
 839     # Copy the repo icon into the repo directory...
 840     icon_dir = os.path.join(repodir, 'icons')
 841     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 842     shutil.copyfile(config['repo_icon'], iconfilename)
 843
 844     # Write a category list in the repo to allow quick access...
 845     catdata = ''
 846     for cat in categories:
 847         catdata += cat + '\n'
 848     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 849     f.write(catdata)
 850     f.close()
 851
 852
 853 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 854
 855     for app in apps:
 856
 857         # Get a list of the apks for this app...
 858         apklist = []
 859         for apk in apks:
 860             if apk['id'] == app['id']:
 861                 apklist.append(apk)
 862
 863         # Sort the apk list into version order...
 864         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 865
 866         if app['Archive Policy']:
 867             keepversions = int(app['Archive Policy'][:-9])
 868         else:
 869             keepversions = defaultkeepversions
 870
 871         if len(apklist) > keepversions:
 872             for apk in apklist[keepversions:]:
 873                 logging.info("Moving " + apk['apkname'] + " to archive")
 874                 shutil.move(os.path.join(repodir, apk['apkname']),
 875                             os.path.join(archivedir, apk['apkname']))
 876                 if 'srcname' in apk:
 877                     shutil.move(os.path.join(repodir, apk['srcname']),
 878                                 os.path.join(archivedir, apk['srcname']))
 879                     # Move GPG signature too...
 880                     sigfile = apk['srcname'] + '.asc'
 881                     sigsrc = os.path.join(repodir, sigfile)
 882                     if os.path.exists(sigsrc):
 883                         shutil.move(sigsrc, os.path.join(archivedir, sigfile))
 884
 885                 archapks.append(apk)
 886                 apks.remove(apk)
 887
 888
 889 config = None
 890 options = None
 891
 892
 893 def main():
 894
 895     global config, options
 896
 897     # Parse command line...
 898     parser = OptionParser()
 899     parser.add_option("-c", "--create-metadata", action="store_true", default=False,
 900                       help="Create skeleton metadata files that are missing")
 901     parser.add_option("--delete-unknown", action="store_true", default=False,
 902                       help="Delete APKs without metadata from the repo")
 903     parser.add_option("-v", "--verbose", action="store_true", default=False,
 904                       help="Spew out even more information than normal")
 905     parser.add_option("-q", "--quiet", action="store_true", default=False,
 906                       help="Restrict output to warnings and errors")
 907     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 908                       help="Report on build data status")
 909     parser.add_option("-i", "--interactive", default=False, action="store_true",
 910                       help="Interactively ask about things that need updating.")
 911     parser.add_option("-I", "--icons", action="store_true", default=False,
 912                       help="Resize all the icons exceeding the max pixel size and exit")
 913     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 914                       help="Specify editor to use in interactive mode. Default " +
 915                       "is /etc/alternatives/editor")
 916     parser.add_option("-w", "--wiki", default=False, action="store_true",
 917                       help="Update the wiki")
 918     parser.add_option("", "--pretty", action="store_true", default=False,
 919                       help="Produce human-readable index.xml")
 920     parser.add_option("--clean", action="store_true", default=False,
 921                       help="Clean update - don't uses caches, reprocess all apks")
 922     (options, args) = parser.parse_args()
 923
 924     config = common.read_config(options)
 925
 926     repodirs = ['repo']
 927     if config['archive_older'] != 0:
 928         repodirs.append('archive')
 929         if not os.path.exists('archive'):
 930             os.mkdir('archive')
 931
 932     if options.icons:
 933         resize_all_icons(repodirs)
 934         sys.exit(0)
 935
 936     # check that icons exist now, rather than fail at the end of `fdroid update`
 937     for k in ['repo_icon', 'archive_icon']:
 938         if k in config:
 939             if not os.path.exists(config[k]):
 940                 logging.error(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
 941                 sys.exit(1)
 942
 943     # Get all apps...
 944     apps = metadata.read_metadata()
 945
 946     # Generate a list of categories...
 947     categories = set()
 948     for app in apps:
 949         categories.update(app['Categories'])
 950
 951     # Read known apks data (will be updated and written back when we've finished)
 952     knownapks = common.KnownApks()
 953
 954     # Gather information about all the apk files in the repo directory, using
 955     # cached data if possible.
 956     apkcachefile = os.path.join('tmp', 'apkcache')
 957     if not options.clean and os.path.exists(apkcachefile):
 958         with open(apkcachefile, 'rb') as cf:
 959             apkcache = pickle.load(cf)
 960     else:
 961         apkcache = {}
 962     cachechanged = False
 963
 964     delete_disabled_builds(apps, apkcache, repodirs)
 965
 966     # Scan all apks in the main repo
 967     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 968     if cc:
 969         cachechanged = True
 970
 971     # Generate warnings for apk's with no metadata (or create skeleton
 972     # metadata files, if requested on the command line)
 973     newmetadata = False
 974     for apk in apks:
 975         found = False
 976         for app in apps:
 977             if app['id'] == apk['id']:
 978                 found = True
 979                 break
 980         if not found:
 981             if options.create_metadata:
 982                 if 'name' not in apk:
 983                     logging.error(apk['id'] + ' does not have a name! Skipping...')
 984                     continue
 985                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 986                 f.write("License:Unknown\n")
 987                 f.write("Web Site:\n")
 988                 f.write("Source Code:\n")
 989                 f.write("Issue Tracker:\n")
 990                 f.write("Summary:" + apk['name'] + "\n")
 991                 f.write("Description:\n")
 992                 f.write(apk['name'] + "\n")
 993                 f.write(".\n")
 994                 f.close()
 995                 logging.info("Generated skeleton metadata for " + apk['id'])
 996                 newmetadata = True
 997             else:
 998                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
 999                 if options.delete_unknown:
1000                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1001                     rmf = os.path.join(repodirs[0], apk['apkname'])
1002                     if not os.path.exists(rmf):
1003                         logging.error("Could not find {0} to remove it".format(rmf))
1004                     else:
1005                         os.remove(rmf)
1006                 else:
1007                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1008
1009     # update the metadata with the newly created ones included
1010     if newmetadata:
1011         apps = metadata.read_metadata()
1012
1013     # Scan the archive repo for apks as well
1014     if len(repodirs) > 1:
1015         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1016         if cc:
1017             cachechanged = True
1018     else:
1019         archapks = []
1020
1021     # Some information from the apks needs to be applied up to the application
1022     # level. When doing this, we use the info from the most recent version's apk.
1023     # We deal with figuring out when the app was added and last updated at the
1024     # same time.
1025     for app in apps:
1026         bestver = 0
1027         added = None
1028         lastupdated = None
1029         for apk in apks + archapks:
1030             if apk['id'] == app['id']:
1031                 if apk['versioncode'] > bestver:
1032                     bestver = apk['versioncode']
1033                     bestapk = apk
1034
1035                 if 'added' in apk:
1036                     if not added or apk['added'] < added:
1037                         added = apk['added']
1038                     if not lastupdated or apk['added'] > lastupdated:
1039                         lastupdated = apk['added']
1040
1041         if added:
1042             app['added'] = added
1043         else:
1044             logging.warn("Don't know when " + app['id'] + " was added")
1045         if lastupdated:
1046             app['lastupdated'] = lastupdated
1047         else:
1048             logging.warn("Don't know when " + app['id'] + " was last updated")
1049
1050         if bestver == 0:
1051             if app['Name'] is None:
1052                 app['Name'] = app['id']
1053             app['icon'] = None
1054             logging.warn("Application " + app['id'] + " has no packages")
1055         else:
1056             if app['Name'] is None:
1057                 app['Name'] = bestapk['name']
1058             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1059
1060     # Sort the app list by name, then the web site doesn't have to by default.
1061     # (we had to wait until we'd scanned the apks to do this, because mostly the
1062     # name comes from there!)
1063     apps = sorted(apps, key=lambda app: app['Name'].upper())
1064
1065     if len(repodirs) > 1:
1066         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1067
1068     # Make the index for the main repo...
1069     make_index(apps, apks, repodirs[0], False, categories)
1070
1071     # If there's an archive repo,  make the index for it. We already scanned it
1072     # earlier on.
1073     if len(repodirs) > 1:
1074         make_index(apps, archapks, repodirs[1], True, categories)
1075
1076     if config['update_stats']:
1077
1078         # Update known apks info...
1079         knownapks.writeifchanged()
1080
1081         # Generate latest apps data for widget
1082         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1083             data = ''
1084             for line in file(os.path.join('stats', 'latestapps.txt')):
1085                 appid = line.rstrip()
1086                 data += appid + "\t"
1087                 for app in apps:
1088                     if app['id'] == appid:
1089                         data += app['Name'] + "\t"
1090                         if app['icon'] is not None:
1091                             data += app['icon'] + "\t"
1092                         data += app['License'] + "\n"
1093                         break
1094             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1095             f.write(data)
1096             f.close()
1097
1098     if cachechanged:
1099         with open(apkcachefile, 'wb') as cf:
1100             pickle.dump(apkcache, cf)
1101
1102     # Update the wiki...
1103     if options.wiki:
1104         update_wiki(apps, apks + archapks)
1105
1106     logging.info("Finished.")
1107
1108 if __name__ == "__main__":
1109     main()