fdroidserver/update.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 #
   4 # update.py - part of the FDroid server tools
   5 # Copyright (C) 2010-2013, Ciaran Gultnieks, ciaran@ciarang.com
   6 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
   7 #
   8 # This program is free software: you can redistribute it and/or modify
   9 # it under the terms of the GNU Affero General Public License as published by
  10 # the Free Software Foundation, either version 3 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Affero General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Affero General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 import sys
  22 import os
  23 import shutil
  24 import glob
  25 import re
  26 import zipfile
  27 import hashlib
  28 import pickle
  29 from xml.dom.minidom import Document
  30 from optparse import OptionParser
  31 import time
  32 from PIL import Image
  33 import logging
  34
  35 import common
  36 import metadata
  37 from common import FDroidPopen
  38 from metadata import MetaDataException
  39
  40
  41 def get_densities():
  42     return ['640', '480', '320', '240', '160', '120']
  43
  44
  45 def dpi_to_px(density):
  46     return (int(density) * 48) / 160
  47
  48
  49 def px_to_dpi(px):
  50     return (int(px) * 160) / 48
  51
  52
  53 def get_icon_dir(repodir, density):
  54     if density is None:
  55         return os.path.join(repodir, "icons")
  56     return os.path.join(repodir, "icons-%s" % density)
  57
  58
  59 def get_icon_dirs(repodir):
  60     for density in get_densities():
  61         yield get_icon_dir(repodir, density)
  62     yield os.path.join(repodir, "icons")
  63
  64
  65 def update_wiki(apps, apks):
  66     """Update the wiki
  67
  68     :param apps: fully populated list of all applications
  69     :param apks: all apks, except...
  70     """
  71     logging.info("Updating wiki")
  72     wikicat = 'Apps'
  73     wikiredircat = 'App Redirects'
  74     import mwclient
  75     site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
  76                          path=config['wiki_path'])
  77     site.login(config['wiki_user'], config['wiki_password'])
  78     generated_pages = {}
  79     generated_redirects = {}
  80     for app in apps:
  81         wikidata = ''
  82         if app['Disabled']:
  83             wikidata += '{{Disabled|' + app['Disabled'] + '}}\n'
  84         if app['AntiFeatures']:
  85             for af in app['AntiFeatures'].split(','):
  86                 wikidata += '{{AntiFeature|' + af + '}}\n'
  87         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|dogecoin=%s|license=%s|root=%s}}\n' % (
  88             app['id'],
  89             app['Name'],
  90             time.strftime('%Y-%m-%d', app['added']) if 'added' in app else '',
  91             time.strftime('%Y-%m-%d', app['lastupdated']) if 'lastupdated' in app else '',
  92             app['Source Code'],
  93             app['Issue Tracker'],
  94             app['Web Site'],
  95             app['Donate'],
  96             app['FlattrID'],
  97             app['Bitcoin'],
  98             app['Litecoin'],
  99             app['Dogecoin'],
 100             app['License'],
 101             app.get('Requires Root', 'No'))
 102
 103         if app['Provides']:
 104             wikidata += "This app provides: %s" % ', '.join(app['Summary'].split(','))
 105
 106         wikidata += app['Summary']
 107         wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
 108
 109         wikidata += "=Description=\n"
 110         wikidata += metadata.description_wiki(app['Description']) + "\n"
 111
 112         wikidata += "=Maintainer Notes=\n"
 113         if 'Maintainer Notes' in app:
 114             wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
 115         wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(app['id'])
 116
 117         # Get a list of all packages for this application...
 118         apklist = []
 119         gotcurrentver = False
 120         cantupdate = False
 121         buildfails = False
 122         for apk in apks:
 123             if apk['id'] == app['id']:
 124                 if str(apk['versioncode']) == app['Current Version Code']:
 125                     gotcurrentver = True
 126                 apklist.append(apk)
 127         # Include ones we can't build, as a special case...
 128         for thisbuild in app['builds']:
 129             if thisbuild['disable']:
 130                 if thisbuild['vercode'] == app['Current Version Code']:
 131                     cantupdate = True
 132                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
 133                 apklist.append({'versioncode': int(thisbuild['vercode']),
 134                                 'version': thisbuild['version'],
 135                                 'buildproblem': thisbuild['disable']
 136                                 })
 137             else:
 138                 builtit = False
 139                 for apk in apklist:
 140                     if apk['versioncode'] == int(thisbuild['vercode']):
 141                         builtit = True
 142                         break
 143                 if not builtit:
 144                     buildfails = True
 145                     apklist.append({'versioncode': int(thisbuild['vercode']),
 146                                     'version': thisbuild['version'],
 147                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild|build log]].".format(app['id'])
 148                                     })
 149         if app['Current Version Code'] == '0':
 150             cantupdate = True
 151         # Sort with most recent first...
 152         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 153
 154         wikidata += "=Versions=\n"
 155         if len(apklist) == 0:
 156             wikidata += "We currently have no versions of this app available."
 157         elif not gotcurrentver:
 158             wikidata += "We don't have the current version of this app."
 159         else:
 160             wikidata += "We have the current version of this app."
 161         wikidata += " (Check mode: " + app['Update Check Mode'] + ") "
 162         wikidata += " (Auto-update mode: " + app['Auto Update Mode'] + ")\n\n"
 163         if len(app['No Source Since']) > 0:
 164             wikidata += "This application has partially or entirely been missing source code since version " + app['No Source Since'] + ".\n\n"
 165         if len(app['Current Version']) > 0:
 166             wikidata += "The current (recommended) version is " + app['Current Version']
 167             wikidata += " (version code " + app['Current Version Code'] + ").\n\n"
 168         validapks = 0
 169         for apk in apklist:
 170             wikidata += "==" + apk['version'] + "==\n"
 171
 172             if 'buildproblem' in apk:
 173                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
 174             else:
 175                 validapks += 1
 176                 wikidata += "This version is built and signed by "
 177                 if 'srcname' in apk:
 178                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
 179                 else:
 180                     wikidata += "the original developer.\n\n"
 181             wikidata += "Version code: " + str(apk['versioncode']) + '\n'
 182
 183         wikidata += '\n[[Category:' + wikicat + ']]\n'
 184         if len(app['No Source Since']) > 0:
 185             wikidata += '\n[[Category:Apps missing source code]]\n'
 186         if validapks == 0 and not app['Disabled']:
 187             wikidata += '\n[[Category:Apps with no packages]]\n'
 188         if cantupdate and not app['Disabled']:
 189             wikidata += "\n[[Category:Apps we can't update]]\n"
 190         if buildfails and not app['Disabled']:
 191             wikidata += "\n[[Category:Apps with failing builds]]\n"
 192         elif not gotcurrentver and not cantupdate and not app['Disabled'] and app['Update Check Mode'] != "Static":
 193             wikidata += '\n[[Category:Apps to Update]]\n'
 194         if app['Disabled']:
 195             wikidata += '\n[[Category:Apps that are disabled]]\n'
 196         if app['Update Check Mode'] == 'None' and not app['Disabled']:
 197             wikidata += '\n[[Category:Apps with no update check]]\n'
 198         for appcat in app['Categories']:
 199             wikidata += '\n[[Category:{0}]]\n'.format(appcat)
 200
 201         # We can't have underscores in the page name, even if they're in
 202         # the package ID, because MediaWiki messes with them...
 203         pagename = app['id'].replace('_', ' ')
 204
 205         # Drop a trailing newline, because mediawiki is going to drop it anyway
 206         # and it we don't we'll think the page has changed when it hasn't...
 207         if wikidata.endswith('\n'):
 208             wikidata = wikidata[:-1]
 209
 210         generated_pages[pagename] = wikidata
 211
 212         # Make a redirect from the name to the ID too, unless there's
 213         # already an existing page with the name and it isn't a redirect.
 214         noclobber = False
 215         apppagename = app['Name'].replace('_', ' ')
 216         apppagename = apppagename.replace('{', '')
 217         apppagename = apppagename.replace('}', ' ')
 218         apppagename = apppagename.replace(':', ' ')
 219         # Drop double spaces caused mostly by replacing ':' above
 220         apppagename = apppagename.replace('  ', ' ')
 221         for expagename in site.allpages(prefix=apppagename,
 222                                         filterredir='nonredirects',
 223                                         generator=False):
 224             if expagename == apppagename:
 225                 noclobber = True
 226         # Another reason not to make the redirect page is if the app name
 227         # is the same as it's ID, because that will overwrite the real page
 228         # with an redirect to itself! (Although it seems like an odd
 229         # scenario this happens a lot, e.g. where there is metadata but no
 230         # builds or binaries to extract a name from.
 231         if apppagename == pagename:
 232             noclobber = True
 233         if not noclobber:
 234             generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
 235
 236     for tcat, genp in [(wikicat, generated_pages),
 237                        (wikiredircat, generated_redirects)]:
 238         catpages = site.Pages['Category:' + tcat]
 239         existingpages = []
 240         for page in catpages:
 241             existingpages.append(page.name)
 242             if page.name in genp:
 243                 pagetxt = page.edit()
 244                 if pagetxt != genp[page.name]:
 245                     logging.debug("Updating modified page " + page.name)
 246                     page.save(genp[page.name], summary='Auto-updated')
 247                 else:
 248                     logging.debug("Page " + page.name + " is unchanged")
 249             else:
 250                 logging.warn("Deleting page " + page.name)
 251                 page.delete('No longer published')
 252         for pagename, text in genp.items():
 253             logging.debug("Checking " + pagename)
 254             if pagename not in existingpages:
 255                 logging.debug("Creating page " + pagename)
 256                 try:
 257                     newpage = site.Pages[pagename]
 258                     newpage.save(text, summary='Auto-created')
 259                 except:
 260                     logging.error("...FAILED to create page")
 261
 262     # Purge server cache to ensure counts are up to date
 263     site.pages['Repository Maintenance'].purge()
 264
 265
 266 def delete_disabled_builds(apps, apkcache, repodirs):
 267     """Delete disabled build outputs.
 268
 269     :param apps: list of all applications, as per metadata.read_metadata
 270     :param apkcache: current apk cache information
 271     :param repodirs: the repo directories to process
 272     """
 273     for app in apps:
 274         for build in app['builds']:
 275             if build['disable']:
 276                 apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk'
 277                 for repodir in repodirs:
 278                     apkpath = os.path.join(repodir, apkfilename)
 279                     srcpath = os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz")
 280                     for name in [apkpath, srcpath]:
 281                         if os.path.exists(name):
 282                             logging.warn("Deleting disabled build output " + apkfilename)
 283                             os.remove(name)
 284                 if apkfilename in apkcache:
 285                     del apkcache[apkfilename]
 286
 287
 288 def resize_icon(iconpath, density):
 289
 290     if not os.path.isfile(iconpath):
 291         return
 292
 293     try:
 294         im = Image.open(iconpath)
 295         size = dpi_to_px(density)
 296
 297         if any(length > size for length in im.size):
 298             oldsize = im.size
 299             im.thumbnail((size, size), Image.ANTIALIAS)
 300             logging.debug("%s was too large at %s - new size is %s" % (
 301                 iconpath, oldsize, im.size))
 302             im.save(iconpath, "PNG")
 303
 304     except Exception, e:
 305         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 306
 307
 308 def resize_all_icons(repodirs):
 309     """Resize all icons that exceed the max size
 310
 311     :param repodirs: the repo directories to process
 312     """
 313     for repodir in repodirs:
 314         for density in get_densities():
 315             icon_dir = get_icon_dir(repodir, density)
 316             icon_glob = os.path.join(icon_dir, '*.png')
 317             for iconpath in glob.glob(icon_glob):
 318                 resize_icon(iconpath, density)
 319
 320
 321 def scan_apks(apps, apkcache, repodir, knownapks):
 322     """Scan the apks in the given repo directory.
 323
 324     This also extracts the icons.
 325
 326     :param apps: list of all applications, as per metadata.read_metadata
 327     :param apkcache: current apk cache information
 328     :param repodir: repo directory to scan
 329     :param knownapks: known apks info
 330     :returns: (apks, cachechanged) where apks is a list of apk information,
 331               and cachechanged is True if the apkcache got changed.
 332     """
 333
 334     cachechanged = False
 335
 336     icon_dirs = get_icon_dirs(repodir)
 337     for icon_dir in icon_dirs:
 338         if os.path.exists(icon_dir):
 339             if options.clean:
 340                 shutil.rmtree(icon_dir)
 341                 os.makedirs(icon_dir)
 342         else:
 343             os.makedirs(icon_dir)
 344
 345     apks = []
 346     name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 347     vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
 348     vername_pat = re.compile(".*versionName='([^']*)'.*")
 349     label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
 350     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
 351     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
 352     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
 353     string_pat = re.compile(".*'([^']*)'.*")
 354     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 355
 356         apkfilename = apkfile[len(repodir) + 1:]
 357         if ' ' in apkfilename:
 358             logging.error("No spaces in APK filenames!")
 359             sys.exit(1)
 360
 361         if apkfilename in apkcache:
 362             logging.debug("Reading " + apkfilename + " from cache")
 363             thisinfo = apkcache[apkfilename]
 364
 365         else:
 366             if options.verbose:
 367                 logging.info("Processing " + apkfilename)
 368             thisinfo = {}
 369             thisinfo['apkname'] = apkfilename
 370             srcfilename = apkfilename[:-4] + "_src.tar.gz"
 371             if os.path.exists(os.path.join(repodir, srcfilename)):
 372                 thisinfo['srcname'] = srcfilename
 373             thisinfo['size'] = os.path.getsize(apkfile)
 374             thisinfo['permissions'] = []
 375             thisinfo['features'] = []
 376             thisinfo['icons_src'] = {}
 377             thisinfo['icons'] = {}
 378             p = FDroidPopen([os.path.join(config['sdk_path'], 'build-tools',
 379                                           config['build_tools'], 'aapt'),
 380                              'dump', 'badging', apkfile])
 381             if p.returncode != 0:
 382                 if options.delete_unknown:
 383                     if os.path.exists(apkfile):
 384                         logging.error("Failed to get apk information, deleting " + apkfile)
 385                         os.remove(apkfile)
 386                     else:
 387                         logging.error("Could not find {0} to remove it".format(apkfile))
 388                 else:
 389                     logging.error("Failed to get apk information, skipping " + apkfile)
 390                 continue
 391             for line in p.stdout.splitlines():
 392                 if line.startswith("package:"):
 393                     try:
 394                         thisinfo['id'] = re.match(name_pat, line).group(1)
 395                         thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
 396                         thisinfo['version'] = re.match(vername_pat, line).group(1)
 397                     except Exception, e:
 398                         logging.info("Package matching failed: " + str(e))
 399                         logging.info("Line was: " + line)
 400                         sys.exit(1)
 401                 elif line.startswith("application:"):
 402                     thisinfo['name'] = re.match(label_pat, line).group(1)
 403                     # Keep path to non-dpi icon in case we need it
 404                     match = re.match(icon_pat_nodpi, line)
 405                     if match:
 406                         thisinfo['icons_src']['-1'] = match.group(1)
 407                 elif line.startswith("launchable-activity:"):
 408                     # Only use launchable-activity as fallback to application
 409                     if not thisinfo['name']:
 410                         thisinfo['name'] = re.match(label_pat, line).group(1)
 411                     if '-1' not in thisinfo['icons_src']:
 412                         match = re.match(icon_pat_nodpi, line)
 413                         if match:
 414                             thisinfo['icons_src']['-1'] = match.group(1)
 415                 elif line.startswith("application-icon-"):
 416                     match = re.match(icon_pat, line)
 417                     if match:
 418                         density = match.group(1)
 419                         path = match.group(2)
 420                         thisinfo['icons_src'][density] = path
 421                 elif line.startswith("sdkVersion:"):
 422                     m = re.match(sdkversion_pat, line)
 423                     if m is None:
 424                         logging.error(line.replace('sdkVersion:', '')
 425                                       + ' is not a valid minSdkVersion!')
 426                     else:
 427                         thisinfo['sdkversion'] = m.group(1)
 428                 elif line.startswith("maxSdkVersion:"):
 429                     thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
 430                 elif line.startswith("native-code:"):
 431                     thisinfo['nativecode'] = []
 432                     for arch in line[13:].split(' '):
 433                         thisinfo['nativecode'].append(arch[1:-1])
 434                 elif line.startswith("uses-permission:"):
 435                     perm = re.match(string_pat, line).group(1)
 436                     if perm.startswith("android.permission."):
 437                         perm = perm[19:]
 438                     thisinfo['permissions'].append(perm)
 439                 elif line.startswith("uses-feature:"):
 440                     perm = re.match(string_pat, line).group(1)
 441                     # Filter out this, it's only added with the latest SDK tools and
 442                     # causes problems for lots of apps.
 443                     if perm != "android.hardware.screen.portrait" \
 444                             and perm != "android.hardware.screen.landscape":
 445                         if perm.startswith("android.feature."):
 446                             perm = perm[16:]
 447                         thisinfo['features'].append(perm)
 448
 449             if 'sdkversion' not in thisinfo:
 450                 logging.warn("no SDK version information found")
 451                 thisinfo['sdkversion'] = 0
 452
 453             # Check for debuggable apks...
 454             if common.isApkDebuggable(apkfile, config):
 455                 logging.warn('{0} is set to android:debuggable="true"!'.format(apkfile))
 456
 457             # Calculate the sha256...
 458             sha = hashlib.sha256()
 459             with open(apkfile, 'rb') as f:
 460                 while True:
 461                     t = f.read(1024)
 462                     if len(t) == 0:
 463                         break
 464                     sha.update(t)
 465                 thisinfo['sha256'] = sha.hexdigest()
 466
 467             # Get the signature (or md5 of, to be precise)...
 468             getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
 469             if not os.path.exists(getsig_dir + "/getsig.class"):
 470                 logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
 471                 sys.exit(1)
 472             p = FDroidPopen(['java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'),
 473                              'getsig', os.path.join(os.getcwd(), apkfile)])
 474             if p.returncode != 0 or not p.stdout.startswith('Result:'):
 475                 logging.critical("Failed to get apk signature")
 476                 sys.exit(1)
 477             thisinfo['sig'] = p.stdout[7:].strip()
 478
 479             apk = zipfile.ZipFile(apkfile, 'r')
 480
 481             iconfilename = "%s.%s.png" % (
 482                 thisinfo['id'],
 483                 thisinfo['versioncode'])
 484
 485             # Extract the icon file...
 486             densities = get_densities()
 487             empty_densities = []
 488             for density in densities:
 489                 if density not in thisinfo['icons_src']:
 490                     empty_densities.append(density)
 491                     continue
 492                 iconsrc = thisinfo['icons_src'][density]
 493                 icon_dir = get_icon_dir(repodir, density)
 494                 icondest = os.path.join(icon_dir, iconfilename)
 495
 496                 try:
 497                     iconfile = open(icondest, 'wb')
 498                     iconfile.write(apk.read(iconsrc))
 499                     iconfile.close()
 500                     thisinfo['icons'][density] = iconfilename
 501
 502                 except:
 503                     logging.warn("Error retrieving icon file")
 504                     del thisinfo['icons'][density]
 505                     del thisinfo['icons_src'][density]
 506                     empty_densities.append(density)
 507
 508             if '-1' in thisinfo['icons_src']:
 509                 iconsrc = thisinfo['icons_src']['-1']
 510                 iconpath = os.path.join(
 511                     get_icon_dir(repodir, None), iconfilename)
 512                 iconfile = open(iconpath, 'wb')
 513                 iconfile.write(apk.read(iconsrc))
 514                 iconfile.close()
 515                 try:
 516                     im = Image.open(iconpath)
 517                     dpi = px_to_dpi(im.size[0])
 518                     for density in densities:
 519                         if density in thisinfo['icons']:
 520                             break
 521                         if density == densities[-1] or dpi >= int(density):
 522                             thisinfo['icons'][density] = iconfilename
 523                             shutil.move(iconpath,
 524                                         os.path.join(get_icon_dir(repodir, density), iconfilename))
 525                             empty_densities.remove(density)
 526                             break
 527                 except Exception, e:
 528                     logging.warn("Failed reading {0} - {1}".format(iconpath, e))
 529
 530             if thisinfo['icons']:
 531                 thisinfo['icon'] = iconfilename
 532
 533             apk.close()
 534
 535             # First try resizing down to not lose quality
 536             last_density = None
 537             for density in densities:
 538                 if density not in empty_densities:
 539                     last_density = density
 540                     continue
 541                 if last_density is None:
 542                     continue
 543                 logging.info("Density %s not available, resizing down from %s"
 544                              % (density, last_density))
 545
 546                 last_iconpath = os.path.join(
 547                     get_icon_dir(repodir, last_density), iconfilename)
 548                 iconpath = os.path.join(
 549                     get_icon_dir(repodir, density), iconfilename)
 550                 try:
 551                     im = Image.open(last_iconpath)
 552                 except:
 553                     logging.warn("Invalid image file at %s" % last_iconpath)
 554                     continue
 555
 556                 size = dpi_to_px(density)
 557
 558                 im.thumbnail((size, size), Image.ANTIALIAS)
 559                 im.save(iconpath, "PNG")
 560                 empty_densities.remove(density)
 561
 562             # Then just copy from the highest resolution available
 563             last_density = None
 564             for density in reversed(densities):
 565                 if density not in empty_densities:
 566                     last_density = density
 567                     continue
 568                 if last_density is None:
 569                     continue
 570                 logging.info("Density %s not available, copying from lower density %s"
 571                              % (density, last_density))
 572
 573                 shutil.copyfile(
 574                     os.path.join(get_icon_dir(repodir, last_density), iconfilename),
 575                     os.path.join(get_icon_dir(repodir, density), iconfilename))
 576
 577                 empty_densities.remove(density)
 578
 579             for density in densities:
 580                 icon_dir = get_icon_dir(repodir, density)
 581                 icondest = os.path.join(icon_dir, iconfilename)
 582                 resize_icon(icondest, density)
 583
 584             # Copy from icons-mdpi to icons since mdpi is the baseline density
 585             baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
 586             if os.path.isfile(baseline):
 587                 shutil.copyfile(baseline,
 588                                 os.path.join(get_icon_dir(repodir, None), iconfilename))
 589
 590             # Record in known apks, getting the added date at the same time..
 591             added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
 592             if added:
 593                 thisinfo['added'] = added
 594
 595             apkcache[apkfilename] = thisinfo
 596             cachechanged = True
 597
 598         apks.append(thisinfo)
 599
 600     return apks, cachechanged
 601
 602
 603 repo_pubkey_fingerprint = None
 604
 605
 606 def make_index(apps, apks, repodir, archive, categories):
 607     """Make a repo index.
 608
 609     :param apps: fully populated apps list
 610     :param apks: full populated apks list
 611     :param repodir: the repo directory
 612     :param archive: True if this is the archive repo, False if it's the
 613                     main one.
 614     :param categories: list of categories
 615     """
 616
 617     doc = Document()
 618
 619     def addElement(name, value, doc, parent):
 620         el = doc.createElement(name)
 621         el.appendChild(doc.createTextNode(value))
 622         parent.appendChild(el)
 623
 624     def addElementCDATA(name, value, doc, parent):
 625         el = doc.createElement(name)
 626         el.appendChild(doc.createCDATASection(value))
 627         parent.appendChild(el)
 628
 629     root = doc.createElement("fdroid")
 630     doc.appendChild(root)
 631
 632     repoel = doc.createElement("repo")
 633
 634     if archive:
 635         repoel.setAttribute("name", config['archive_name'])
 636         if config['repo_maxage'] != 0:
 637             repoel.setAttribute("maxage", str(config['repo_maxage']))
 638         repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
 639         repoel.setAttribute("url", config['archive_url'])
 640         addElement('description', config['archive_description'], doc, repoel)
 641
 642     else:
 643         repoel.setAttribute("name", config['repo_name'])
 644         if config['repo_maxage'] != 0:
 645             repoel.setAttribute("maxage", str(config['repo_maxage']))
 646         repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
 647         repoel.setAttribute("url", config['repo_url'])
 648         addElement('description', config['repo_description'], doc, repoel)
 649
 650     repoel.setAttribute("version", "12")
 651     repoel.setAttribute("timestamp", str(int(time.time())))
 652
 653     if 'repo_keyalias' in config:
 654
 655         # Generate a certificate fingerprint the same way keytool does it
 656         # (but with slightly different formatting)
 657         def cert_fingerprint(data):
 658             digest = hashlib.sha256(data).digest()
 659             ret = []
 660             ret.append(' '.join("%02X" % ord(b) for b in digest))
 661             return " ".join(ret)
 662
 663         def extract_pubkey():
 664             p = FDroidPopen(['keytool', '-exportcert',
 665                              '-alias', config['repo_keyalias'],
 666                              '-keystore', config['keystore'],
 667                              '-storepass:file', config['keystorepassfile']]
 668                             + config['smartcardoptions'])
 669             if p.returncode != 0:
 670                 msg = "Failed to get repo pubkey!"
 671                 if config['keystore'] == 'NONE':
 672                     msg += ' Is your crypto smartcard plugged in?'
 673                 logging.critical(msg)
 674                 sys.exit(1)
 675             global repo_pubkey_fingerprint
 676             repo_pubkey_fingerprint = cert_fingerprint(p.stdout)
 677             return "".join("%02x" % ord(b) for b in p.stdout)
 678
 679         repoel.setAttribute("pubkey", extract_pubkey())
 680
 681     root.appendChild(repoel)
 682
 683     for app in apps:
 684
 685         if app['Disabled'] is not None:
 686             continue
 687
 688         # Get a list of the apks for this app...
 689         apklist = []
 690         for apk in apks:
 691             if apk['id'] == app['id']:
 692                 apklist.append(apk)
 693
 694         if len(apklist) == 0:
 695             continue
 696
 697         apel = doc.createElement("application")
 698         apel.setAttribute("id", app['id'])
 699         root.appendChild(apel)
 700
 701         addElement('id', app['id'], doc, apel)
 702         if 'added' in app:
 703             addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel)
 704         if 'lastupdated' in app:
 705             addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel)
 706         addElement('name', app['Name'], doc, apel)
 707         addElement('summary', app['Summary'], doc, apel)
 708         if app['icon']:
 709             addElement('icon', app['icon'], doc, apel)
 710
 711         def linkres(link):
 712             for app in apps:
 713                 if app['id'] == link:
 714                     return ("fdroid.app:" + link, app['Name'])
 715             raise MetaDataException("Cannot resolve app id " + link)
 716         addElement('desc',
 717                    metadata.description_html(app['Description'], linkres),
 718                    doc, apel)
 719         addElement('license', app['License'], doc, apel)
 720         if 'Categories' in app:
 721             addElement('categories', ','.join(app["Categories"]), doc, apel)
 722             # We put the first (primary) category in LAST, which will have
 723             # the desired effect of making clients that only understand one
 724             # category see that one.
 725             addElement('category', app["Categories"][0], doc, apel)
 726         addElement('web', app['Web Site'], doc, apel)
 727         addElement('source', app['Source Code'], doc, apel)
 728         addElement('tracker', app['Issue Tracker'], doc, apel)
 729         if app['Donate']:
 730             addElement('donate', app['Donate'], doc, apel)
 731         if app['Bitcoin']:
 732             addElement('bitcoin', app['Bitcoin'], doc, apel)
 733         if app['Litecoin']:
 734             addElement('litecoin', app['Litecoin'], doc, apel)
 735         if app['Dogecoin']:
 736             addElement('dogecoin', app['Dogecoin'], doc, apel)
 737         if app['FlattrID']:
 738             addElement('flattr', app['FlattrID'], doc, apel)
 739
 740         # These elements actually refer to the current version (i.e. which
 741         # one is recommended. They are historically mis-named, and need
 742         # changing, but stay like this for now to support existing clients.
 743         addElement('marketversion', app['Current Version'], doc, apel)
 744         addElement('marketvercode', app['Current Version Code'], doc, apel)
 745
 746         if app['AntiFeatures']:
 747             af = app['AntiFeatures'].split(',')
 748             # TODO: Temporarily not including UpstreamNonFree in the index,
 749             # because current F-Droid clients do not understand it, and also
 750             # look ugly when they encounter an unknown antifeature. This
 751             # filtering can be removed in time...
 752             if 'UpstreamNonFree' in af:
 753                 af.remove('UpstreamNonFree')
 754             if af:
 755                 addElement('antifeatures', ','.join(af), doc, apel)
 756         if app['Provides']:
 757             pv = app['Provides'].split(',')
 758             addElement('provides', ','.join(pv), doc, apel)
 759         if app['Requires Root']:
 760             addElement('requirements', 'root', doc, apel)
 761
 762         # Sort the apk list into version order, just so the web site
 763         # doesn't have to do any work by default...
 764         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 765
 766         # Check for duplicates - they will make the client unhappy...
 767         for i in range(len(apklist) - 1):
 768             if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
 769                 logging.critical("duplicate versions: '%s' - '%s'" % (
 770                     apklist[i]['apkname'], apklist[i + 1]['apkname']))
 771                 sys.exit(1)
 772
 773         for apk in apklist:
 774             apkel = doc.createElement("package")
 775             apel.appendChild(apkel)
 776             addElement('version', apk['version'], doc, apkel)
 777             addElement('versioncode', str(apk['versioncode']), doc, apkel)
 778             addElement('apkname', apk['apkname'], doc, apkel)
 779             if 'srcname' in apk:
 780                 addElement('srcname', apk['srcname'], doc, apkel)
 781             for hash_type in ['sha256']:
 782                 if hash_type not in apk:
 783                     continue
 784                 hashel = doc.createElement("hash")
 785                 hashel.setAttribute("type", hash_type)
 786                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
 787                 apkel.appendChild(hashel)
 788             addElement('sig', apk['sig'], doc, apkel)
 789             addElement('size', str(apk['size']), doc, apkel)
 790             addElement('sdkver', str(apk['sdkversion']), doc, apkel)
 791             if 'maxsdkversion' in apk:
 792                 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
 793             if 'added' in apk:
 794                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 795             if app['Requires Root']:
 796                 if 'ACCESS_SUPERUSER' not in apk['permissions']:
 797                     apk['permissions'].append('ACCESS_SUPERUSER')
 798
 799             if len(apk['permissions']) > 0:
 800                 addElement('permissions', ','.join(apk['permissions']), doc, apkel)
 801             if 'nativecode' in apk and len(apk['nativecode']) > 0:
 802                 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
 803             if len(apk['features']) > 0:
 804                 addElement('features', ','.join(apk['features']), doc, apkel)
 805
 806     of = open(os.path.join(repodir, 'index.xml'), 'wb')
 807     if options.pretty:
 808         output = doc.toprettyxml()
 809     else:
 810         output = doc.toxml()
 811     of.write(output)
 812     of.close()
 813
 814     if 'repo_keyalias' in config:
 815
 816         logging.info("Creating signed index with this key (SHA256):")
 817         logging.info("%s" % repo_pubkey_fingerprint)
 818
 819         # Create a jar of the index...
 820         p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
 821         if p.returncode != 0:
 822             logging.critical("Failed to create jar file")
 823             sys.exit(1)
 824
 825         # Sign the index...
 826         args = ['jarsigner', '-keystore', config['keystore'],
 827                 '-storepass:file', config['keystorepassfile'],
 828                 '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
 829                 os.path.join(repodir, 'index.jar'), config['repo_keyalias']]
 830         if config['keystore'] == 'NONE':
 831             args += config['smartcardoptions']
 832         else:  # smardcards never use -keypass
 833             args += ['-keypass:file', config['keypassfile']]
 834         p = FDroidPopen(args)
 835         # TODO keypass should be sent via stdin
 836         if p.returncode != 0:
 837             logging.info("Failed to sign index")
 838             sys.exit(1)
 839
 840     # Copy the repo icon into the repo directory...
 841     icon_dir = os.path.join(repodir, 'icons')
 842     iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
 843     shutil.copyfile(config['repo_icon'], iconfilename)
 844
 845     # Write a category list in the repo to allow quick access...
 846     catdata = ''
 847     for cat in categories:
 848         catdata += cat + '\n'
 849     f = open(os.path.join(repodir, 'categories.txt'), 'w')
 850     f.write(catdata)
 851     f.close()
 852
 853
 854 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
 855
 856     for app in apps:
 857
 858         # Get a list of the apks for this app...
 859         apklist = []
 860         for apk in apks:
 861             if apk['id'] == app['id']:
 862                 apklist.append(apk)
 863
 864         # Sort the apk list into version order...
 865         apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
 866
 867         if app['Archive Policy']:
 868             keepversions = int(app['Archive Policy'][:-9])
 869         else:
 870             keepversions = defaultkeepversions
 871
 872         if len(apklist) > keepversions:
 873             for apk in apklist[keepversions:]:
 874                 logging.info("Moving " + apk['apkname'] + " to archive")
 875                 shutil.move(os.path.join(repodir, apk['apkname']),
 876                             os.path.join(archivedir, apk['apkname']))
 877                 if 'srcname' in apk:
 878                     shutil.move(os.path.join(repodir, apk['srcname']),
 879                                 os.path.join(archivedir, apk['srcname']))
 880                 archapks.append(apk)
 881                 apks.remove(apk)
 882
 883
 884 config = None
 885 options = None
 886
 887
 888 def main():
 889
 890     global config, options
 891
 892     # Parse command line...
 893     parser = OptionParser()
 894     parser.add_option("-c", "--create-metadata", action="store_true", default=False,
 895                       help="Create skeleton metadata files that are missing")
 896     parser.add_option("--delete-unknown", action="store_true", default=False,
 897                       help="Delete APKs without metadata from the repo")
 898     parser.add_option("-v", "--verbose", action="store_true", default=False,
 899                       help="Spew out even more information than normal")
 900     parser.add_option("-q", "--quiet", action="store_true", default=False,
 901                       help="Restrict output to warnings and errors")
 902     parser.add_option("-b", "--buildreport", action="store_true", default=False,
 903                       help="Report on build data status")
 904     parser.add_option("-i", "--interactive", default=False, action="store_true",
 905                       help="Interactively ask about things that need updating.")
 906     parser.add_option("-I", "--icons", action="store_true", default=False,
 907                       help="Resize all the icons exceeding the max pixel size and exit")
 908     parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
 909                       help="Specify editor to use in interactive mode. Default " +
 910                       "is /etc/alternatives/editor")
 911     parser.add_option("-w", "--wiki", default=False, action="store_true",
 912                       help="Update the wiki")
 913     parser.add_option("", "--pretty", action="store_true", default=False,
 914                       help="Produce human-readable index.xml")
 915     parser.add_option("--clean", action="store_true", default=False,
 916                       help="Clean update - don't uses caches, reprocess all apks")
 917     (options, args) = parser.parse_args()
 918
 919     config = common.read_config(options)
 920
 921     repodirs = ['repo']
 922     if config['archive_older'] != 0:
 923         repodirs.append('archive')
 924         if not os.path.exists('archive'):
 925             os.mkdir('archive')
 926
 927     if options.icons:
 928         resize_all_icons(repodirs)
 929         sys.exit(0)
 930
 931     # Get all apps...
 932     apps = metadata.read_metadata()
 933
 934     # Generate a list of categories...
 935     categories = set()
 936     for app in apps:
 937         categories.update(app['Categories'])
 938
 939     # Read known apks data (will be updated and written back when we've finished)
 940     knownapks = common.KnownApks()
 941
 942     # Gather information about all the apk files in the repo directory, using
 943     # cached data if possible.
 944     apkcachefile = os.path.join('tmp', 'apkcache')
 945     if not options.clean and os.path.exists(apkcachefile):
 946         with open(apkcachefile, 'rb') as cf:
 947             apkcache = pickle.load(cf)
 948     else:
 949         apkcache = {}
 950     cachechanged = False
 951
 952     delete_disabled_builds(apps, apkcache, repodirs)
 953
 954     # Scan all apks in the main repo
 955     apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
 956     if cc:
 957         cachechanged = True
 958
 959     # Generate warnings for apk's with no metadata (or create skeleton
 960     # metadata files, if requested on the command line)
 961     newmetadata = False
 962     for apk in apks:
 963         found = False
 964         for app in apps:
 965             if app['id'] == apk['id']:
 966                 found = True
 967                 break
 968         if not found:
 969             if options.create_metadata:
 970                 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
 971                 f.write("License:Unknown\n")
 972                 f.write("Web Site:\n")
 973                 f.write("Source Code:\n")
 974                 f.write("Issue Tracker:\n")
 975                 f.write("Summary:" + apk['name'] + "\n")
 976                 f.write("Description:\n")
 977                 f.write(apk['name'] + "\n")
 978                 f.write(".\n")
 979                 f.close()
 980                 logging.info("Generated skeleton metadata for " + apk['id'])
 981                 newmetadata = True
 982             else:
 983                 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
 984                 if options.delete_unknown:
 985                     logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
 986                     rmf = os.path.join(repodirs[0], apk['apkname'])
 987                     if not os.path.exists(rmf):
 988                         logging.error("Could not find {0} to remove it".format(rmf))
 989                     else:
 990                         os.remove(rmf)
 991                 else:
 992                     logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
 993
 994     # update the metadata with the newly created ones included
 995     if newmetadata:
 996         apps = metadata.read_metadata()
 997
 998     # Scan the archive repo for apks as well
 999     if len(repodirs) > 1:
1000         archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1001         if cc:
1002             cachechanged = True
1003     else:
1004         archapks = []
1005
1006     # Some information from the apks needs to be applied up to the application
1007     # level. When doing this, we use the info from the most recent version's apk.
1008     # We deal with figuring out when the app was added and last updated at the
1009     # same time.
1010     for app in apps:
1011         bestver = 0
1012         added = None
1013         lastupdated = None
1014         for apk in apks + archapks:
1015             if apk['id'] == app['id']:
1016                 if apk['versioncode'] > bestver:
1017                     bestver = apk['versioncode']
1018                     bestapk = apk
1019
1020                 if 'added' in apk:
1021                     if not added or apk['added'] < added:
1022                         added = apk['added']
1023                     if not lastupdated or apk['added'] > lastupdated:
1024                         lastupdated = apk['added']
1025
1026         if added:
1027             app['added'] = added
1028         else:
1029             logging.warn("Don't know when " + app['id'] + " was added")
1030         if lastupdated:
1031             app['lastupdated'] = lastupdated
1032         else:
1033             logging.warn("Don't know when " + app['id'] + " was last updated")
1034
1035         if bestver == 0:
1036             if app['Name'] is None:
1037                 app['Name'] = app['id']
1038             app['icon'] = None
1039             logging.warn("Application " + app['id'] + " has no packages")
1040         else:
1041             if app['Name'] is None:
1042                 app['Name'] = bestapk['name']
1043             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
1044
1045     # Sort the app list by name, then the web site doesn't have to by default.
1046     # (we had to wait until we'd scanned the apks to do this, because mostly the
1047     # name comes from there!)
1048     apps = sorted(apps, key=lambda app: app['Name'].upper())
1049
1050     if len(repodirs) > 1:
1051         archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1052
1053     # Make the index for the main repo...
1054     make_index(apps, apks, repodirs[0], False, categories)
1055
1056     # If there's an archive repo,  make the index for it. We already scanned it
1057     # earlier on.
1058     if len(repodirs) > 1:
1059         make_index(apps, archapks, repodirs[1], True, categories)
1060
1061     if config['update_stats']:
1062
1063         # Update known apks info...
1064         knownapks.writeifchanged()
1065
1066         # Generate latest apps data for widget
1067         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1068             data = ''
1069             for line in file(os.path.join('stats', 'latestapps.txt')):
1070                 appid = line.rstrip()
1071                 data += appid + "\t"
1072                 for app in apps:
1073                     if app['id'] == appid:
1074                         data += app['Name'] + "\t"
1075                         if app['icon'] is not None:
1076                             data += app['icon'] + "\t"
1077                         data += app['License'] + "\n"
1078                         break
1079             f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
1080             f.write(data)
1081             f.close()
1082
1083     if cachechanged:
1084         with open(apkcachefile, 'wb') as cf:
1085             pickle.dump(apkcache, cf)
1086
1087     # Update the wiki...
1088     if options.wiki:
1089         update_wiki(apps, apks + archapks)
1090
1091     logging.info("Finished.")
1092
1093 if __name__ == "__main__":
1094     main()