2 # -*- coding: utf-8 -*-
4 # update.py - part of the FDroid server tools
5 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
6 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU Affero General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU Affero General Public License for more details.
18 # You should have received a copy of the GNU Affero General Public License
19 # along with this program. If not, see <http://www.gnu.org/licenses/>.
30 from datetime import datetime, timedelta
31 from xml.dom.minidom import Document
32 from argparse import ArgumentParser
34 from pyasn1.error import PyAsn1Error
35 from pyasn1.codec.der import decoder, encoder
36 from pyasn1_modules import rfc2315
37 from hashlib import md5
38 from binascii import hexlify, unhexlify
45 from common import FDroidPopen, SdkToolsPopen
46 from metadata import MetaDataException
48 screen_densities = ['640', '480', '320', '240', '160', '120']
50 all_screen_densities = ['0'] + screen_densities
53 def dpi_to_px(density):
54 return (int(density) * 48) / 160
58 return (int(px) * 160) / 48
61 def get_icon_dir(repodir, density):
63 return os.path.join(repodir, "icons")
64 return os.path.join(repodir, "icons-%s" % density)
67 def get_icon_dirs(repodir):
68 for density in screen_densities:
69 yield get_icon_dir(repodir, density)
72 def get_all_icon_dirs(repodir):
73 for density in all_screen_densities:
74 yield get_icon_dir(repodir, density)
77 def update_wiki(apps, sortedids, apks):
80 :param apps: fully populated list of all applications
81 :param apks: all apks, except...
83 logging.info("Updating wiki")
85 wikiredircat = 'App Redirects'
87 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
88 path=config['wiki_path'])
89 site.login(config['wiki_user'], config['wiki_password'])
91 generated_redirects = {}
93 for appid in sortedids:
98 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
100 for af in app.AntiFeatures:
101 wikidata += '{{AntiFeature|' + af + '}}\n'
106 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
109 time.strftime('%Y-%m-%d', app.added) if app.added else '',
110 time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
125 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
127 wikidata += app.Summary
128 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
130 wikidata += "=Description=\n"
131 wikidata += metadata.description_wiki(app.Description) + "\n"
133 wikidata += "=Maintainer Notes=\n"
134 if app.MaintainerNotes:
135 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
136 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
138 # Get a list of all packages for this application...
140 gotcurrentver = False
144 if apk['id'] == appid:
145 if str(apk['versioncode']) == app.CurrentVersionCode:
148 # Include ones we can't build, as a special case...
149 for build in app.builds:
151 if build.vercode == app.CurrentVersionCode:
153 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
154 apklist.append({'versioncode': int(build.vercode),
155 'version': build.version,
156 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
161 if apk['versioncode'] == int(build.vercode):
166 apklist.append({'versioncode': int(build.vercode),
167 'version': build.version,
168 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
170 if app.CurrentVersionCode == '0':
172 # Sort with most recent first...
173 apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
175 wikidata += "=Versions=\n"
176 if len(apklist) == 0:
177 wikidata += "We currently have no versions of this app available."
178 elif not gotcurrentver:
179 wikidata += "We don't have the current version of this app."
181 wikidata += "We have the current version of this app."
182 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
183 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
184 if len(app.NoSourceSince) > 0:
185 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
186 if len(app.CurrentVersion) > 0:
187 wikidata += "The current (recommended) version is " + app.CurrentVersion
188 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
191 wikidata += "==" + apk['version'] + "==\n"
193 if 'buildproblem' in apk:
194 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
197 wikidata += "This version is built and signed by "
199 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
201 wikidata += "the original developer.\n\n"
202 wikidata += "Version code: " + str(apk['versioncode']) + '\n'
204 wikidata += '\n[[Category:' + wikicat + ']]\n'
205 if len(app.NoSourceSince) > 0:
206 wikidata += '\n[[Category:Apps missing source code]]\n'
207 if validapks == 0 and not app.Disabled:
208 wikidata += '\n[[Category:Apps with no packages]]\n'
209 if cantupdate and not app.Disabled:
210 wikidata += "\n[[Category:Apps we can't update]]\n"
211 if buildfails and not app.Disabled:
212 wikidata += "\n[[Category:Apps with failing builds]]\n"
213 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
214 wikidata += '\n[[Category:Apps to Update]]\n'
216 wikidata += '\n[[Category:Apps that are disabled]]\n'
217 if app.UpdateCheckMode == 'None' and not app.Disabled:
218 wikidata += '\n[[Category:Apps with no update check]]\n'
219 for appcat in app.Categories:
220 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
222 # We can't have underscores in the page name, even if they're in
223 # the package ID, because MediaWiki messes with them...
224 pagename = appid.replace('_', ' ')
226 # Drop a trailing newline, because mediawiki is going to drop it anyway
227 # and it we don't we'll think the page has changed when it hasn't...
228 if wikidata.endswith('\n'):
229 wikidata = wikidata[:-1]
231 generated_pages[pagename] = wikidata
233 # Make a redirect from the name to the ID too, unless there's
234 # already an existing page with the name and it isn't a redirect.
236 apppagename = app.Name.replace('_', ' ')
237 apppagename = apppagename.replace('{', '')
238 apppagename = apppagename.replace('}', ' ')
239 apppagename = apppagename.replace(':', ' ')
240 # Drop double spaces caused mostly by replacing ':' above
241 apppagename = apppagename.replace(' ', ' ')
242 for expagename in site.allpages(prefix=apppagename,
243 filterredir='nonredirects',
245 if expagename == apppagename:
247 # Another reason not to make the redirect page is if the app name
248 # is the same as it's ID, because that will overwrite the real page
249 # with an redirect to itself! (Although it seems like an odd
250 # scenario this happens a lot, e.g. where there is metadata but no
251 # builds or binaries to extract a name from.
252 if apppagename == pagename:
255 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
257 for tcat, genp in [(wikicat, generated_pages),
258 (wikiredircat, generated_redirects)]:
259 catpages = site.Pages['Category:' + tcat]
261 for page in catpages:
262 existingpages.append(page.name)
263 if page.name in genp:
264 pagetxt = page.edit()
265 if pagetxt != genp[page.name]:
266 logging.debug("Updating modified page " + page.name)
267 page.save(genp[page.name], summary='Auto-updated')
269 logging.debug("Page " + page.name + " is unchanged")
271 logging.warn("Deleting page " + page.name)
272 page.delete('No longer published')
273 for pagename, text in genp.items():
274 logging.debug("Checking " + pagename)
275 if pagename not in existingpages:
276 logging.debug("Creating page " + pagename)
278 newpage = site.Pages[pagename]
279 newpage.save(text, summary='Auto-created')
281 logging.error("...FAILED to create page '{0}'".format(pagename))
283 # Purge server cache to ensure counts are up to date
284 site.pages['Repository Maintenance'].purge()
287 def delete_disabled_builds(apps, apkcache, repodirs):
288 """Delete disabled build outputs.
290 :param apps: list of all applications, as per metadata.read_metadata
291 :param apkcache: current apk cache information
292 :param repodirs: the repo directories to process
294 for appid, app in apps.iteritems():
295 for build in app.builds:
296 if not build.disable:
298 apkfilename = appid + '_' + str(build.vercode) + '.apk'
299 iconfilename = "%s.%s.png" % (
302 for repodir in repodirs:
304 os.path.join(repodir, apkfilename),
305 os.path.join(repodir, apkfilename + '.asc'),
306 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
308 for density in all_screen_densities:
309 repo_dir = get_icon_dir(repodir, density)
310 files.append(os.path.join(repo_dir, iconfilename))
313 if os.path.exists(f):
314 logging.info("Deleting disabled build output " + f)
316 if apkfilename in apkcache:
317 del apkcache[apkfilename]
320 def resize_icon(iconpath, density):
322 if not os.path.isfile(iconpath):
326 im = Image.open(iconpath)
327 size = dpi_to_px(density)
329 if any(length > size for length in im.size):
331 im.thumbnail((size, size), Image.ANTIALIAS)
332 logging.debug("%s was too large at %s - new size is %s" % (
333 iconpath, oldsize, im.size))
334 im.save(iconpath, "PNG")
336 except Exception as e:
337 logging.error("Failed resizing {0} - {1}".format(iconpath, e))
340 def resize_all_icons(repodirs):
341 """Resize all icons that exceed the max size
343 :param repodirs: the repo directories to process
345 for repodir in repodirs:
346 for density in screen_densities:
347 icon_dir = get_icon_dir(repodir, density)
348 icon_glob = os.path.join(icon_dir, '*.png')
349 for iconpath in glob.glob(icon_glob):
350 resize_icon(iconpath, density)
353 # A signature block file with a .DSA, .RSA, or .EC extension
354 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
358 """ Get the signing certificate of an apk. To get the same md5 has that
359 Android gets, we encode the .RSA certificate in a specific format and pass
360 it hex-encoded to the md5 digest algorithm.
362 :param apkpath: path to the apk
363 :returns: A string containing the md5 of the signature of the apk or None
364 if an error occurred.
369 # verify the jar signature is correct
370 args = ['jarsigner', '-verify', apkpath]
371 p = FDroidPopen(args)
372 if p.returncode != 0:
373 logging.critical(apkpath + " has a bad signature!")
376 with zipfile.ZipFile(apkpath, 'r') as apk:
378 certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
381 logging.error("Found no signing certificates on %s" % apkpath)
384 logging.error("Found multiple signing certificates on %s" % apkpath)
387 cert = apk.read(certs[0])
389 content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
390 if content.getComponentByName('contentType') != rfc2315.signedData:
391 logging.error("Unexpected format.")
394 content = decoder.decode(content.getComponentByName('content'),
395 asn1Spec=rfc2315.SignedData())[0]
397 certificates = content.getComponentByName('certificates')
399 logging.error("Certificates not found.")
402 cert_encoded = encoder.encode(certificates)[4:]
404 return md5(cert_encoded.encode('hex')).hexdigest()
407 def scan_apks(apps, apkcache, repodir, knownapks):
408 """Scan the apks in the given repo directory.
410 This also extracts the icons.
412 :param apps: list of all applications, as per metadata.read_metadata
413 :param apkcache: current apk cache information
414 :param repodir: repo directory to scan
415 :param knownapks: known apks info
416 :returns: (apks, cachechanged) where apks is a list of apk information,
417 and cachechanged is True if the apkcache got changed.
422 for icon_dir in get_all_icon_dirs(repodir):
423 if os.path.exists(icon_dir):
425 shutil.rmtree(icon_dir)
426 os.makedirs(icon_dir)
428 os.makedirs(icon_dir)
431 name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
432 vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
433 vername_pat = re.compile(".*versionName='([^']*)'.*")
434 label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
435 icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
436 icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
437 sdkversion_pat = re.compile(".*'([0-9]*)'.*")
438 string_pat = re.compile(".*'([^']*)'.*")
439 for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
441 apkfilename = apkfile[len(repodir) + 1:]
442 if ' ' in apkfilename:
443 logging.critical("Spaces in filenames are not allowed.")
446 # Calculate the sha256...
447 sha = hashlib.sha256()
448 with open(apkfile, 'rb') as f:
454 shasum = sha.hexdigest()
457 if apkfilename in apkcache:
458 apk = apkcache[apkfilename]
459 if apk['sha256'] == shasum:
460 logging.debug("Reading " + apkfilename + " from cache")
463 logging.debug("Ignoring stale cache data for " + apkfilename)
466 logging.debug("Processing " + apkfilename)
468 apk['apkname'] = apkfilename
469 apk['sha256'] = shasum
470 srcfilename = apkfilename[:-4] + "_src.tar.gz"
471 if os.path.exists(os.path.join(repodir, srcfilename)):
472 apk['srcname'] = srcfilename
473 apk['size'] = os.path.getsize(apkfile)
474 apk['permissions'] = set()
475 apk['features'] = set()
476 apk['icons_src'] = {}
478 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
479 if p.returncode != 0:
480 if options.delete_unknown:
481 if os.path.exists(apkfile):
482 logging.error("Failed to get apk information, deleting " + apkfile)
485 logging.error("Could not find {0} to remove it".format(apkfile))
487 logging.error("Failed to get apk information, skipping " + apkfile)
489 for line in p.output.splitlines():
490 if line.startswith("package:"):
492 apk['id'] = re.match(name_pat, line).group(1)
493 apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
494 apk['version'] = re.match(vername_pat, line).group(1)
495 except Exception as e:
496 logging.error("Package matching failed: " + str(e))
497 logging.info("Line was: " + line)
499 elif line.startswith("application:"):
500 apk['name'] = re.match(label_pat, line).group(1)
501 # Keep path to non-dpi icon in case we need it
502 match = re.match(icon_pat_nodpi, line)
504 apk['icons_src']['-1'] = match.group(1)
505 elif line.startswith("launchable-activity:"):
506 # Only use launchable-activity as fallback to application
508 apk['name'] = re.match(label_pat, line).group(1)
509 if '-1' not in apk['icons_src']:
510 match = re.match(icon_pat_nodpi, line)
512 apk['icons_src']['-1'] = match.group(1)
513 elif line.startswith("application-icon-"):
514 match = re.match(icon_pat, line)
516 density = match.group(1)
517 path = match.group(2)
518 apk['icons_src'][density] = path
519 elif line.startswith("sdkVersion:"):
520 m = re.match(sdkversion_pat, line)
522 logging.error(line.replace('sdkVersion:', '')
523 + ' is not a valid minSdkVersion!')
525 apk['sdkversion'] = m.group(1)
526 elif line.startswith("maxSdkVersion:"):
527 apk['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
528 elif line.startswith("native-code:"):
529 apk['nativecode'] = []
530 for arch in line[13:].split(' '):
531 apk['nativecode'].append(arch[1:-1])
532 elif line.startswith("uses-permission:"):
533 perm = re.match(string_pat, line).group(1)
534 if perm.startswith("android.permission."):
536 apk['permissions'].add(perm)
537 elif line.startswith("uses-feature:"):
538 perm = re.match(string_pat, line).group(1)
539 # Filter out this, it's only added with the latest SDK tools and
540 # causes problems for lots of apps.
541 if perm != "android.hardware.screen.portrait" \
542 and perm != "android.hardware.screen.landscape":
543 if perm.startswith("android.feature."):
545 apk['features'].add(perm)
547 if 'sdkversion' not in apk:
548 logging.warn("No SDK version information found in {0}".format(apkfile))
549 apk['sdkversion'] = 0
551 # Check for debuggable apks...
552 if common.isApkDebuggable(apkfile, config):
553 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
555 # Get the signature (or md5 of, to be precise)...
556 logging.debug('Getting signature of {0}'.format(apkfile))
557 apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
559 logging.critical("Failed to get apk signature")
562 apkzip = zipfile.ZipFile(apkfile, 'r')
564 # if an APK has files newer than the system time, suggest updating
565 # the system clock. This is useful for offline systems, used for
566 # signing, which do not have another source of clock sync info. It
567 # has to be more than 24 hours newer because ZIP/APK files do not
568 # store timezone info
569 manifest = apkzip.getinfo('AndroidManifest.xml')
570 dt_obj = datetime(*manifest.date_time)
571 checkdt = dt_obj - timedelta(1)
572 if datetime.today() < checkdt:
573 logging.warn('System clock is older than manifest in: '
574 + apkfilename + '\nSet clock to that time using:\n'
575 + 'sudo date -s "' + str(dt_obj) + '"')
577 iconfilename = "%s.%s.png" % (
581 # Extract the icon file...
583 for density in screen_densities:
584 if density not in apk['icons_src']:
585 empty_densities.append(density)
587 iconsrc = apk['icons_src'][density]
588 icon_dir = get_icon_dir(repodir, density)
589 icondest = os.path.join(icon_dir, iconfilename)
592 with open(icondest, 'wb') as f:
593 f.write(apkzip.read(iconsrc))
594 apk['icons'][density] = iconfilename
597 logging.warn("Error retrieving icon file")
598 del apk['icons'][density]
599 del apk['icons_src'][density]
600 empty_densities.append(density)
602 if '-1' in apk['icons_src']:
603 iconsrc = apk['icons_src']['-1']
604 iconpath = os.path.join(
605 get_icon_dir(repodir, '0'), iconfilename)
606 with open(iconpath, 'wb') as f:
607 f.write(apkzip.read(iconsrc))
609 im = Image.open(iconpath)
610 dpi = px_to_dpi(im.size[0])
611 for density in screen_densities:
612 if density in apk['icons']:
614 if density == screen_densities[-1] or dpi >= int(density):
615 apk['icons'][density] = iconfilename
616 shutil.move(iconpath,
617 os.path.join(get_icon_dir(repodir, density), iconfilename))
618 empty_densities.remove(density)
620 except Exception as e:
621 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
624 apk['icon'] = iconfilename
628 # First try resizing down to not lose quality
630 for density in screen_densities:
631 if density not in empty_densities:
632 last_density = density
634 if last_density is None:
636 logging.debug("Density %s not available, resizing down from %s"
637 % (density, last_density))
639 last_iconpath = os.path.join(
640 get_icon_dir(repodir, last_density), iconfilename)
641 iconpath = os.path.join(
642 get_icon_dir(repodir, density), iconfilename)
644 im = Image.open(last_iconpath)
646 logging.warn("Invalid image file at %s" % last_iconpath)
649 size = dpi_to_px(density)
651 im.thumbnail((size, size), Image.ANTIALIAS)
652 im.save(iconpath, "PNG")
653 empty_densities.remove(density)
655 # Then just copy from the highest resolution available
657 for density in reversed(screen_densities):
658 if density not in empty_densities:
659 last_density = density
661 if last_density is None:
663 logging.debug("Density %s not available, copying from lower density %s"
664 % (density, last_density))
667 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
668 os.path.join(get_icon_dir(repodir, density), iconfilename))
670 empty_densities.remove(density)
672 for density in screen_densities:
673 icon_dir = get_icon_dir(repodir, density)
674 icondest = os.path.join(icon_dir, iconfilename)
675 resize_icon(icondest, density)
677 # Copy from icons-mdpi to icons since mdpi is the baseline density
678 baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
679 if os.path.isfile(baseline):
680 apk['icons']['0'] = iconfilename
681 shutil.copyfile(baseline,
682 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
684 # Record in known apks, getting the added date at the same time..
685 added = knownapks.recordapk(apk['apkname'], apk['id'])
689 apkcache[apkfilename] = apk
694 return apks, cachechanged
697 repo_pubkey_fingerprint = None
700 # Generate a certificate fingerprint the same way keytool does it
701 # (but with slightly different formatting)
702 def cert_fingerprint(data):
703 digest = hashlib.sha256(data).digest()
705 ret.append(' '.join("%02X" % ord(b) for b in digest))
709 def extract_pubkey():
710 global repo_pubkey_fingerprint
711 if 'repo_pubkey' in config:
712 pubkey = unhexlify(config['repo_pubkey'])
714 p = FDroidPopen(['keytool', '-exportcert',
715 '-alias', config['repo_keyalias'],
716 '-keystore', config['keystore'],
717 '-storepass:file', config['keystorepassfile']]
718 + config['smartcardoptions'], output=False)
719 if p.returncode != 0 or len(p.output) < 20:
720 msg = "Failed to get repo pubkey!"
721 if config['keystore'] == 'NONE':
722 msg += ' Is your crypto smartcard plugged in?'
723 logging.critical(msg)
726 repo_pubkey_fingerprint = cert_fingerprint(pubkey)
727 return hexlify(pubkey)
730 def make_index(apps, sortedids, apks, repodir, archive, categories):
731 """Make a repo index.
733 :param apps: fully populated apps list
734 :param apks: full populated apks list
735 :param repodir: the repo directory
736 :param archive: True if this is the archive repo, False if it's the
738 :param categories: list of categories
743 def addElement(name, value, doc, parent):
744 el = doc.createElement(name)
745 el.appendChild(doc.createTextNode(value))
746 parent.appendChild(el)
748 def addElementNonEmpty(name, value, doc, parent):
751 addElement(name, value, doc, parent)
753 def addElementCDATA(name, value, doc, parent):
754 el = doc.createElement(name)
755 el.appendChild(doc.createCDATASection(value))
756 parent.appendChild(el)
758 root = doc.createElement("fdroid")
759 doc.appendChild(root)
761 repoel = doc.createElement("repo")
764 repoel.setAttribute("name", config['archive_name'])
765 if config['repo_maxage'] != 0:
766 repoel.setAttribute("maxage", str(config['repo_maxage']))
767 repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
768 repoel.setAttribute("url", config['archive_url'])
769 addElement('description', config['archive_description'], doc, repoel)
772 repoel.setAttribute("name", config['repo_name'])
773 if config['repo_maxage'] != 0:
774 repoel.setAttribute("maxage", str(config['repo_maxage']))
775 repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
776 repoel.setAttribute("url", config['repo_url'])
777 addElement('description', config['repo_description'], doc, repoel)
779 repoel.setAttribute("version", "14")
780 repoel.setAttribute("timestamp", str(int(time.time())))
783 if not options.nosign:
784 if 'repo_keyalias' not in config:
786 logging.critical("'repo_keyalias' not found in config.py!")
787 if 'keystore' not in config:
789 logging.critical("'keystore' not found in config.py!")
790 if 'keystorepass' not in config and 'keystorepassfile' not in config:
792 logging.critical("'keystorepass' not found in config.py!")
793 if 'keypass' not in config and 'keypassfile' not in config:
795 logging.critical("'keypass' not found in config.py!")
796 if not os.path.exists(config['keystore']):
798 logging.critical("'" + config['keystore'] + "' does not exist!")
800 logging.warning("`fdroid update` requires a signing key, you can create one using:")
801 logging.warning("\tfdroid update --create-key")
804 repoel.setAttribute("pubkey", extract_pubkey())
805 root.appendChild(repoel)
807 for appid in sortedids:
810 if app.Disabled is not None:
813 # Get a list of the apks for this app...
816 if apk['id'] == appid:
819 if len(apklist) == 0:
822 apel = doc.createElement("application")
823 apel.setAttribute("id", app.id)
824 root.appendChild(apel)
826 addElement('id', app.id, doc, apel)
828 addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
830 addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
831 addElement('name', app.Name, doc, apel)
832 addElement('summary', app.Summary, doc, apel)
834 addElement('icon', app.icon, doc, apel)
838 return ("fdroid.app:" + appid, apps[appid].Name)
839 raise MetaDataException("Cannot resolve app id " + appid)
842 metadata.description_html(app.Description, linkres),
844 addElement('license', app.License, doc, apel)
846 addElement('categories', ','.join(app.Categories), doc, apel)
847 # We put the first (primary) category in LAST, which will have
848 # the desired effect of making clients that only understand one
849 # category see that one.
850 addElement('category', app.Categories[0], doc, apel)
851 addElement('web', app.WebSite, doc, apel)
852 addElement('source', app.SourceCode, doc, apel)
853 addElement('tracker', app.IssueTracker, doc, apel)
854 addElementNonEmpty('changelog', app.Changelog, doc, apel)
855 addElementNonEmpty('author', app.AuthorName, doc, apel)
856 addElementNonEmpty('email', app.AuthorEmail, doc, apel)
857 addElementNonEmpty('donate', app.Donate, doc, apel)
858 addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
859 addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
860 addElementNonEmpty('flattr', app.FlattrID, doc, apel)
862 # These elements actually refer to the current version (i.e. which
863 # one is recommended. They are historically mis-named, and need
864 # changing, but stay like this for now to support existing clients.
865 addElement('marketversion', app.CurrentVersion, doc, apel)
866 addElement('marketvercode', app.CurrentVersionCode, doc, apel)
869 af = app.AntiFeatures
871 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
873 pv = app.Provides.split(',')
874 addElementNonEmpty('provides', ','.join(pv), doc, apel)
876 addElement('requirements', 'root', doc, apel)
878 # Sort the apk list into version order, just so the web site
879 # doesn't have to do any work by default...
880 apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
882 # Check for duplicates - they will make the client unhappy...
883 for i in range(len(apklist) - 1):
884 if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
885 logging.critical("duplicate versions: '%s' - '%s'" % (
886 apklist[i]['apkname'], apklist[i + 1]['apkname']))
889 current_version_code = 0
890 current_version_file = None
892 # find the APK for the "Current Version"
893 if current_version_code < apk['versioncode']:
894 current_version_code = apk['versioncode']
895 if current_version_code < int(app.CurrentVersionCode):
896 current_version_file = apk['apkname']
898 apkel = doc.createElement("package")
899 apel.appendChild(apkel)
900 addElement('version', apk['version'], doc, apkel)
901 addElement('versioncode', str(apk['versioncode']), doc, apkel)
902 addElement('apkname', apk['apkname'], doc, apkel)
904 addElement('srcname', apk['srcname'], doc, apkel)
905 for hash_type in ['sha256']:
906 if hash_type not in apk:
908 hashel = doc.createElement("hash")
909 hashel.setAttribute("type", hash_type)
910 hashel.appendChild(doc.createTextNode(apk[hash_type]))
911 apkel.appendChild(hashel)
912 addElement('sig', apk['sig'], doc, apkel)
913 addElement('size', str(apk['size']), doc, apkel)
914 addElement('sdkver', str(apk['sdkversion']), doc, apkel)
915 if 'maxsdkversion' in apk:
916 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
918 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
919 addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
920 if 'nativecode' in apk:
921 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
922 addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
924 if current_version_file is not None \
925 and config['make_current_version_link'] \
926 and repodir == 'repo': # only create these
927 namefield = config['current_version_name_source']
928 sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
929 apklinkname = sanitized_name + '.apk'
930 current_version_path = os.path.join(repodir, current_version_file)
931 if os.path.islink(apklinkname):
932 os.remove(apklinkname)
933 os.symlink(current_version_path, apklinkname)
934 # also symlink gpg signature, if it exists
935 for extension in ('.asc', '.sig'):
936 sigfile_path = current_version_path + extension
937 if os.path.exists(sigfile_path):
938 siglinkname = apklinkname + extension
939 if os.path.islink(siglinkname):
940 os.remove(siglinkname)
941 os.symlink(sigfile_path, siglinkname)
944 output = doc.toprettyxml()
948 with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
951 if 'repo_keyalias' in config:
954 logging.info("Creating unsigned index in preparation for signing")
956 logging.info("Creating signed index with this key (SHA256):")
957 logging.info("%s" % repo_pubkey_fingerprint)
959 # Create a jar of the index...
960 jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
961 p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
962 if p.returncode != 0:
963 logging.critical("Failed to create {0}".format(jar_output))
967 signed = os.path.join(repodir, 'index.jar')
969 # Remove old signed index if not signing
970 if os.path.exists(signed):
973 args = ['jarsigner', '-keystore', config['keystore'],
974 '-storepass:file', config['keystorepassfile'],
975 '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
976 signed, config['repo_keyalias']]
977 if config['keystore'] == 'NONE':
978 args += config['smartcardoptions']
979 else: # smardcards never use -keypass
980 args += ['-keypass:file', config['keypassfile']]
981 p = FDroidPopen(args)
982 if p.returncode != 0:
983 logging.critical("Failed to sign index")
986 # Copy the repo icon into the repo directory...
987 icon_dir = os.path.join(repodir, 'icons')
988 iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
989 shutil.copyfile(config['repo_icon'], iconfilename)
991 # Write a category list in the repo to allow quick access...
993 for cat in categories:
994 catdata += cat + '\n'
995 with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
999 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1001 for appid, app in apps.iteritems():
1003 if app.ArchivePolicy:
1004 keepversions = int(app.ArchivePolicy[:-9])
1006 keepversions = defaultkeepversions
1008 def filter_apk_list_sorted(apk_list):
1010 for apk in apk_list:
1011 if apk['id'] == appid:
1014 # Sort the apk list by version code. First is highest/newest.
1015 return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1017 def move_file(from_dir, to_dir, filename, ignore_missing):
1018 from_path = os.path.join(from_dir, filename)
1019 if ignore_missing and not os.path.exists(from_path):
1021 to_path = os.path.join(to_dir, filename)
1022 shutil.move(from_path, to_path)
1024 if len(apks) > keepversions:
1025 apklist = filter_apk_list_sorted(apks)
1026 # Move back the ones we don't want.
1027 for apk in apklist[keepversions:]:
1028 logging.info("Moving " + apk['apkname'] + " to archive")
1029 move_file(repodir, archivedir, apk['apkname'], False)
1030 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1031 for density in all_screen_densities:
1032 repo_icon_dir = get_icon_dir(repodir, density)
1033 archive_icon_dir = get_icon_dir(archivedir, density)
1034 if density not in apk['icons']:
1036 move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1037 if 'srcname' in apk:
1038 move_file(repodir, archivedir, apk['srcname'], False)
1039 archapks.append(apk)
1041 elif len(apks) < keepversions and len(archapks) > 0:
1042 required = keepversions - len(apks)
1043 archapklist = filter_apk_list_sorted(archapks)
1044 # Move forward the ones we want again.
1045 for apk in archapklist[:required]:
1046 logging.info("Moving " + apk['apkname'] + " from archive")
1047 move_file(archivedir, repodir, apk['apkname'], False)
1048 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1049 for density in all_screen_densities:
1050 repo_icon_dir = get_icon_dir(repodir, density)
1051 archive_icon_dir = get_icon_dir(archivedir, density)
1052 if density not in apk['icons']:
1054 move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1055 if 'srcname' in apk:
1056 move_file(archivedir, repodir, apk['srcname'], False)
1057 archapks.remove(apk)
1061 def add_apks_to_per_app_repos(repodir, apks):
1062 apks_per_app = dict()
1064 apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1065 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1066 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1067 apks_per_app[apk['id']] = apk
1069 if not os.path.exists(apk['per_app_icons']):
1070 logging.info('Adding new repo for only ' + apk['id'])
1071 os.makedirs(apk['per_app_icons'])
1073 apkpath = os.path.join(repodir, apk['apkname'])
1074 shutil.copy(apkpath, apk['per_app_repo'])
1075 apksigpath = apkpath + '.sig'
1076 if os.path.exists(apksigpath):
1077 shutil.copy(apksigpath, apk['per_app_repo'])
1078 apkascpath = apkpath + '.asc'
1079 if os.path.exists(apkascpath):
1080 shutil.copy(apkascpath, apk['per_app_repo'])
1089 global config, options
1091 # Parse command line...
1092 parser = ArgumentParser()
1093 common.setup_global_opts(parser)
1094 parser.add_argument("--create-key", action="store_true", default=False,
1095 help="Create a repo signing key in a keystore")
1096 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1097 help="Create skeleton metadata files that are missing")
1098 parser.add_argument("--delete-unknown", action="store_true", default=False,
1099 help="Delete APKs without metadata from the repo")
1100 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1101 help="Report on build data status")
1102 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1103 help="Interactively ask about things that need updating.")
1104 parser.add_argument("-I", "--icons", action="store_true", default=False,
1105 help="Resize all the icons exceeding the max pixel size and exit")
1106 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1107 help="Specify editor to use in interactive mode. Default " +
1108 "is /etc/alternatives/editor")
1109 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1110 help="Update the wiki")
1111 parser.add_argument("--pretty", action="store_true", default=False,
1112 help="Produce human-readable index.xml")
1113 parser.add_argument("--clean", action="store_true", default=False,
1114 help="Clean update - don't uses caches, reprocess all apks")
1115 parser.add_argument("--nosign", action="store_true", default=False,
1116 help="When configured for signed indexes, create only unsigned indexes at this stage")
1117 options = parser.parse_args()
1119 config = common.read_config(options)
1122 if config['archive_older'] != 0:
1123 repodirs.append('archive')
1124 if not os.path.exists('archive'):
1128 resize_all_icons(repodirs)
1131 # check that icons exist now, rather than fail at the end of `fdroid update`
1132 for k in ['repo_icon', 'archive_icon']:
1134 if not os.path.exists(config[k]):
1135 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1138 # if the user asks to create a keystore, do it now, reusing whatever it can
1139 if options.create_key:
1140 if os.path.exists(config['keystore']):
1141 logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1142 logging.critical("\t'" + config['keystore'] + "'")
1145 if 'repo_keyalias' not in config:
1146 config['repo_keyalias'] = socket.getfqdn()
1147 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1148 if 'keydname' not in config:
1149 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1150 common.write_to_config(config, 'keydname', config['keydname'])
1151 if 'keystore' not in config:
1152 config['keystore'] = common.default_config.keystore
1153 common.write_to_config(config, 'keystore', config['keystore'])
1155 password = common.genpassword()
1156 if 'keystorepass' not in config:
1157 config['keystorepass'] = password
1158 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1159 if 'keypass' not in config:
1160 config['keypass'] = password
1161 common.write_to_config(config, 'keypass', config['keypass'])
1162 common.genkeystore(config)
1165 apps = metadata.read_metadata()
1167 # Generate a list of categories...
1169 for app in apps.itervalues():
1170 categories.update(app.Categories)
1172 # Read known apks data (will be updated and written back when we've finished)
1173 knownapks = common.KnownApks()
1175 # Gather information about all the apk files in the repo directory, using
1176 # cached data if possible.
1177 apkcachefile = os.path.join('tmp', 'apkcache')
1178 if not options.clean and os.path.exists(apkcachefile):
1179 with open(apkcachefile, 'rb') as cf:
1180 apkcache = pickle.load(cf)
1184 delete_disabled_builds(apps, apkcache, repodirs)
1186 # Scan all apks in the main repo
1187 apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks)
1189 # Generate warnings for apk's with no metadata (or create skeleton
1190 # metadata files, if requested on the command line)
1193 if apk['id'] not in apps:
1194 if options.create_metadata:
1195 if 'name' not in apk:
1196 logging.error(apk['id'] + ' does not have a name! Skipping...')
1198 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1199 f.write("License:Unknown\n")
1200 f.write("Web Site:\n")
1201 f.write("Source Code:\n")
1202 f.write("Issue Tracker:\n")
1203 f.write("Changelog:\n")
1204 f.write("Summary:" + apk['name'] + "\n")
1205 f.write("Description:\n")
1206 f.write(apk['name'] + "\n")
1209 logging.info("Generated skeleton metadata for " + apk['id'])
1212 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1213 if options.delete_unknown:
1214 logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1215 rmf = os.path.join(repodirs[0], apk['apkname'])
1216 if not os.path.exists(rmf):
1217 logging.error("Could not find {0} to remove it".format(rmf))
1221 logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1223 # update the metadata with the newly created ones included
1225 apps = metadata.read_metadata()
1227 # Scan the archive repo for apks as well
1228 if len(repodirs) > 1:
1229 archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1235 # Some information from the apks needs to be applied up to the application
1236 # level. When doing this, we use the info from the most recent version's apk.
1237 # We deal with figuring out when the app was added and last updated at the
1239 for appid, app in apps.iteritems():
1241 for apk in apks + archapks:
1242 if apk['id'] == appid:
1243 if apk['versioncode'] > bestver:
1244 bestver = apk['versioncode']
1248 if not app.added or apk['added'] < app.added:
1249 app.added = apk['added']
1250 if not app.lastupdated or apk['added'] > app.lastupdated:
1251 app.lastupdated = apk['added']
1254 logging.debug("Don't know when " + appid + " was added")
1255 if not app.lastupdated:
1256 logging.debug("Don't know when " + appid + " was last updated")
1259 if app.Name is None:
1260 app.Name = app.AutoName or appid
1262 logging.debug("Application " + appid + " has no packages")
1264 if app.Name is None:
1265 app.Name = bestapk['name']
1266 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1268 # Sort the app list by name, then the web site doesn't have to by default.
1269 # (we had to wait until we'd scanned the apks to do this, because mostly the
1270 # name comes from there!)
1271 sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid].Name.upper())
1273 # APKs are placed into multiple repos based on the app package, providing
1274 # per-app subscription feeds for nightly builds and things like it
1275 if config['per_app_repos']:
1276 add_apks_to_per_app_repos(repodirs[0], apks)
1277 for appid, app in apps.iteritems():
1278 repodir = os.path.join(appid, 'fdroid', 'repo')
1280 appdict[appid] = app
1281 if os.path.isdir(repodir):
1282 make_index(appdict, [appid], apks, repodir, False, categories)
1284 logging.info('Skipping index generation for ' + appid)
1287 if len(repodirs) > 1:
1288 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1290 # Make the index for the main repo...
1291 make_index(apps, sortedids, apks, repodirs[0], False, categories)
1293 # If there's an archive repo, make the index for it. We already scanned it
1295 if len(repodirs) > 1:
1296 make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1298 if config['update_stats']:
1300 # Update known apks info...
1301 knownapks.writeifchanged()
1303 # Generate latest apps data for widget
1304 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1306 for line in file(os.path.join('stats', 'latestapps.txt')):
1307 appid = line.rstrip()
1308 data += appid + "\t"
1310 data += app.Name + "\t"
1311 if app.icon is not None:
1312 data += app.icon + "\t"
1313 data += app.License + "\n"
1314 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
1318 with open(apkcachefile, 'wb') as cf:
1319 pickle.dump(apkcache, cf)
1321 # Update the wiki...
1323 update_wiki(apps, sortedids, apks + archapks)
1325 logging.info("Finished.")
1327 if __name__ == "__main__":