2 # -*- coding: utf-8 -*-
4 # update.py - part of the FDroid server tools
5 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
6 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU Affero General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU Affero General Public License for more details.
18 # You should have received a copy of the GNU Affero General Public License
19 # along with this program. If not, see <http://www.gnu.org/licenses/>.
30 from datetime import datetime, timedelta
31 from xml.dom.minidom import Document
32 from argparse import ArgumentParser
34 from pyasn1.error import PyAsn1Error
35 from pyasn1.codec.der import decoder, encoder
36 from pyasn1_modules import rfc2315
37 from hashlib import md5
38 from binascii import hexlify, unhexlify
45 from common import FDroidPopen, SdkToolsPopen
46 from metadata import MetaDataException
48 screen_densities = ['640', '480', '320', '240', '160', '120']
50 all_screen_densities = ['0'] + screen_densities
53 def dpi_to_px(density):
54 return (int(density) * 48) / 160
58 return (int(px) * 160) / 48
61 def get_icon_dir(repodir, density):
63 return os.path.join(repodir, "icons")
64 return os.path.join(repodir, "icons-%s" % density)
67 def get_icon_dirs(repodir):
68 for density in screen_densities:
69 yield get_icon_dir(repodir, density)
72 def get_all_icon_dirs(repodir):
73 for density in all_screen_densities:
74 yield get_icon_dir(repodir, density)
77 def update_wiki(apps, sortedids, apks):
80 :param apps: fully populated list of all applications
81 :param apks: all apks, except...
83 logging.info("Updating wiki")
85 wikiredircat = 'App Redirects'
87 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
88 path=config['wiki_path'])
89 site.login(config['wiki_user'], config['wiki_password'])
91 generated_redirects = {}
93 for appid in sortedids:
98 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
100 for af in app.AntiFeatures:
101 wikidata += '{{AntiFeature|' + af + '}}\n'
106 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s}}\n' % (
109 time.strftime('%Y-%m-%d', app.added) if app.added else '',
110 time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
123 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
125 wikidata += app.Summary
126 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
128 wikidata += "=Description=\n"
129 wikidata += metadata.description_wiki(app.Description) + "\n"
131 wikidata += "=Maintainer Notes=\n"
132 if app.MaintainerNotes:
133 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
134 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
136 # Get a list of all packages for this application...
138 gotcurrentver = False
142 if apk['id'] == appid:
143 if str(apk['versioncode']) == app.CurrentVersionCode:
146 # Include ones we can't build, as a special case...
147 for build in app.builds:
149 if build.vercode == app.CurrentVersionCode:
151 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
152 apklist.append({'versioncode': int(build.vercode),
153 'version': build.version,
154 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
159 if apk['versioncode'] == int(build.vercode):
164 apklist.append({'versioncode': int(build.vercode),
165 'version': build.version,
166 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
168 if app.CurrentVersionCode == '0':
170 # Sort with most recent first...
171 apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
173 wikidata += "=Versions=\n"
174 if len(apklist) == 0:
175 wikidata += "We currently have no versions of this app available."
176 elif not gotcurrentver:
177 wikidata += "We don't have the current version of this app."
179 wikidata += "We have the current version of this app."
180 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
181 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
182 if len(app.NoSourceSince) > 0:
183 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
184 if len(app.CurrentVersion) > 0:
185 wikidata += "The current (recommended) version is " + app.CurrentVersion
186 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
189 wikidata += "==" + apk['version'] + "==\n"
191 if 'buildproblem' in apk:
192 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
195 wikidata += "This version is built and signed by "
197 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
199 wikidata += "the original developer.\n\n"
200 wikidata += "Version code: " + str(apk['versioncode']) + '\n'
202 wikidata += '\n[[Category:' + wikicat + ']]\n'
203 if len(app.NoSourceSince) > 0:
204 wikidata += '\n[[Category:Apps missing source code]]\n'
205 if validapks == 0 and not app.Disabled:
206 wikidata += '\n[[Category:Apps with no packages]]\n'
207 if cantupdate and not app.Disabled:
208 wikidata += "\n[[Category:Apps we can't update]]\n"
209 if buildfails and not app.Disabled:
210 wikidata += "\n[[Category:Apps with failing builds]]\n"
211 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
212 wikidata += '\n[[Category:Apps to Update]]\n'
214 wikidata += '\n[[Category:Apps that are disabled]]\n'
215 if app.UpdateCheckMode == 'None' and not app.Disabled:
216 wikidata += '\n[[Category:Apps with no update check]]\n'
217 for appcat in app.Categories:
218 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
220 # We can't have underscores in the page name, even if they're in
221 # the package ID, because MediaWiki messes with them...
222 pagename = appid.replace('_', ' ')
224 # Drop a trailing newline, because mediawiki is going to drop it anyway
225 # and it we don't we'll think the page has changed when it hasn't...
226 if wikidata.endswith('\n'):
227 wikidata = wikidata[:-1]
229 generated_pages[pagename] = wikidata
231 # Make a redirect from the name to the ID too, unless there's
232 # already an existing page with the name and it isn't a redirect.
234 apppagename = app.Name.replace('_', ' ')
235 apppagename = apppagename.replace('{', '')
236 apppagename = apppagename.replace('}', ' ')
237 apppagename = apppagename.replace(':', ' ')
238 # Drop double spaces caused mostly by replacing ':' above
239 apppagename = apppagename.replace(' ', ' ')
240 for expagename in site.allpages(prefix=apppagename,
241 filterredir='nonredirects',
243 if expagename == apppagename:
245 # Another reason not to make the redirect page is if the app name
246 # is the same as it's ID, because that will overwrite the real page
247 # with an redirect to itself! (Although it seems like an odd
248 # scenario this happens a lot, e.g. where there is metadata but no
249 # builds or binaries to extract a name from.
250 if apppagename == pagename:
253 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
255 for tcat, genp in [(wikicat, generated_pages),
256 (wikiredircat, generated_redirects)]:
257 catpages = site.Pages['Category:' + tcat]
259 for page in catpages:
260 existingpages.append(page.name)
261 if page.name in genp:
262 pagetxt = page.edit()
263 if pagetxt != genp[page.name]:
264 logging.debug("Updating modified page " + page.name)
265 page.save(genp[page.name], summary='Auto-updated')
267 logging.debug("Page " + page.name + " is unchanged")
269 logging.warn("Deleting page " + page.name)
270 page.delete('No longer published')
271 for pagename, text in genp.items():
272 logging.debug("Checking " + pagename)
273 if pagename not in existingpages:
274 logging.debug("Creating page " + pagename)
276 newpage = site.Pages[pagename]
277 newpage.save(text, summary='Auto-created')
279 logging.error("...FAILED to create page '{0}'".format(pagename))
281 # Purge server cache to ensure counts are up to date
282 site.pages['Repository Maintenance'].purge()
285 def delete_disabled_builds(apps, apkcache, repodirs):
286 """Delete disabled build outputs.
288 :param apps: list of all applications, as per metadata.read_metadata
289 :param apkcache: current apk cache information
290 :param repodirs: the repo directories to process
292 for appid, app in apps.iteritems():
293 for build in app.builds:
294 if not build.disable:
296 apkfilename = appid + '_' + str(build.vercode) + '.apk'
297 iconfilename = "%s.%s.png" % (
300 for repodir in repodirs:
302 os.path.join(repodir, apkfilename),
303 os.path.join(repodir, apkfilename + '.asc'),
304 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
306 for density in all_screen_densities:
307 repo_dir = get_icon_dir(repodir, density)
308 files.append(os.path.join(repo_dir, iconfilename))
311 if os.path.exists(f):
312 logging.info("Deleting disabled build output " + f)
314 if apkfilename in apkcache:
315 del apkcache[apkfilename]
318 def resize_icon(iconpath, density):
320 if not os.path.isfile(iconpath):
324 im = Image.open(iconpath)
325 size = dpi_to_px(density)
327 if any(length > size for length in im.size):
329 im.thumbnail((size, size), Image.ANTIALIAS)
330 logging.debug("%s was too large at %s - new size is %s" % (
331 iconpath, oldsize, im.size))
332 im.save(iconpath, "PNG")
334 except Exception as e:
335 logging.error("Failed resizing {0} - {1}".format(iconpath, e))
338 def resize_all_icons(repodirs):
339 """Resize all icons that exceed the max size
341 :param repodirs: the repo directories to process
343 for repodir in repodirs:
344 for density in screen_densities:
345 icon_dir = get_icon_dir(repodir, density)
346 icon_glob = os.path.join(icon_dir, '*.png')
347 for iconpath in glob.glob(icon_glob):
348 resize_icon(iconpath, density)
351 # A signature block file with a .DSA, .RSA, or .EC extension
352 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
356 """ Get the signing certificate of an apk. To get the same md5 has that
357 Android gets, we encode the .RSA certificate in a specific format and pass
358 it hex-encoded to the md5 digest algorithm.
360 :param apkpath: path to the apk
361 :returns: A string containing the md5 of the signature of the apk or None
362 if an error occurred.
367 # verify the jar signature is correct
368 args = ['jarsigner', '-verify', apkpath]
369 p = FDroidPopen(args)
370 if p.returncode != 0:
371 logging.critical(apkpath + " has a bad signature!")
374 with zipfile.ZipFile(apkpath, 'r') as apk:
376 certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
379 logging.error("Found no signing certificates on %s" % apkpath)
382 logging.error("Found multiple signing certificates on %s" % apkpath)
385 cert = apk.read(certs[0])
387 content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
388 if content.getComponentByName('contentType') != rfc2315.signedData:
389 logging.error("Unexpected format.")
392 content = decoder.decode(content.getComponentByName('content'),
393 asn1Spec=rfc2315.SignedData())[0]
395 certificates = content.getComponentByName('certificates')
397 logging.error("Certificates not found.")
400 cert_encoded = encoder.encode(certificates)[4:]
402 return md5(cert_encoded.encode('hex')).hexdigest()
405 def scan_apks(apps, apkcache, repodir, knownapks):
406 """Scan the apks in the given repo directory.
408 This also extracts the icons.
410 :param apps: list of all applications, as per metadata.read_metadata
411 :param apkcache: current apk cache information
412 :param repodir: repo directory to scan
413 :param knownapks: known apks info
414 :returns: (apks, cachechanged) where apks is a list of apk information,
415 and cachechanged is True if the apkcache got changed.
420 for icon_dir in get_all_icon_dirs(repodir):
421 if os.path.exists(icon_dir):
423 shutil.rmtree(icon_dir)
424 os.makedirs(icon_dir)
426 os.makedirs(icon_dir)
429 name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
430 vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
431 vername_pat = re.compile(".*versionName='([^']*)'.*")
432 label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
433 icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
434 icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
435 sdkversion_pat = re.compile(".*'([0-9]*)'.*")
436 string_pat = re.compile(".*'([^']*)'.*")
437 for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
439 apkfilename = apkfile[len(repodir) + 1:]
440 if ' ' in apkfilename:
441 logging.critical("Spaces in filenames are not allowed.")
444 # Calculate the sha256...
445 sha = hashlib.sha256()
446 with open(apkfile, 'rb') as f:
452 shasum = sha.hexdigest()
455 if apkfilename in apkcache:
456 apk = apkcache[apkfilename]
457 if apk['sha256'] == shasum:
458 logging.debug("Reading " + apkfilename + " from cache")
461 logging.debug("Ignoring stale cache data for " + apkfilename)
464 logging.debug("Processing " + apkfilename)
466 apk['apkname'] = apkfilename
467 apk['sha256'] = shasum
468 srcfilename = apkfilename[:-4] + "_src.tar.gz"
469 if os.path.exists(os.path.join(repodir, srcfilename)):
470 apk['srcname'] = srcfilename
471 apk['size'] = os.path.getsize(apkfile)
472 apk['permissions'] = set()
473 apk['features'] = set()
474 apk['icons_src'] = {}
476 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
477 if p.returncode != 0:
478 if options.delete_unknown:
479 if os.path.exists(apkfile):
480 logging.error("Failed to get apk information, deleting " + apkfile)
483 logging.error("Could not find {0} to remove it".format(apkfile))
485 logging.error("Failed to get apk information, skipping " + apkfile)
487 for line in p.output.splitlines():
488 if line.startswith("package:"):
490 apk['id'] = re.match(name_pat, line).group(1)
491 apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
492 apk['version'] = re.match(vername_pat, line).group(1)
493 except Exception as e:
494 logging.error("Package matching failed: " + str(e))
495 logging.info("Line was: " + line)
497 elif line.startswith("application:"):
498 apk['name'] = re.match(label_pat, line).group(1)
499 # Keep path to non-dpi icon in case we need it
500 match = re.match(icon_pat_nodpi, line)
502 apk['icons_src']['-1'] = match.group(1)
503 elif line.startswith("launchable-activity:"):
504 # Only use launchable-activity as fallback to application
506 apk['name'] = re.match(label_pat, line).group(1)
507 if '-1' not in apk['icons_src']:
508 match = re.match(icon_pat_nodpi, line)
510 apk['icons_src']['-1'] = match.group(1)
511 elif line.startswith("application-icon-"):
512 match = re.match(icon_pat, line)
514 density = match.group(1)
515 path = match.group(2)
516 apk['icons_src'][density] = path
517 elif line.startswith("sdkVersion:"):
518 m = re.match(sdkversion_pat, line)
520 logging.error(line.replace('sdkVersion:', '')
521 + ' is not a valid minSdkVersion!')
523 apk['sdkversion'] = m.group(1)
524 elif line.startswith("maxSdkVersion:"):
525 apk['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
526 elif line.startswith("native-code:"):
527 apk['nativecode'] = []
528 for arch in line[13:].split(' '):
529 apk['nativecode'].append(arch[1:-1])
530 elif line.startswith("uses-permission:"):
531 perm = re.match(string_pat, line).group(1)
532 if perm.startswith("android.permission."):
534 apk['permissions'].add(perm)
535 elif line.startswith("uses-feature:"):
536 perm = re.match(string_pat, line).group(1)
537 # Filter out this, it's only added with the latest SDK tools and
538 # causes problems for lots of apps.
539 if perm != "android.hardware.screen.portrait" \
540 and perm != "android.hardware.screen.landscape":
541 if perm.startswith("android.feature."):
543 apk['features'].add(perm)
545 if 'sdkversion' not in apk:
546 logging.warn("No SDK version information found in {0}".format(apkfile))
547 apk['sdkversion'] = 0
549 # Check for debuggable apks...
550 if common.isApkDebuggable(apkfile, config):
551 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
553 # Get the signature (or md5 of, to be precise)...
554 logging.debug('Getting signature of {0}'.format(apkfile))
555 apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
557 logging.critical("Failed to get apk signature")
560 apkzip = zipfile.ZipFile(apkfile, 'r')
562 # if an APK has files newer than the system time, suggest updating
563 # the system clock. This is useful for offline systems, used for
564 # signing, which do not have another source of clock sync info. It
565 # has to be more than 24 hours newer because ZIP/APK files do not
566 # store timezone info
567 manifest = apkzip.getinfo('AndroidManifest.xml')
568 dt_obj = datetime(*manifest.date_time)
569 checkdt = dt_obj - timedelta(1)
570 if datetime.today() < checkdt:
571 logging.warn('System clock is older than manifest in: '
572 + apkfilename + '\nSet clock to that time using:\n'
573 + 'sudo date -s "' + str(dt_obj) + '"')
575 iconfilename = "%s.%s.png" % (
579 # Extract the icon file...
581 for density in screen_densities:
582 if density not in apk['icons_src']:
583 empty_densities.append(density)
585 iconsrc = apk['icons_src'][density]
586 icon_dir = get_icon_dir(repodir, density)
587 icondest = os.path.join(icon_dir, iconfilename)
590 with open(icondest, 'wb') as f:
591 f.write(apkzip.read(iconsrc))
592 apk['icons'][density] = iconfilename
595 logging.warn("Error retrieving icon file")
596 del apk['icons'][density]
597 del apk['icons_src'][density]
598 empty_densities.append(density)
600 if '-1' in apk['icons_src']:
601 iconsrc = apk['icons_src']['-1']
602 iconpath = os.path.join(
603 get_icon_dir(repodir, '0'), iconfilename)
604 with open(iconpath, 'wb') as f:
605 f.write(apkzip.read(iconsrc))
607 im = Image.open(iconpath)
608 dpi = px_to_dpi(im.size[0])
609 for density in screen_densities:
610 if density in apk['icons']:
612 if density == screen_densities[-1] or dpi >= int(density):
613 apk['icons'][density] = iconfilename
614 shutil.move(iconpath,
615 os.path.join(get_icon_dir(repodir, density), iconfilename))
616 empty_densities.remove(density)
618 except Exception as e:
619 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
622 apk['icon'] = iconfilename
626 # First try resizing down to not lose quality
628 for density in screen_densities:
629 if density not in empty_densities:
630 last_density = density
632 if last_density is None:
634 logging.debug("Density %s not available, resizing down from %s"
635 % (density, last_density))
637 last_iconpath = os.path.join(
638 get_icon_dir(repodir, last_density), iconfilename)
639 iconpath = os.path.join(
640 get_icon_dir(repodir, density), iconfilename)
642 im = Image.open(last_iconpath)
644 logging.warn("Invalid image file at %s" % last_iconpath)
647 size = dpi_to_px(density)
649 im.thumbnail((size, size), Image.ANTIALIAS)
650 im.save(iconpath, "PNG")
651 empty_densities.remove(density)
653 # Then just copy from the highest resolution available
655 for density in reversed(screen_densities):
656 if density not in empty_densities:
657 last_density = density
659 if last_density is None:
661 logging.debug("Density %s not available, copying from lower density %s"
662 % (density, last_density))
665 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
666 os.path.join(get_icon_dir(repodir, density), iconfilename))
668 empty_densities.remove(density)
670 for density in screen_densities:
671 icon_dir = get_icon_dir(repodir, density)
672 icondest = os.path.join(icon_dir, iconfilename)
673 resize_icon(icondest, density)
675 # Copy from icons-mdpi to icons since mdpi is the baseline density
676 baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
677 if os.path.isfile(baseline):
678 apk['icons']['0'] = iconfilename
679 shutil.copyfile(baseline,
680 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
682 # Record in known apks, getting the added date at the same time..
683 added = knownapks.recordapk(apk['apkname'], apk['id'])
687 apkcache[apkfilename] = apk
692 return apks, cachechanged
695 repo_pubkey_fingerprint = None
698 # Generate a certificate fingerprint the same way keytool does it
699 # (but with slightly different formatting)
700 def cert_fingerprint(data):
701 digest = hashlib.sha256(data).digest()
703 ret.append(' '.join("%02X" % ord(b) for b in digest))
707 def extract_pubkey():
708 global repo_pubkey_fingerprint
709 if 'repo_pubkey' in config:
710 pubkey = unhexlify(config['repo_pubkey'])
712 p = FDroidPopen(['keytool', '-exportcert',
713 '-alias', config['repo_keyalias'],
714 '-keystore', config['keystore'],
715 '-storepass:file', config['keystorepassfile']]
716 + config['smartcardoptions'], output=False)
717 if p.returncode != 0 or len(p.output) < 20:
718 msg = "Failed to get repo pubkey!"
719 if config['keystore'] == 'NONE':
720 msg += ' Is your crypto smartcard plugged in?'
721 logging.critical(msg)
724 repo_pubkey_fingerprint = cert_fingerprint(pubkey)
725 return hexlify(pubkey)
728 def make_index(apps, sortedids, apks, repodir, archive, categories):
729 """Make a repo index.
731 :param apps: fully populated apps list
732 :param apks: full populated apks list
733 :param repodir: the repo directory
734 :param archive: True if this is the archive repo, False if it's the
736 :param categories: list of categories
741 def addElement(name, value, doc, parent):
742 el = doc.createElement(name)
743 el.appendChild(doc.createTextNode(value))
744 parent.appendChild(el)
746 def addElementNonEmpty(name, value, doc, parent):
749 addElement(name, value, doc, parent)
751 def addElementCDATA(name, value, doc, parent):
752 el = doc.createElement(name)
753 el.appendChild(doc.createCDATASection(value))
754 parent.appendChild(el)
756 root = doc.createElement("fdroid")
757 doc.appendChild(root)
759 repoel = doc.createElement("repo")
762 repoel.setAttribute("name", config['archive_name'])
763 if config['repo_maxage'] != 0:
764 repoel.setAttribute("maxage", str(config['repo_maxage']))
765 repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
766 repoel.setAttribute("url", config['archive_url'])
767 addElement('description', config['archive_description'], doc, repoel)
770 repoel.setAttribute("name", config['repo_name'])
771 if config['repo_maxage'] != 0:
772 repoel.setAttribute("maxage", str(config['repo_maxage']))
773 repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
774 repoel.setAttribute("url", config['repo_url'])
775 addElement('description', config['repo_description'], doc, repoel)
777 repoel.setAttribute("version", "14")
778 repoel.setAttribute("timestamp", str(int(time.time())))
781 if not options.nosign:
782 if 'repo_keyalias' not in config:
784 logging.critical("'repo_keyalias' not found in config.py!")
785 if 'keystore' not in config:
787 logging.critical("'keystore' not found in config.py!")
788 if 'keystorepass' not in config and 'keystorepassfile' not in config:
790 logging.critical("'keystorepass' not found in config.py!")
791 if 'keypass' not in config and 'keypassfile' not in config:
793 logging.critical("'keypass' not found in config.py!")
794 if not os.path.exists(config['keystore']):
796 logging.critical("'" + config['keystore'] + "' does not exist!")
798 logging.warning("`fdroid update` requires a signing key, you can create one using:")
799 logging.warning("\tfdroid update --create-key")
802 repoel.setAttribute("pubkey", extract_pubkey())
803 root.appendChild(repoel)
805 for appid in sortedids:
808 if app.Disabled is not None:
811 # Get a list of the apks for this app...
814 if apk['id'] == appid:
817 if len(apklist) == 0:
820 apel = doc.createElement("application")
821 apel.setAttribute("id", app.id)
822 root.appendChild(apel)
824 addElement('id', app.id, doc, apel)
826 addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
828 addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
829 addElement('name', app.Name, doc, apel)
830 addElement('summary', app.Summary, doc, apel)
832 addElement('icon', app.icon, doc, apel)
836 return ("fdroid.app:" + appid, apps[appid].Name)
837 raise MetaDataException("Cannot resolve app id " + appid)
840 metadata.description_html(app.Description, linkres),
842 addElement('license', app.License, doc, apel)
844 addElement('categories', ','.join(app.Categories), doc, apel)
845 # We put the first (primary) category in LAST, which will have
846 # the desired effect of making clients that only understand one
847 # category see that one.
848 addElement('category', app.Categories[0], doc, apel)
849 addElement('web', app.WebSite, doc, apel)
850 addElement('source', app.SourceCode, doc, apel)
851 addElement('tracker', app.IssueTracker, doc, apel)
852 addElementNonEmpty('changelog', app.Changelog, doc, apel)
853 addElementNonEmpty('donate', app.Donate, doc, apel)
854 addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
855 addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
856 addElementNonEmpty('flattr', app.FlattrID, doc, apel)
858 # These elements actually refer to the current version (i.e. which
859 # one is recommended. They are historically mis-named, and need
860 # changing, but stay like this for now to support existing clients.
861 addElement('marketversion', app.CurrentVersion, doc, apel)
862 addElement('marketvercode', app.CurrentVersionCode, doc, apel)
865 af = app.AntiFeatures
867 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
869 pv = app.Provides.split(',')
870 addElementNonEmpty('provides', ','.join(pv), doc, apel)
872 addElement('requirements', 'root', doc, apel)
874 # Sort the apk list into version order, just so the web site
875 # doesn't have to do any work by default...
876 apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
878 # Check for duplicates - they will make the client unhappy...
879 for i in range(len(apklist) - 1):
880 if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
881 logging.critical("duplicate versions: '%s' - '%s'" % (
882 apklist[i]['apkname'], apklist[i + 1]['apkname']))
885 current_version_code = 0
886 current_version_file = None
888 # find the APK for the "Current Version"
889 if current_version_code < apk['versioncode']:
890 current_version_code = apk['versioncode']
891 if current_version_code < int(app.CurrentVersionCode):
892 current_version_file = apk['apkname']
894 apkel = doc.createElement("package")
895 apel.appendChild(apkel)
896 addElement('version', apk['version'], doc, apkel)
897 addElement('versioncode', str(apk['versioncode']), doc, apkel)
898 addElement('apkname', apk['apkname'], doc, apkel)
900 addElement('srcname', apk['srcname'], doc, apkel)
901 for hash_type in ['sha256']:
902 if hash_type not in apk:
904 hashel = doc.createElement("hash")
905 hashel.setAttribute("type", hash_type)
906 hashel.appendChild(doc.createTextNode(apk[hash_type]))
907 apkel.appendChild(hashel)
908 addElement('sig', apk['sig'], doc, apkel)
909 addElement('size', str(apk['size']), doc, apkel)
910 addElement('sdkver', str(apk['sdkversion']), doc, apkel)
911 if 'maxsdkversion' in apk:
912 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
914 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
915 addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
916 if 'nativecode' in apk:
917 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
918 addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
920 if current_version_file is not None \
921 and config['make_current_version_link'] \
922 and repodir == 'repo': # only create these
923 namefield = config['current_version_name_source']
924 sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
925 apklinkname = sanitized_name + '.apk'
926 current_version_path = os.path.join(repodir, current_version_file)
927 if os.path.islink(apklinkname):
928 os.remove(apklinkname)
929 os.symlink(current_version_path, apklinkname)
930 # also symlink gpg signature, if it exists
931 for extension in ('.asc', '.sig'):
932 sigfile_path = current_version_path + extension
933 if os.path.exists(sigfile_path):
934 siglinkname = apklinkname + extension
935 if os.path.islink(siglinkname):
936 os.remove(siglinkname)
937 os.symlink(sigfile_path, siglinkname)
940 output = doc.toprettyxml()
944 with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
947 if 'repo_keyalias' in config:
950 logging.info("Creating unsigned index in preparation for signing")
952 logging.info("Creating signed index with this key (SHA256):")
953 logging.info("%s" % repo_pubkey_fingerprint)
955 # Create a jar of the index...
956 jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
957 p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
958 if p.returncode != 0:
959 logging.critical("Failed to create {0}".format(jar_output))
963 signed = os.path.join(repodir, 'index.jar')
965 # Remove old signed index if not signing
966 if os.path.exists(signed):
969 args = ['jarsigner', '-keystore', config['keystore'],
970 '-storepass:file', config['keystorepassfile'],
971 '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
972 signed, config['repo_keyalias']]
973 if config['keystore'] == 'NONE':
974 args += config['smartcardoptions']
975 else: # smardcards never use -keypass
976 args += ['-keypass:file', config['keypassfile']]
977 p = FDroidPopen(args)
978 if p.returncode != 0:
979 logging.critical("Failed to sign index")
982 # Copy the repo icon into the repo directory...
983 icon_dir = os.path.join(repodir, 'icons')
984 iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
985 shutil.copyfile(config['repo_icon'], iconfilename)
987 # Write a category list in the repo to allow quick access...
989 for cat in categories:
990 catdata += cat + '\n'
991 with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
995 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
997 for appid, app in apps.iteritems():
999 if app.ArchivePolicy:
1000 keepversions = int(app.ArchivePolicy[:-9])
1002 keepversions = defaultkeepversions
1004 def filter_apk_list_sorted(apk_list):
1006 for apk in apk_list:
1007 if apk['id'] == appid:
1010 # Sort the apk list by version code. First is highest/newest.
1011 return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1013 def move_file(from_dir, to_dir, filename, ignore_missing):
1014 from_path = os.path.join(from_dir, filename)
1015 if ignore_missing and not os.path.exists(from_path):
1017 to_path = os.path.join(to_dir, filename)
1018 shutil.move(from_path, to_path)
1020 if len(apks) > keepversions:
1021 apklist = filter_apk_list_sorted(apks)
1022 # Move back the ones we don't want.
1023 for apk in apklist[keepversions:]:
1024 logging.info("Moving " + apk['apkname'] + " to archive")
1025 move_file(repodir, archivedir, apk['apkname'], False)
1026 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1027 for density in all_screen_densities:
1028 repo_icon_dir = get_icon_dir(repodir, density)
1029 archive_icon_dir = get_icon_dir(archivedir, density)
1030 if density not in apk['icons']:
1032 move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1033 if 'srcname' in apk:
1034 move_file(repodir, archivedir, apk['srcname'], False)
1035 archapks.append(apk)
1037 elif len(apks) < keepversions and len(archapks) > 0:
1038 required = keepversions - len(apks)
1039 archapklist = filter_apk_list_sorted(archapks)
1040 # Move forward the ones we want again.
1041 for apk in archapklist[:required]:
1042 logging.info("Moving " + apk['apkname'] + " from archive")
1043 move_file(archivedir, repodir, apk['apkname'], False)
1044 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1045 for density in all_screen_densities:
1046 repo_icon_dir = get_icon_dir(repodir, density)
1047 archive_icon_dir = get_icon_dir(archivedir, density)
1048 if density not in apk['icons']:
1050 move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1051 if 'srcname' in apk:
1052 move_file(archivedir, repodir, apk['srcname'], False)
1053 archapks.remove(apk)
1057 def add_apks_to_per_app_repos(repodir, apks):
1058 apks_per_app = dict()
1060 apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1061 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1062 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1063 apks_per_app[apk['id']] = apk
1065 if not os.path.exists(apk['per_app_icons']):
1066 logging.info('Adding new repo for only ' + apk['id'])
1067 os.makedirs(apk['per_app_icons'])
1069 apkpath = os.path.join(repodir, apk['apkname'])
1070 shutil.copy(apkpath, apk['per_app_repo'])
1071 apksigpath = apkpath + '.sig'
1072 if os.path.exists(apksigpath):
1073 shutil.copy(apksigpath, apk['per_app_repo'])
1074 apkascpath = apkpath + '.asc'
1075 if os.path.exists(apkascpath):
1076 shutil.copy(apkascpath, apk['per_app_repo'])
1085 global config, options
1087 # Parse command line...
1088 parser = ArgumentParser()
1089 common.setup_global_opts(parser)
1090 parser.add_argument("--create-key", action="store_true", default=False,
1091 help="Create a repo signing key in a keystore")
1092 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1093 help="Create skeleton metadata files that are missing")
1094 parser.add_argument("--delete-unknown", action="store_true", default=False,
1095 help="Delete APKs without metadata from the repo")
1096 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1097 help="Report on build data status")
1098 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1099 help="Interactively ask about things that need updating.")
1100 parser.add_argument("-I", "--icons", action="store_true", default=False,
1101 help="Resize all the icons exceeding the max pixel size and exit")
1102 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1103 help="Specify editor to use in interactive mode. Default " +
1104 "is /etc/alternatives/editor")
1105 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1106 help="Update the wiki")
1107 parser.add_argument("--pretty", action="store_true", default=False,
1108 help="Produce human-readable index.xml")
1109 parser.add_argument("--clean", action="store_true", default=False,
1110 help="Clean update - don't uses caches, reprocess all apks")
1111 parser.add_argument("--nosign", action="store_true", default=False,
1112 help="When configured for signed indexes, create only unsigned indexes at this stage")
1113 options = parser.parse_args()
1115 config = common.read_config(options)
1118 if config['archive_older'] != 0:
1119 repodirs.append('archive')
1120 if not os.path.exists('archive'):
1124 resize_all_icons(repodirs)
1127 # check that icons exist now, rather than fail at the end of `fdroid update`
1128 for k in ['repo_icon', 'archive_icon']:
1130 if not os.path.exists(config[k]):
1131 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1134 # if the user asks to create a keystore, do it now, reusing whatever it can
1135 if options.create_key:
1136 if os.path.exists(config['keystore']):
1137 logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1138 logging.critical("\t'" + config['keystore'] + "'")
1141 if 'repo_keyalias' not in config:
1142 config['repo_keyalias'] = socket.getfqdn()
1143 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1144 if 'keydname' not in config:
1145 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1146 common.write_to_config(config, 'keydname', config['keydname'])
1147 if 'keystore' not in config:
1148 config['keystore'] = common.default_config.keystore
1149 common.write_to_config(config, 'keystore', config['keystore'])
1151 password = common.genpassword()
1152 if 'keystorepass' not in config:
1153 config['keystorepass'] = password
1154 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1155 if 'keypass' not in config:
1156 config['keypass'] = password
1157 common.write_to_config(config, 'keypass', config['keypass'])
1158 common.genkeystore(config)
1161 apps = metadata.read_metadata()
1163 # Generate a list of categories...
1165 for app in apps.itervalues():
1166 categories.update(app.Categories)
1168 # Read known apks data (will be updated and written back when we've finished)
1169 knownapks = common.KnownApks()
1171 # Gather information about all the apk files in the repo directory, using
1172 # cached data if possible.
1173 apkcachefile = os.path.join('tmp', 'apkcache')
1174 if not options.clean and os.path.exists(apkcachefile):
1175 with open(apkcachefile, 'rb') as cf:
1176 apkcache = pickle.load(cf)
1180 delete_disabled_builds(apps, apkcache, repodirs)
1182 # Scan all apks in the main repo
1183 apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks)
1185 # Generate warnings for apk's with no metadata (or create skeleton
1186 # metadata files, if requested on the command line)
1189 if apk['id'] not in apps:
1190 if options.create_metadata:
1191 if 'name' not in apk:
1192 logging.error(apk['id'] + ' does not have a name! Skipping...')
1194 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1195 f.write("License:Unknown\n")
1196 f.write("Web Site:\n")
1197 f.write("Source Code:\n")
1198 f.write("Issue Tracker:\n")
1199 f.write("Changelog:\n")
1200 f.write("Summary:" + apk['name'] + "\n")
1201 f.write("Description:\n")
1202 f.write(apk['name'] + "\n")
1205 logging.info("Generated skeleton metadata for " + apk['id'])
1208 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1209 if options.delete_unknown:
1210 logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1211 rmf = os.path.join(repodirs[0], apk['apkname'])
1212 if not os.path.exists(rmf):
1213 logging.error("Could not find {0} to remove it".format(rmf))
1217 logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1219 # update the metadata with the newly created ones included
1221 apps = metadata.read_metadata()
1223 # Scan the archive repo for apks as well
1224 if len(repodirs) > 1:
1225 archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
1231 # Some information from the apks needs to be applied up to the application
1232 # level. When doing this, we use the info from the most recent version's apk.
1233 # We deal with figuring out when the app was added and last updated at the
1235 for appid, app in apps.iteritems():
1237 for apk in apks + archapks:
1238 if apk['id'] == appid:
1239 if apk['versioncode'] > bestver:
1240 bestver = apk['versioncode']
1244 if not app.added or apk['added'] < app.added:
1245 app.added = apk['added']
1246 if not app.lastupdated or apk['added'] > app.lastupdated:
1247 app.lastupdated = apk['added']
1250 logging.debug("Don't know when " + appid + " was added")
1251 if not app.lastupdated:
1252 logging.debug("Don't know when " + appid + " was last updated")
1255 if app.Name is None:
1256 app.Name = app.AutoName or appid
1258 logging.debug("Application " + appid + " has no packages")
1260 if app.Name is None:
1261 app.Name = bestapk['name']
1262 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1264 # Sort the app list by name, then the web site doesn't have to by default.
1265 # (we had to wait until we'd scanned the apks to do this, because mostly the
1266 # name comes from there!)
1267 sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid].Name.upper())
1269 # APKs are placed into multiple repos based on the app package, providing
1270 # per-app subscription feeds for nightly builds and things like it
1271 if config['per_app_repos']:
1272 add_apks_to_per_app_repos(repodirs[0], apks)
1273 for appid, app in apps.iteritems():
1274 repodir = os.path.join(appid, 'fdroid', 'repo')
1276 appdict[appid] = app
1277 if os.path.isdir(repodir):
1278 make_index(appdict, [appid], apks, repodir, False, categories)
1280 logging.info('Skipping index generation for ' + appid)
1283 if len(repodirs) > 1:
1284 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1286 # Make the index for the main repo...
1287 make_index(apps, sortedids, apks, repodirs[0], False, categories)
1289 # If there's an archive repo, make the index for it. We already scanned it
1291 if len(repodirs) > 1:
1292 make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1294 if config['update_stats']:
1296 # Update known apks info...
1297 knownapks.writeifchanged()
1299 # Generate latest apps data for widget
1300 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1302 for line in file(os.path.join('stats', 'latestapps.txt')):
1303 appid = line.rstrip()
1304 data += appid + "\t"
1306 data += app.Name + "\t"
1307 if app.icon is not None:
1308 data += app.icon + "\t"
1309 data += app.License + "\n"
1310 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
1314 with open(apkcachefile, 'wb') as cf:
1315 pickle.dump(apkcache, cf)
1317 # Update the wiki...
1319 update_wiki(apps, sortedids, apks + archapks)
1321 logging.info("Finished.")
1323 if __name__ == "__main__":