3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
30 from datetime import datetime, timedelta
31 from xml.dom.minidom import Document
32 from argparse import ArgumentParser
34 from pyasn1.error import PyAsn1Error
35 from pyasn1.codec.der import decoder, encoder
36 from pyasn1_modules import rfc2315
37 from binascii import hexlify, unhexlify
43 from . import metadata
44 from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
45 from .metadata import MetaDataException
49 screen_densities = ['640', '480', '320', '240', '160', '120']
51 all_screen_densities = ['0'] + screen_densities
54 def dpi_to_px(density):
55 return (int(density) * 48) / 160
59 return (int(px) * 160) / 48
62 def get_icon_dir(repodir, density):
64 return os.path.join(repodir, "icons")
65 return os.path.join(repodir, "icons-%s" % density)
68 def get_icon_dirs(repodir):
69 for density in screen_densities:
70 yield get_icon_dir(repodir, density)
73 def get_all_icon_dirs(repodir):
74 for density in all_screen_densities:
75 yield get_icon_dir(repodir, density)
78 def update_wiki(apps, sortedids, apks):
81 :param apps: fully populated list of all applications
82 :param apks: all apks, except...
84 logging.info("Updating wiki")
86 wikiredircat = 'App Redirects'
88 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
89 path=config['wiki_path'])
90 site.login(config['wiki_user'], config['wiki_password'])
92 generated_redirects = {}
94 for appid in sortedids:
99 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
101 for af in app.AntiFeatures:
102 wikidata += '{{AntiFeature|' + af + '}}\n'
107 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
110 time.strftime('%Y-%m-%d', app.added) if app.added else '',
111 time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
126 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
128 wikidata += app.Summary
129 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
131 wikidata += "=Description=\n"
132 wikidata += metadata.description_wiki(app.Description) + "\n"
134 wikidata += "=Maintainer Notes=\n"
135 if app.MaintainerNotes:
136 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
137 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
139 # Get a list of all packages for this application...
141 gotcurrentver = False
145 if apk['id'] == appid:
146 if str(apk['versioncode']) == app.CurrentVersionCode:
149 # Include ones we can't build, as a special case...
150 for build in app.builds:
152 if build.vercode == app.CurrentVersionCode:
154 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
155 apklist.append({'versioncode': int(build.vercode),
156 'version': build.version,
157 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
162 if apk['versioncode'] == int(build.vercode):
167 apklist.append({'versioncode': int(build.vercode),
168 'version': build.version,
169 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
171 if app.CurrentVersionCode == '0':
173 # Sort with most recent first...
174 apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
176 wikidata += "=Versions=\n"
177 if len(apklist) == 0:
178 wikidata += "We currently have no versions of this app available."
179 elif not gotcurrentver:
180 wikidata += "We don't have the current version of this app."
182 wikidata += "We have the current version of this app."
183 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
184 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
185 if len(app.NoSourceSince) > 0:
186 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
187 if len(app.CurrentVersion) > 0:
188 wikidata += "The current (recommended) version is " + app.CurrentVersion
189 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
192 wikidata += "==" + apk['version'] + "==\n"
194 if 'buildproblem' in apk:
195 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
198 wikidata += "This version is built and signed by "
200 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
202 wikidata += "the original developer.\n\n"
203 wikidata += "Version code: " + str(apk['versioncode']) + '\n'
205 wikidata += '\n[[Category:' + wikicat + ']]\n'
206 if len(app.NoSourceSince) > 0:
207 wikidata += '\n[[Category:Apps missing source code]]\n'
208 if validapks == 0 and not app.Disabled:
209 wikidata += '\n[[Category:Apps with no packages]]\n'
210 if cantupdate and not app.Disabled:
211 wikidata += "\n[[Category:Apps we cannot update]]\n"
212 if buildfails and not app.Disabled:
213 wikidata += "\n[[Category:Apps with failing builds]]\n"
214 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
215 wikidata += '\n[[Category:Apps to Update]]\n'
217 wikidata += '\n[[Category:Apps that are disabled]]\n'
218 if app.UpdateCheckMode == 'None' and not app.Disabled:
219 wikidata += '\n[[Category:Apps with no update check]]\n'
220 for appcat in app.Categories:
221 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
223 # We can't have underscores in the page name, even if they're in
224 # the package ID, because MediaWiki messes with them...
225 pagename = appid.replace('_', ' ')
227 # Drop a trailing newline, because mediawiki is going to drop it anyway
228 # and it we don't we'll think the page has changed when it hasn't...
229 if wikidata.endswith('\n'):
230 wikidata = wikidata[:-1]
232 generated_pages[pagename] = wikidata
234 # Make a redirect from the name to the ID too, unless there's
235 # already an existing page with the name and it isn't a redirect.
237 apppagename = app.Name.replace('_', ' ')
238 apppagename = apppagename.replace('{', '')
239 apppagename = apppagename.replace('}', ' ')
240 apppagename = apppagename.replace(':', ' ')
241 # Drop double spaces caused mostly by replacing ':' above
242 apppagename = apppagename.replace(' ', ' ')
243 for expagename in site.allpages(prefix=apppagename,
244 filterredir='nonredirects',
246 if expagename == apppagename:
248 # Another reason not to make the redirect page is if the app name
249 # is the same as it's ID, because that will overwrite the real page
250 # with an redirect to itself! (Although it seems like an odd
251 # scenario this happens a lot, e.g. where there is metadata but no
252 # builds or binaries to extract a name from.
253 if apppagename == pagename:
256 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
258 for tcat, genp in [(wikicat, generated_pages),
259 (wikiredircat, generated_redirects)]:
260 catpages = site.Pages['Category:' + tcat]
262 for page in catpages:
263 existingpages.append(page.name)
264 if page.name in genp:
265 pagetxt = page.edit()
266 if pagetxt != genp[page.name]:
267 logging.debug("Updating modified page " + page.name)
268 page.save(genp[page.name], summary='Auto-updated')
270 logging.debug("Page " + page.name + " is unchanged")
272 logging.warn("Deleting page " + page.name)
273 page.delete('No longer published')
274 for pagename, text in genp.items():
275 logging.debug("Checking " + pagename)
276 if pagename not in existingpages:
277 logging.debug("Creating page " + pagename)
279 newpage = site.Pages[pagename]
280 newpage.save(text, summary='Auto-created')
282 logging.error("...FAILED to create page '{0}'".format(pagename))
284 # Purge server cache to ensure counts are up to date
285 site.pages['Repository Maintenance'].purge()
288 def delete_disabled_builds(apps, apkcache, repodirs):
289 """Delete disabled build outputs.
291 :param apps: list of all applications, as per metadata.read_metadata
292 :param apkcache: current apk cache information
293 :param repodirs: the repo directories to process
295 for appid, app in apps.items():
296 for build in app.builds:
297 if not build.disable:
299 apkfilename = appid + '_' + str(build.vercode) + '.apk'
300 iconfilename = "%s.%s.png" % (
303 for repodir in repodirs:
305 os.path.join(repodir, apkfilename),
306 os.path.join(repodir, apkfilename + '.asc'),
307 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
309 for density in all_screen_densities:
310 repo_dir = get_icon_dir(repodir, density)
311 files.append(os.path.join(repo_dir, iconfilename))
314 if os.path.exists(f):
315 logging.info("Deleting disabled build output " + f)
317 if apkfilename in apkcache:
318 del apkcache[apkfilename]
321 def resize_icon(iconpath, density):
323 if not os.path.isfile(iconpath):
327 im = Image.open(iconpath)
328 size = dpi_to_px(density)
330 if any(length > size for length in im.size):
332 im.thumbnail((size, size), Image.ANTIALIAS)
333 logging.debug("%s was too large at %s - new size is %s" % (
334 iconpath, oldsize, im.size))
335 im.save(iconpath, "PNG")
337 except Exception as e:
338 logging.error("Failed resizing {0} - {1}".format(iconpath, e))
341 def resize_all_icons(repodirs):
342 """Resize all icons that exceed the max size
344 :param repodirs: the repo directories to process
346 for repodir in repodirs:
347 for density in screen_densities:
348 icon_dir = get_icon_dir(repodir, density)
349 icon_glob = os.path.join(icon_dir, '*.png')
350 for iconpath in glob.glob(icon_glob):
351 resize_icon(iconpath, density)
354 # A signature block file with a .DSA, .RSA, or .EC extension
355 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
359 """ Get the signing certificate of an apk. To get the same md5 has that
360 Android gets, we encode the .RSA certificate in a specific format and pass
361 it hex-encoded to the md5 digest algorithm.
363 :param apkpath: path to the apk
364 :returns: A string containing the md5 of the signature of the apk or None
365 if an error occurred.
370 # verify the jar signature is correct
371 args = [config['jarsigner'], '-verify', apkpath]
372 p = FDroidPopen(args)
373 if p.returncode != 0:
374 logging.critical(apkpath + " has a bad signature!")
377 with zipfile.ZipFile(apkpath, 'r') as apk:
379 certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
382 logging.error("Found no signing certificates on %s" % apkpath)
385 logging.error("Found multiple signing certificates on %s" % apkpath)
388 cert = apk.read(certs[0])
390 content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
391 if content.getComponentByName('contentType') != rfc2315.signedData:
392 logging.error("Unexpected format.")
395 content = decoder.decode(content.getComponentByName('content'),
396 asn1Spec=rfc2315.SignedData())[0]
398 certificates = content.getComponentByName('certificates')
400 logging.error("Certificates not found.")
403 cert_encoded = encoder.encode(certificates)[4:]
405 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
408 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
409 """Scan the apks in the given repo directory.
411 This also extracts the icons.
413 :param apps: list of all applications, as per metadata.read_metadata
414 :param apkcache: current apk cache information
415 :param repodir: repo directory to scan
416 :param knownapks: known apks info
417 :param use_date_from_apk: use date from APK (instead of current date)
419 :returns: (apks, cachechanged) where apks is a list of apk information,
420 and cachechanged is True if the apkcache got changed.
425 for icon_dir in get_all_icon_dirs(repodir):
426 if os.path.exists(icon_dir):
428 shutil.rmtree(icon_dir)
429 os.makedirs(icon_dir)
431 os.makedirs(icon_dir)
434 name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
435 vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
436 vername_pat = re.compile(".*versionName='([^']*)'.*")
437 label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
438 icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
439 icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
440 sdkversion_pat = re.compile(".*'([0-9]*)'.*")
441 string_pat = re.compile(".*'([^']*)'.*")
442 for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
444 apkfilename = apkfile[len(repodir) + 1:]
445 if ' ' in apkfilename:
446 logging.critical("Spaces in filenames are not allowed.")
449 # Calculate the sha256...
450 sha = hashlib.sha256()
451 with open(apkfile, 'rb') as f:
457 shasum = sha.hexdigest()
460 if apkfilename in apkcache:
461 apk = apkcache[apkfilename]
462 if apk['sha256'] == shasum:
463 logging.debug("Reading " + apkfilename + " from cache")
466 logging.debug("Ignoring stale cache data for " + apkfilename)
469 logging.debug("Processing " + apkfilename)
471 apk['apkname'] = apkfilename
472 apk['sha256'] = shasum
473 srcfilename = apkfilename[:-4] + "_src.tar.gz"
474 if os.path.exists(os.path.join(repodir, srcfilename)):
475 apk['srcname'] = srcfilename
476 apk['size'] = os.path.getsize(apkfile)
477 apk['permissions'] = set()
478 apk['features'] = set()
479 apk['icons_src'] = {}
481 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
482 if p.returncode != 0:
483 if options.delete_unknown:
484 if os.path.exists(apkfile):
485 logging.error("Failed to get apk information, deleting " + apkfile)
488 logging.error("Could not find {0} to remove it".format(apkfile))
490 logging.error("Failed to get apk information, skipping " + apkfile)
492 for line in p.output.splitlines():
493 if line.startswith("package:"):
495 apk['id'] = re.match(name_pat, line).group(1)
496 apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
497 apk['version'] = re.match(vername_pat, line).group(1)
498 except Exception as e:
499 logging.error("Package matching failed: " + str(e))
500 logging.info("Line was: " + line)
502 elif line.startswith("application:"):
503 apk['name'] = re.match(label_pat, line).group(1)
504 # Keep path to non-dpi icon in case we need it
505 match = re.match(icon_pat_nodpi, line)
507 apk['icons_src']['-1'] = match.group(1)
508 elif line.startswith("launchable-activity:"):
509 # Only use launchable-activity as fallback to application
511 apk['name'] = re.match(label_pat, line).group(1)
512 if '-1' not in apk['icons_src']:
513 match = re.match(icon_pat_nodpi, line)
515 apk['icons_src']['-1'] = match.group(1)
516 elif line.startswith("application-icon-"):
517 match = re.match(icon_pat, line)
519 density = match.group(1)
520 path = match.group(2)
521 apk['icons_src'][density] = path
522 elif line.startswith("sdkVersion:"):
523 m = re.match(sdkversion_pat, line)
525 logging.error(line.replace('sdkVersion:', '')
526 + ' is not a valid minSdkVersion!')
528 apk['minSdkVersion'] = m.group(1)
529 # if target not set, default to min
530 if 'targetSdkVersion' not in apk:
531 apk['targetSdkVersion'] = m.group(1)
532 elif line.startswith("targetSdkVersion:"):
533 m = re.match(sdkversion_pat, line)
535 logging.error(line.replace('targetSdkVersion:', '')
536 + ' is not a valid targetSdkVersion!')
538 apk['targetSdkVersion'] = m.group(1)
539 elif line.startswith("maxSdkVersion:"):
540 apk['maxSdkVersion'] = re.match(sdkversion_pat, line).group(1)
541 elif line.startswith("native-code:"):
542 apk['nativecode'] = []
543 for arch in line[13:].split(' '):
544 apk['nativecode'].append(arch[1:-1])
545 elif line.startswith("uses-permission:"):
546 perm = re.match(string_pat, line).group(1)
547 if perm.startswith("android.permission."):
549 apk['permissions'].add(perm)
550 elif line.startswith("uses-feature:"):
551 perm = re.match(string_pat, line).group(1)
552 # Filter out this, it's only added with the latest SDK tools and
553 # causes problems for lots of apps.
554 if perm != "android.hardware.screen.portrait" \
555 and perm != "android.hardware.screen.landscape":
556 if perm.startswith("android.feature."):
558 apk['features'].add(perm)
560 if 'minSdkVersion' not in apk:
561 logging.warn("No SDK version information found in {0}".format(apkfile))
562 apk['minSdkVersion'] = 1
564 # Check for debuggable apks...
565 if common.isApkDebuggable(apkfile, config):
566 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
568 # Get the signature (or md5 of, to be precise)...
569 logging.debug('Getting signature of {0}'.format(apkfile))
570 apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
572 logging.critical("Failed to get apk signature")
575 apkzip = zipfile.ZipFile(apkfile, 'r')
577 # if an APK has files newer than the system time, suggest updating
578 # the system clock. This is useful for offline systems, used for
579 # signing, which do not have another source of clock sync info. It
580 # has to be more than 24 hours newer because ZIP/APK files do not
581 # store timezone info
582 manifest = apkzip.getinfo('AndroidManifest.xml')
583 if manifest.date_time[1] == 0: # month can't be zero
584 logging.debug('AndroidManifest.xml has no date')
586 dt_obj = datetime(*manifest.date_time)
587 checkdt = dt_obj - timedelta(1)
588 if datetime.today() < checkdt:
589 logging.warn('System clock is older than manifest in: '
591 + '\nSet clock to that time using:\n'
592 + 'sudo date -s "' + str(dt_obj) + '"')
594 iconfilename = "%s.%s.png" % (
598 # Extract the icon file...
600 for density in screen_densities:
601 if density not in apk['icons_src']:
602 empty_densities.append(density)
604 iconsrc = apk['icons_src'][density]
605 icon_dir = get_icon_dir(repodir, density)
606 icondest = os.path.join(icon_dir, iconfilename)
609 with open(icondest, 'wb') as f:
610 f.write(apkzip.read(iconsrc))
611 apk['icons'][density] = iconfilename
614 logging.warn("Error retrieving icon file")
615 del apk['icons'][density]
616 del apk['icons_src'][density]
617 empty_densities.append(density)
619 if '-1' in apk['icons_src']:
620 iconsrc = apk['icons_src']['-1']
621 iconpath = os.path.join(
622 get_icon_dir(repodir, '0'), iconfilename)
623 with open(iconpath, 'wb') as f:
624 f.write(apkzip.read(iconsrc))
626 im = Image.open(iconpath)
627 dpi = px_to_dpi(im.size[0])
628 for density in screen_densities:
629 if density in apk['icons']:
631 if density == screen_densities[-1] or dpi >= int(density):
632 apk['icons'][density] = iconfilename
633 shutil.move(iconpath,
634 os.path.join(get_icon_dir(repodir, density), iconfilename))
635 empty_densities.remove(density)
637 except Exception as e:
638 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
641 apk['icon'] = iconfilename
645 # First try resizing down to not lose quality
647 for density in screen_densities:
648 if density not in empty_densities:
649 last_density = density
651 if last_density is None:
653 logging.debug("Density %s not available, resizing down from %s"
654 % (density, last_density))
656 last_iconpath = os.path.join(
657 get_icon_dir(repodir, last_density), iconfilename)
658 iconpath = os.path.join(
659 get_icon_dir(repodir, density), iconfilename)
661 im = Image.open(last_iconpath)
663 logging.warn("Invalid image file at %s" % last_iconpath)
666 size = dpi_to_px(density)
668 im.thumbnail((size, size), Image.ANTIALIAS)
669 im.save(iconpath, "PNG")
670 empty_densities.remove(density)
672 # Then just copy from the highest resolution available
674 for density in reversed(screen_densities):
675 if density not in empty_densities:
676 last_density = density
678 if last_density is None:
680 logging.debug("Density %s not available, copying from lower density %s"
681 % (density, last_density))
684 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
685 os.path.join(get_icon_dir(repodir, density), iconfilename))
687 empty_densities.remove(density)
689 for density in screen_densities:
690 icon_dir = get_icon_dir(repodir, density)
691 icondest = os.path.join(icon_dir, iconfilename)
692 resize_icon(icondest, density)
694 # Copy from icons-mdpi to icons since mdpi is the baseline density
695 baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
696 if os.path.isfile(baseline):
697 apk['icons']['0'] = iconfilename
698 shutil.copyfile(baseline,
699 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
701 if use_date_from_apk and manifest.date_time[1] != 0:
702 default_date_param = datetime(*manifest.date_time).utctimetuple()
704 default_date_param = None
706 # Record in known apks, getting the added date at the same time..
707 added = knownapks.recordapk(apk['apkname'], apk['id'], default_date=default_date_param)
711 apkcache[apkfilename] = apk
716 return apks, cachechanged
719 repo_pubkey_fingerprint = None
722 # Generate a certificate fingerprint the same way keytool does it
723 # (but with slightly different formatting)
724 def cert_fingerprint(data):
725 digest = hashlib.sha256(data).digest()
727 ret.append(' '.join("%02X" % b for b in bytearray(digest)))
731 def extract_pubkey():
732 global repo_pubkey_fingerprint
733 if 'repo_pubkey' in config:
734 pubkey = unhexlify(config['repo_pubkey'])
736 p = FDroidPopenBytes([config['keytool'], '-exportcert',
737 '-alias', config['repo_keyalias'],
738 '-keystore', config['keystore'],
739 '-storepass:file', config['keystorepassfile']]
740 + config['smartcardoptions'],
741 output=False, stderr_to_stdout=False)
742 if p.returncode != 0 or len(p.output) < 20:
743 msg = "Failed to get repo pubkey!"
744 if config['keystore'] == 'NONE':
745 msg += ' Is your crypto smartcard plugged in?'
746 logging.critical(msg)
749 repo_pubkey_fingerprint = cert_fingerprint(pubkey)
750 return hexlify(pubkey)
753 def make_index(apps, sortedids, apks, repodir, archive, categories):
754 """Make a repo index.
756 :param apps: fully populated apps list
757 :param apks: full populated apks list
758 :param repodir: the repo directory
759 :param archive: True if this is the archive repo, False if it's the
761 :param categories: list of categories
766 def addElement(name, value, doc, parent):
767 el = doc.createElement(name)
768 el.appendChild(doc.createTextNode(value))
769 parent.appendChild(el)
771 def addElementNonEmpty(name, value, doc, parent):
774 addElement(name, value, doc, parent)
776 def addElementCDATA(name, value, doc, parent):
777 el = doc.createElement(name)
778 el.appendChild(doc.createCDATASection(value))
779 parent.appendChild(el)
781 root = doc.createElement("fdroid")
782 doc.appendChild(root)
784 repoel = doc.createElement("repo")
786 mirrorcheckfailed = False
787 for mirror in config.get('mirrors', []):
788 base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
789 if config.get('nonstandardwebroot') is not True and base != 'fdroid':
790 logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
791 mirrorcheckfailed = True
792 if mirrorcheckfailed:
796 repoel.setAttribute("name", config['archive_name'])
797 if config['repo_maxage'] != 0:
798 repoel.setAttribute("maxage", str(config['repo_maxage']))
799 repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
800 repoel.setAttribute("url", config['archive_url'])
801 addElement('description', config['archive_description'], doc, repoel)
802 urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
803 for mirror in config.get('mirrors', []):
804 addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
807 repoel.setAttribute("name", config['repo_name'])
808 if config['repo_maxage'] != 0:
809 repoel.setAttribute("maxage", str(config['repo_maxage']))
810 repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
811 repoel.setAttribute("url", config['repo_url'])
812 addElement('description', config['repo_description'], doc, repoel)
813 urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
814 for mirror in config.get('mirrors', []):
815 addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
817 repoel.setAttribute("version", str(METADATA_VERSION))
818 repoel.setAttribute("timestamp", str(int(time.time())))
821 if not options.nosign:
822 if 'repo_keyalias' not in config:
824 logging.critical("'repo_keyalias' not found in config.py!")
825 if 'keystore' not in config:
827 logging.critical("'keystore' not found in config.py!")
828 if 'keystorepass' not in config and 'keystorepassfile' not in config:
830 logging.critical("'keystorepass' not found in config.py!")
831 if 'keypass' not in config and 'keypassfile' not in config:
833 logging.critical("'keypass' not found in config.py!")
834 if not os.path.exists(config['keystore']):
836 logging.critical("'" + config['keystore'] + "' does not exist!")
838 logging.warning("`fdroid update` requires a signing key, you can create one using:")
839 logging.warning("\tfdroid update --create-key")
842 repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
843 root.appendChild(repoel)
845 for appid in sortedids:
848 if app.Disabled is not None:
851 # Get a list of the apks for this app...
854 if apk['id'] == appid:
857 if len(apklist) == 0:
860 apel = doc.createElement("application")
861 apel.setAttribute("id", app.id)
862 root.appendChild(apel)
864 addElement('id', app.id, doc, apel)
866 addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
868 addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
869 addElement('name', app.Name, doc, apel)
870 addElement('summary', app.Summary, doc, apel)
872 addElement('icon', app.icon, doc, apel)
876 return ("fdroid.app:" + appid, apps[appid].Name)
877 raise MetaDataException("Cannot resolve app id " + appid)
880 metadata.description_html(app.Description, linkres),
882 addElement('license', app.License, doc, apel)
884 addElement('categories', ','.join(app.Categories), doc, apel)
885 # We put the first (primary) category in LAST, which will have
886 # the desired effect of making clients that only understand one
887 # category see that one.
888 addElement('category', app.Categories[0], doc, apel)
889 addElement('web', app.WebSite, doc, apel)
890 addElement('source', app.SourceCode, doc, apel)
891 addElement('tracker', app.IssueTracker, doc, apel)
892 addElementNonEmpty('changelog', app.Changelog, doc, apel)
893 addElementNonEmpty('author', app.AuthorName, doc, apel)
894 addElementNonEmpty('email', app.AuthorEmail, doc, apel)
895 addElementNonEmpty('donate', app.Donate, doc, apel)
896 addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
897 addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
898 addElementNonEmpty('flattr', app.FlattrID, doc, apel)
900 # These elements actually refer to the current version (i.e. which
901 # one is recommended. They are historically mis-named, and need
902 # changing, but stay like this for now to support existing clients.
903 addElement('marketversion', app.CurrentVersion, doc, apel)
904 addElement('marketvercode', app.CurrentVersionCode, doc, apel)
907 af = app.AntiFeatures
909 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
911 pv = app.Provides.split(',')
912 addElementNonEmpty('provides', ','.join(pv), doc, apel)
914 addElement('requirements', 'root', doc, apel)
916 # Sort the apk list into version order, just so the web site
917 # doesn't have to do any work by default...
918 apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
920 # Check for duplicates - they will make the client unhappy...
921 for i in range(len(apklist) - 1):
922 if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
923 logging.critical("duplicate versions: '%s' - '%s'" % (
924 apklist[i]['apkname'], apklist[i + 1]['apkname']))
927 current_version_code = 0
928 current_version_file = None
930 # find the APK for the "Current Version"
931 if current_version_code < apk['versioncode']:
932 current_version_code = apk['versioncode']
933 if current_version_code < int(app.CurrentVersionCode):
934 current_version_file = apk['apkname']
936 apkel = doc.createElement("package")
937 apel.appendChild(apkel)
938 addElement('version', apk['version'], doc, apkel)
939 addElement('versioncode', str(apk['versioncode']), doc, apkel)
940 addElement('apkname', apk['apkname'], doc, apkel)
942 addElement('srcname', apk['srcname'], doc, apkel)
943 for hash_type in ['sha256']:
944 if hash_type not in apk:
946 hashel = doc.createElement("hash")
947 hashel.setAttribute("type", hash_type)
948 hashel.appendChild(doc.createTextNode(apk[hash_type]))
949 apkel.appendChild(hashel)
950 addElement('sig', apk['sig'], doc, apkel)
951 addElement('size', str(apk['size']), doc, apkel)
952 addElement('sdkver', str(apk['minSdkVersion']), doc, apkel)
953 if 'targetSdkVersion' in apk:
954 addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
955 if 'maxSdkVersion' in apk:
956 addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
958 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
959 addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
960 if 'nativecode' in apk:
961 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
962 addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
964 if current_version_file is not None \
965 and config['make_current_version_link'] \
966 and repodir == 'repo': # only create these
967 namefield = config['current_version_name_source']
968 sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
969 apklinkname = sanitized_name + '.apk'
970 current_version_path = os.path.join(repodir, current_version_file)
971 if os.path.islink(apklinkname):
972 os.remove(apklinkname)
973 os.symlink(current_version_path, apklinkname)
974 # also symlink gpg signature, if it exists
975 for extension in ('.asc', '.sig'):
976 sigfile_path = current_version_path + extension
977 if os.path.exists(sigfile_path):
978 siglinkname = apklinkname + extension
979 if os.path.islink(siglinkname):
980 os.remove(siglinkname)
981 os.symlink(sigfile_path, siglinkname)
984 output = doc.toprettyxml(encoding='utf-8')
986 output = doc.toxml(encoding='utf-8')
988 with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
991 if 'repo_keyalias' in config:
994 logging.info("Creating unsigned index in preparation for signing")
996 logging.info("Creating signed index with this key (SHA256):")
997 logging.info("%s" % repo_pubkey_fingerprint)
999 # Create a jar of the index...
1000 jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
1001 p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
1002 if p.returncode != 0:
1003 logging.critical("Failed to create {0}".format(jar_output))
1007 signed = os.path.join(repodir, 'index.jar')
1009 # Remove old signed index if not signing
1010 if os.path.exists(signed):
1013 args = [config['jarsigner'], '-keystore', config['keystore'],
1014 '-storepass:file', config['keystorepassfile'],
1015 '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1016 signed, config['repo_keyalias']]
1017 if config['keystore'] == 'NONE':
1018 args += config['smartcardoptions']
1019 else: # smardcards never use -keypass
1020 args += ['-keypass:file', config['keypassfile']]
1021 p = FDroidPopen(args)
1022 if p.returncode != 0:
1023 logging.critical("Failed to sign index")
1026 # Copy the repo icon into the repo directory...
1027 icon_dir = os.path.join(repodir, 'icons')
1028 iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1029 shutil.copyfile(config['repo_icon'], iconfilename)
1031 # Write a category list in the repo to allow quick access...
1033 for cat in categories:
1034 catdata += cat + '\n'
1035 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1039 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1041 for appid, app in apps.items():
1043 if app.ArchivePolicy:
1044 keepversions = int(app.ArchivePolicy[:-9])
1046 keepversions = defaultkeepversions
1048 def filter_apk_list_sorted(apk_list):
1050 for apk in apk_list:
1051 if apk['id'] == appid:
1054 # Sort the apk list by version code. First is highest/newest.
1055 return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1057 def move_file(from_dir, to_dir, filename, ignore_missing):
1058 from_path = os.path.join(from_dir, filename)
1059 if ignore_missing and not os.path.exists(from_path):
1061 to_path = os.path.join(to_dir, filename)
1062 shutil.move(from_path, to_path)
1064 logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1065 .format(appid, len(apks), keepversions, len(archapks)))
1067 if len(apks) > keepversions:
1068 apklist = filter_apk_list_sorted(apks)
1069 # Move back the ones we don't want.
1070 for apk in apklist[keepversions:]:
1071 logging.info("Moving " + apk['apkname'] + " to archive")
1072 move_file(repodir, archivedir, apk['apkname'], False)
1073 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1074 for density in all_screen_densities:
1075 repo_icon_dir = get_icon_dir(repodir, density)
1076 archive_icon_dir = get_icon_dir(archivedir, density)
1077 if density not in apk['icons']:
1079 move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1080 if 'srcname' in apk:
1081 move_file(repodir, archivedir, apk['srcname'], False)
1082 archapks.append(apk)
1084 elif len(apks) < keepversions and len(archapks) > 0:
1085 required = keepversions - len(apks)
1086 archapklist = filter_apk_list_sorted(archapks)
1087 # Move forward the ones we want again.
1088 for apk in archapklist[:required]:
1089 logging.info("Moving " + apk['apkname'] + " from archive")
1090 move_file(archivedir, repodir, apk['apkname'], False)
1091 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1092 for density in all_screen_densities:
1093 repo_icon_dir = get_icon_dir(repodir, density)
1094 archive_icon_dir = get_icon_dir(archivedir, density)
1095 if density not in apk['icons']:
1097 move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1098 if 'srcname' in apk:
1099 move_file(archivedir, repodir, apk['srcname'], False)
1100 archapks.remove(apk)
1104 def add_apks_to_per_app_repos(repodir, apks):
1105 apks_per_app = dict()
1107 apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1108 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1109 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1110 apks_per_app[apk['id']] = apk
1112 if not os.path.exists(apk['per_app_icons']):
1113 logging.info('Adding new repo for only ' + apk['id'])
1114 os.makedirs(apk['per_app_icons'])
1116 apkpath = os.path.join(repodir, apk['apkname'])
1117 shutil.copy(apkpath, apk['per_app_repo'])
1118 apksigpath = apkpath + '.sig'
1119 if os.path.exists(apksigpath):
1120 shutil.copy(apksigpath, apk['per_app_repo'])
1121 apkascpath = apkpath + '.asc'
1122 if os.path.exists(apkascpath):
1123 shutil.copy(apkascpath, apk['per_app_repo'])
1132 global config, options
1134 # Parse command line...
1135 parser = ArgumentParser()
1136 common.setup_global_opts(parser)
1137 parser.add_argument("--create-key", action="store_true", default=False,
1138 help="Create a repo signing key in a keystore")
1139 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1140 help="Create skeleton metadata files that are missing")
1141 parser.add_argument("--delete-unknown", action="store_true", default=False,
1142 help="Delete APKs without metadata from the repo")
1143 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1144 help="Report on build data status")
1145 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1146 help="Interactively ask about things that need updating.")
1147 parser.add_argument("-I", "--icons", action="store_true", default=False,
1148 help="Resize all the icons exceeding the max pixel size and exit")
1149 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1150 help="Specify editor to use in interactive mode. Default " +
1151 "is /etc/alternatives/editor")
1152 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1153 help="Update the wiki")
1154 parser.add_argument("--pretty", action="store_true", default=False,
1155 help="Produce human-readable index.xml")
1156 parser.add_argument("--clean", action="store_true", default=False,
1157 help="Clean update - don't uses caches, reprocess all apks")
1158 parser.add_argument("--nosign", action="store_true", default=False,
1159 help="When configured for signed indexes, create only unsigned indexes at this stage")
1160 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1161 help="Use date from apk instead of current time for newly added apks")
1162 options = parser.parse_args()
1164 config = common.read_config(options)
1166 if not ('jarsigner' in config and 'keytool' in config):
1167 logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1171 if config['archive_older'] != 0:
1172 repodirs.append('archive')
1173 if not os.path.exists('archive'):
1177 resize_all_icons(repodirs)
1180 # check that icons exist now, rather than fail at the end of `fdroid update`
1181 for k in ['repo_icon', 'archive_icon']:
1183 if not os.path.exists(config[k]):
1184 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1187 # if the user asks to create a keystore, do it now, reusing whatever it can
1188 if options.create_key:
1189 if os.path.exists(config['keystore']):
1190 logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1191 logging.critical("\t'" + config['keystore'] + "'")
1194 if 'repo_keyalias' not in config:
1195 config['repo_keyalias'] = socket.getfqdn()
1196 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1197 if 'keydname' not in config:
1198 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1199 common.write_to_config(config, 'keydname', config['keydname'])
1200 if 'keystore' not in config:
1201 config['keystore'] = common.default_config.keystore
1202 common.write_to_config(config, 'keystore', config['keystore'])
1204 password = common.genpassword()
1205 if 'keystorepass' not in config:
1206 config['keystorepass'] = password
1207 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1208 if 'keypass' not in config:
1209 config['keypass'] = password
1210 common.write_to_config(config, 'keypass', config['keypass'])
1211 common.genkeystore(config)
1214 apps = metadata.read_metadata()
1216 # Generate a list of categories...
1218 for app in apps.values():
1219 categories.update(app.Categories)
1221 # Read known apks data (will be updated and written back when we've finished)
1222 knownapks = common.KnownApks()
1224 # Gather information about all the apk files in the repo directory, using
1225 # cached data if possible.
1226 apkcachefile = os.path.join('tmp', 'apkcache')
1227 if not options.clean and os.path.exists(apkcachefile):
1228 with open(apkcachefile, 'rb') as cf:
1229 apkcache = pickle.load(cf, encoding='utf-8')
1230 if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
1235 delete_disabled_builds(apps, apkcache, repodirs)
1237 # Scan all apks in the main repo
1238 apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1240 # Generate warnings for apk's with no metadata (or create skeleton
1241 # metadata files, if requested on the command line)
1244 if apk['id'] not in apps:
1245 if options.create_metadata:
1246 if 'name' not in apk:
1247 logging.error(apk['id'] + ' does not have a name! Skipping...')
1249 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
1250 f.write("License:Unknown\n")
1251 f.write("Web Site:\n")
1252 f.write("Source Code:\n")
1253 f.write("Issue Tracker:\n")
1254 f.write("Changelog:\n")
1255 f.write("Summary:" + apk['name'] + "\n")
1256 f.write("Description:\n")
1257 f.write(apk['name'] + "\n")
1260 logging.info("Generated skeleton metadata for " + apk['id'])
1263 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1264 if options.delete_unknown:
1265 logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1266 rmf = os.path.join(repodirs[0], apk['apkname'])
1267 if not os.path.exists(rmf):
1268 logging.error("Could not find {0} to remove it".format(rmf))
1272 logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1274 # update the metadata with the newly created ones included
1276 apps = metadata.read_metadata()
1278 # Scan the archive repo for apks as well
1279 if len(repodirs) > 1:
1280 archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1286 # Some information from the apks needs to be applied up to the application
1287 # level. When doing this, we use the info from the most recent version's apk.
1288 # We deal with figuring out when the app was added and last updated at the
1290 for appid, app in apps.items():
1292 for apk in apks + archapks:
1293 if apk['id'] == appid:
1294 if apk['versioncode'] > bestver:
1295 bestver = apk['versioncode']
1299 if not app.added or apk['added'] < app.added:
1300 app.added = apk['added']
1301 if not app.lastupdated or apk['added'] > app.lastupdated:
1302 app.lastupdated = apk['added']
1305 logging.debug("Don't know when " + appid + " was added")
1306 if not app.lastupdated:
1307 logging.debug("Don't know when " + appid + " was last updated")
1310 if app.Name is None:
1311 app.Name = app.AutoName or appid
1313 logging.debug("Application " + appid + " has no packages")
1315 if app.Name is None:
1316 app.Name = bestapk['name']
1317 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1318 if app.CurrentVersionCode is None:
1319 app.CurrentVersionCode = str(bestver)
1321 # Sort the app list by name, then the web site doesn't have to by default.
1322 # (we had to wait until we'd scanned the apks to do this, because mostly the
1323 # name comes from there!)
1324 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1326 # APKs are placed into multiple repos based on the app package, providing
1327 # per-app subscription feeds for nightly builds and things like it
1328 if config['per_app_repos']:
1329 add_apks_to_per_app_repos(repodirs[0], apks)
1330 for appid, app in apps.items():
1331 repodir = os.path.join(appid, 'fdroid', 'repo')
1333 appdict[appid] = app
1334 if os.path.isdir(repodir):
1335 make_index(appdict, [appid], apks, repodir, False, categories)
1337 logging.info('Skipping index generation for ' + appid)
1340 if len(repodirs) > 1:
1341 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1343 # Make the index for the main repo...
1344 make_index(apps, sortedids, apks, repodirs[0], False, categories)
1346 # If there's an archive repo, make the index for it. We already scanned it
1348 if len(repodirs) > 1:
1349 make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1351 if config['update_stats']:
1353 # Update known apks info...
1354 knownapks.writeifchanged()
1356 # Generate latest apps data for widget
1357 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1359 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1361 appid = line.rstrip()
1362 data += appid + "\t"
1364 data += app.Name + "\t"
1365 if app.icon is not None:
1366 data += app.icon + "\t"
1367 data += app.License + "\n"
1368 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1372 apkcache["METADATA_VERSION"] = METADATA_VERSION
1373 with open(apkcachefile, 'wb') as cf:
1374 pickle.dump(apkcache, cf)
1376 # Update the wiki...
1378 update_wiki(apps, sortedids, apks + archapks)
1380 logging.info("Finished.")
1382 if __name__ == "__main__":