3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
30 from datetime import datetime, timedelta
31 from xml.dom.minidom import Document
32 from argparse import ArgumentParser
34 from pyasn1.error import PyAsn1Error
35 from pyasn1.codec.der import decoder, encoder
36 from pyasn1_modules import rfc2315
37 from binascii import hexlify, unhexlify
43 from . import metadata
44 from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
45 from .metadata import MetaDataException
47 screen_densities = ['640', '480', '320', '240', '160', '120']
49 all_screen_densities = ['0'] + screen_densities
52 def dpi_to_px(density):
53 return (int(density) * 48) / 160
57 return (int(px) * 160) / 48
60 def get_icon_dir(repodir, density):
62 return os.path.join(repodir, "icons")
63 return os.path.join(repodir, "icons-%s" % density)
66 def get_icon_dirs(repodir):
67 for density in screen_densities:
68 yield get_icon_dir(repodir, density)
71 def get_all_icon_dirs(repodir):
72 for density in all_screen_densities:
73 yield get_icon_dir(repodir, density)
76 def update_wiki(apps, sortedids, apks):
79 :param apps: fully populated list of all applications
80 :param apks: all apks, except...
82 logging.info("Updating wiki")
84 wikiredircat = 'App Redirects'
86 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
87 path=config['wiki_path'])
88 site.login(config['wiki_user'], config['wiki_password'])
90 generated_redirects = {}
92 for appid in sortedids:
97 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
99 for af in app.AntiFeatures:
100 wikidata += '{{AntiFeature|' + af + '}}\n'
105 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
108 time.strftime('%Y-%m-%d', app.added) if app.added else '',
109 time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
124 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
126 wikidata += app.Summary
127 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
129 wikidata += "=Description=\n"
130 wikidata += metadata.description_wiki(app.Description) + "\n"
132 wikidata += "=Maintainer Notes=\n"
133 if app.MaintainerNotes:
134 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
135 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
137 # Get a list of all packages for this application...
139 gotcurrentver = False
143 if apk['id'] == appid:
144 if str(apk['versioncode']) == app.CurrentVersionCode:
147 # Include ones we can't build, as a special case...
148 for build in app.builds:
150 if build.vercode == app.CurrentVersionCode:
152 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
153 apklist.append({'versioncode': int(build.vercode),
154 'version': build.version,
155 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
160 if apk['versioncode'] == int(build.vercode):
165 apklist.append({'versioncode': int(build.vercode),
166 'version': build.version,
167 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
169 if app.CurrentVersionCode == '0':
171 # Sort with most recent first...
172 apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
174 wikidata += "=Versions=\n"
175 if len(apklist) == 0:
176 wikidata += "We currently have no versions of this app available."
177 elif not gotcurrentver:
178 wikidata += "We don't have the current version of this app."
180 wikidata += "We have the current version of this app."
181 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
182 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
183 if len(app.NoSourceSince) > 0:
184 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
185 if len(app.CurrentVersion) > 0:
186 wikidata += "The current (recommended) version is " + app.CurrentVersion
187 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
190 wikidata += "==" + apk['version'] + "==\n"
192 if 'buildproblem' in apk:
193 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
196 wikidata += "This version is built and signed by "
198 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
200 wikidata += "the original developer.\n\n"
201 wikidata += "Version code: " + str(apk['versioncode']) + '\n'
203 wikidata += '\n[[Category:' + wikicat + ']]\n'
204 if len(app.NoSourceSince) > 0:
205 wikidata += '\n[[Category:Apps missing source code]]\n'
206 if validapks == 0 and not app.Disabled:
207 wikidata += '\n[[Category:Apps with no packages]]\n'
208 if cantupdate and not app.Disabled:
209 wikidata += "\n[[Category:Apps we cannot update]]\n"
210 if buildfails and not app.Disabled:
211 wikidata += "\n[[Category:Apps with failing builds]]\n"
212 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
213 wikidata += '\n[[Category:Apps to Update]]\n'
215 wikidata += '\n[[Category:Apps that are disabled]]\n'
216 if app.UpdateCheckMode == 'None' and not app.Disabled:
217 wikidata += '\n[[Category:Apps with no update check]]\n'
218 for appcat in app.Categories:
219 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
221 # We can't have underscores in the page name, even if they're in
222 # the package ID, because MediaWiki messes with them...
223 pagename = appid.replace('_', ' ')
225 # Drop a trailing newline, because mediawiki is going to drop it anyway
226 # and it we don't we'll think the page has changed when it hasn't...
227 if wikidata.endswith('\n'):
228 wikidata = wikidata[:-1]
230 generated_pages[pagename] = wikidata
232 # Make a redirect from the name to the ID too, unless there's
233 # already an existing page with the name and it isn't a redirect.
235 apppagename = app.Name.replace('_', ' ')
236 apppagename = apppagename.replace('{', '')
237 apppagename = apppagename.replace('}', ' ')
238 apppagename = apppagename.replace(':', ' ')
239 # Drop double spaces caused mostly by replacing ':' above
240 apppagename = apppagename.replace(' ', ' ')
241 for expagename in site.allpages(prefix=apppagename,
242 filterredir='nonredirects',
244 if expagename == apppagename:
246 # Another reason not to make the redirect page is if the app name
247 # is the same as it's ID, because that will overwrite the real page
248 # with an redirect to itself! (Although it seems like an odd
249 # scenario this happens a lot, e.g. where there is metadata but no
250 # builds or binaries to extract a name from.
251 if apppagename == pagename:
254 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
256 for tcat, genp in [(wikicat, generated_pages),
257 (wikiredircat, generated_redirects)]:
258 catpages = site.Pages['Category:' + tcat]
260 for page in catpages:
261 existingpages.append(page.name)
262 if page.name in genp:
263 pagetxt = page.edit()
264 if pagetxt != genp[page.name]:
265 logging.debug("Updating modified page " + page.name)
266 page.save(genp[page.name], summary='Auto-updated')
268 logging.debug("Page " + page.name + " is unchanged")
270 logging.warn("Deleting page " + page.name)
271 page.delete('No longer published')
272 for pagename, text in genp.items():
273 logging.debug("Checking " + pagename)
274 if pagename not in existingpages:
275 logging.debug("Creating page " + pagename)
277 newpage = site.Pages[pagename]
278 newpage.save(text, summary='Auto-created')
280 logging.error("...FAILED to create page '{0}'".format(pagename))
282 # Purge server cache to ensure counts are up to date
283 site.pages['Repository Maintenance'].purge()
286 def delete_disabled_builds(apps, apkcache, repodirs):
287 """Delete disabled build outputs.
289 :param apps: list of all applications, as per metadata.read_metadata
290 :param apkcache: current apk cache information
291 :param repodirs: the repo directories to process
293 for appid, app in apps.items():
294 for build in app.builds:
295 if not build.disable:
297 apkfilename = appid + '_' + str(build.vercode) + '.apk'
298 iconfilename = "%s.%s.png" % (
301 for repodir in repodirs:
303 os.path.join(repodir, apkfilename),
304 os.path.join(repodir, apkfilename + '.asc'),
305 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
307 for density in all_screen_densities:
308 repo_dir = get_icon_dir(repodir, density)
309 files.append(os.path.join(repo_dir, iconfilename))
312 if os.path.exists(f):
313 logging.info("Deleting disabled build output " + f)
315 if apkfilename in apkcache:
316 del apkcache[apkfilename]
319 def resize_icon(iconpath, density):
321 if not os.path.isfile(iconpath):
325 im = Image.open(iconpath)
326 size = dpi_to_px(density)
328 if any(length > size for length in im.size):
330 im.thumbnail((size, size), Image.ANTIALIAS)
331 logging.debug("%s was too large at %s - new size is %s" % (
332 iconpath, oldsize, im.size))
333 im.save(iconpath, "PNG")
335 except Exception as e:
336 logging.error("Failed resizing {0} - {1}".format(iconpath, e))
339 def resize_all_icons(repodirs):
340 """Resize all icons that exceed the max size
342 :param repodirs: the repo directories to process
344 for repodir in repodirs:
345 for density in screen_densities:
346 icon_dir = get_icon_dir(repodir, density)
347 icon_glob = os.path.join(icon_dir, '*.png')
348 for iconpath in glob.glob(icon_glob):
349 resize_icon(iconpath, density)
352 # A signature block file with a .DSA, .RSA, or .EC extension
353 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
357 """ Get the signing certificate of an apk. To get the same md5 has that
358 Android gets, we encode the .RSA certificate in a specific format and pass
359 it hex-encoded to the md5 digest algorithm.
361 :param apkpath: path to the apk
362 :returns: A string containing the md5 of the signature of the apk or None
363 if an error occurred.
368 # verify the jar signature is correct
369 args = [config['jarsigner'], '-verify', apkpath]
370 p = FDroidPopen(args)
371 if p.returncode != 0:
372 logging.critical(apkpath + " has a bad signature!")
375 with zipfile.ZipFile(apkpath, 'r') as apk:
377 certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
380 logging.error("Found no signing certificates on %s" % apkpath)
383 logging.error("Found multiple signing certificates on %s" % apkpath)
386 cert = apk.read(certs[0])
388 content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
389 if content.getComponentByName('contentType') != rfc2315.signedData:
390 logging.error("Unexpected format.")
393 content = decoder.decode(content.getComponentByName('content'),
394 asn1Spec=rfc2315.SignedData())[0]
396 certificates = content.getComponentByName('certificates')
398 logging.error("Certificates not found.")
401 cert_encoded = encoder.encode(certificates)[4:]
403 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
406 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
407 """Scan the apks in the given repo directory.
409 This also extracts the icons.
411 :param apps: list of all applications, as per metadata.read_metadata
412 :param apkcache: current apk cache information
413 :param repodir: repo directory to scan
414 :param knownapks: known apks info
415 :param use_date_from_apk: use date from APK (instead of current date)
417 :returns: (apks, cachechanged) where apks is a list of apk information,
418 and cachechanged is True if the apkcache got changed.
423 for icon_dir in get_all_icon_dirs(repodir):
424 if os.path.exists(icon_dir):
426 shutil.rmtree(icon_dir)
427 os.makedirs(icon_dir)
429 os.makedirs(icon_dir)
432 name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
433 vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
434 vername_pat = re.compile(".*versionName='([^']*)'.*")
435 label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
436 icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
437 icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
438 sdkversion_pat = re.compile(".*'([0-9]*)'.*")
439 string_pat = re.compile(".*'([^']*)'.*")
440 for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
442 apkfilename = apkfile[len(repodir) + 1:]
443 if ' ' in apkfilename:
444 logging.critical("Spaces in filenames are not allowed.")
447 # Calculate the sha256...
448 sha = hashlib.sha256()
449 with open(apkfile, 'rb') as f:
455 shasum = sha.hexdigest()
458 if apkfilename in apkcache:
459 apk = apkcache[apkfilename]
460 if apk['sha256'] == shasum:
461 logging.debug("Reading " + apkfilename + " from cache")
464 logging.debug("Ignoring stale cache data for " + apkfilename)
467 logging.debug("Processing " + apkfilename)
469 apk['apkname'] = apkfilename
470 apk['sha256'] = shasum
471 srcfilename = apkfilename[:-4] + "_src.tar.gz"
472 if os.path.exists(os.path.join(repodir, srcfilename)):
473 apk['srcname'] = srcfilename
474 apk['size'] = os.path.getsize(apkfile)
475 apk['permissions'] = set()
476 apk['features'] = set()
477 apk['icons_src'] = {}
479 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
480 if p.returncode != 0:
481 if options.delete_unknown:
482 if os.path.exists(apkfile):
483 logging.error("Failed to get apk information, deleting " + apkfile)
486 logging.error("Could not find {0} to remove it".format(apkfile))
488 logging.error("Failed to get apk information, skipping " + apkfile)
490 for line in p.output.splitlines():
491 if line.startswith("package:"):
493 apk['id'] = re.match(name_pat, line).group(1)
494 apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
495 apk['version'] = re.match(vername_pat, line).group(1)
496 except Exception as e:
497 logging.error("Package matching failed: " + str(e))
498 logging.info("Line was: " + line)
500 elif line.startswith("application:"):
501 apk['name'] = re.match(label_pat, line).group(1)
502 # Keep path to non-dpi icon in case we need it
503 match = re.match(icon_pat_nodpi, line)
505 apk['icons_src']['-1'] = match.group(1)
506 elif line.startswith("launchable-activity:"):
507 # Only use launchable-activity as fallback to application
509 apk['name'] = re.match(label_pat, line).group(1)
510 if '-1' not in apk['icons_src']:
511 match = re.match(icon_pat_nodpi, line)
513 apk['icons_src']['-1'] = match.group(1)
514 elif line.startswith("application-icon-"):
515 match = re.match(icon_pat, line)
517 density = match.group(1)
518 path = match.group(2)
519 apk['icons_src'][density] = path
520 elif line.startswith("sdkVersion:"):
521 m = re.match(sdkversion_pat, line)
523 logging.error(line.replace('sdkVersion:', '')
524 + ' is not a valid minSdkVersion!')
526 apk['sdkversion'] = m.group(1)
527 elif line.startswith("maxSdkVersion:"):
528 apk['maxSdkVersion'] = re.match(sdkversion_pat, line).group(1)
529 elif line.startswith("native-code:"):
530 apk['nativecode'] = []
531 for arch in line[13:].split(' '):
532 apk['nativecode'].append(arch[1:-1])
533 elif line.startswith("uses-permission:"):
534 perm = re.match(string_pat, line).group(1)
535 if perm.startswith("android.permission."):
537 apk['permissions'].add(perm)
538 elif line.startswith("uses-feature:"):
539 perm = re.match(string_pat, line).group(1)
540 # Filter out this, it's only added with the latest SDK tools and
541 # causes problems for lots of apps.
542 if perm != "android.hardware.screen.portrait" \
543 and perm != "android.hardware.screen.landscape":
544 if perm.startswith("android.feature."):
546 apk['features'].add(perm)
548 if 'sdkversion' not in apk:
549 logging.warn("No SDK version information found in {0}".format(apkfile))
550 apk['sdkversion'] = 0
552 # Check for debuggable apks...
553 if common.isApkDebuggable(apkfile, config):
554 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
556 # Get the signature (or md5 of, to be precise)...
557 logging.debug('Getting signature of {0}'.format(apkfile))
558 apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
560 logging.critical("Failed to get apk signature")
563 apkzip = zipfile.ZipFile(apkfile, 'r')
565 # if an APK has files newer than the system time, suggest updating
566 # the system clock. This is useful for offline systems, used for
567 # signing, which do not have another source of clock sync info. It
568 # has to be more than 24 hours newer because ZIP/APK files do not
569 # store timezone info
570 manifest = apkzip.getinfo('AndroidManifest.xml')
571 if manifest.date_time[1] == 0: # month can't be zero
572 logging.debug('AndroidManifest.xml has no date')
574 dt_obj = datetime(*manifest.date_time)
575 checkdt = dt_obj - timedelta(1)
576 if datetime.today() < checkdt:
577 logging.warn('System clock is older than manifest in: '
579 + '\nSet clock to that time using:\n'
580 + 'sudo date -s "' + str(dt_obj) + '"')
582 iconfilename = "%s.%s.png" % (
586 # Extract the icon file...
588 for density in screen_densities:
589 if density not in apk['icons_src']:
590 empty_densities.append(density)
592 iconsrc = apk['icons_src'][density]
593 icon_dir = get_icon_dir(repodir, density)
594 icondest = os.path.join(icon_dir, iconfilename)
597 with open(icondest, 'wb') as f:
598 f.write(apkzip.read(iconsrc))
599 apk['icons'][density] = iconfilename
602 logging.warn("Error retrieving icon file")
603 del apk['icons'][density]
604 del apk['icons_src'][density]
605 empty_densities.append(density)
607 if '-1' in apk['icons_src']:
608 iconsrc = apk['icons_src']['-1']
609 iconpath = os.path.join(
610 get_icon_dir(repodir, '0'), iconfilename)
611 with open(iconpath, 'wb') as f:
612 f.write(apkzip.read(iconsrc))
614 im = Image.open(iconpath)
615 dpi = px_to_dpi(im.size[0])
616 for density in screen_densities:
617 if density in apk['icons']:
619 if density == screen_densities[-1] or dpi >= int(density):
620 apk['icons'][density] = iconfilename
621 shutil.move(iconpath,
622 os.path.join(get_icon_dir(repodir, density), iconfilename))
623 empty_densities.remove(density)
625 except Exception as e:
626 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
629 apk['icon'] = iconfilename
633 # First try resizing down to not lose quality
635 for density in screen_densities:
636 if density not in empty_densities:
637 last_density = density
639 if last_density is None:
641 logging.debug("Density %s not available, resizing down from %s"
642 % (density, last_density))
644 last_iconpath = os.path.join(
645 get_icon_dir(repodir, last_density), iconfilename)
646 iconpath = os.path.join(
647 get_icon_dir(repodir, density), iconfilename)
649 im = Image.open(last_iconpath)
651 logging.warn("Invalid image file at %s" % last_iconpath)
654 size = dpi_to_px(density)
656 im.thumbnail((size, size), Image.ANTIALIAS)
657 im.save(iconpath, "PNG")
658 empty_densities.remove(density)
660 # Then just copy from the highest resolution available
662 for density in reversed(screen_densities):
663 if density not in empty_densities:
664 last_density = density
666 if last_density is None:
668 logging.debug("Density %s not available, copying from lower density %s"
669 % (density, last_density))
672 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
673 os.path.join(get_icon_dir(repodir, density), iconfilename))
675 empty_densities.remove(density)
677 for density in screen_densities:
678 icon_dir = get_icon_dir(repodir, density)
679 icondest = os.path.join(icon_dir, iconfilename)
680 resize_icon(icondest, density)
682 # Copy from icons-mdpi to icons since mdpi is the baseline density
683 baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
684 if os.path.isfile(baseline):
685 apk['icons']['0'] = iconfilename
686 shutil.copyfile(baseline,
687 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
689 # Record in known apks, getting the added date at the same time..
690 added = knownapks.recordapk(apk['apkname'], apk['id'])
692 if use_date_from_apk and manifest.date_time[1] != 0:
693 added = datetime(*manifest.date_time).timetuple()
694 logging.debug("Using date from APK")
698 apkcache[apkfilename] = apk
703 return apks, cachechanged
706 repo_pubkey_fingerprint = None
709 # Generate a certificate fingerprint the same way keytool does it
710 # (but with slightly different formatting)
711 def cert_fingerprint(data):
712 digest = hashlib.sha256(data).digest()
714 ret.append(' '.join("%02X" % b for b in bytearray(digest)))
718 def extract_pubkey():
719 global repo_pubkey_fingerprint
720 if 'repo_pubkey' in config:
721 pubkey = unhexlify(config['repo_pubkey'])
723 p = FDroidPopenBytes([config['keytool'], '-exportcert',
724 '-alias', config['repo_keyalias'],
725 '-keystore', config['keystore'],
726 '-storepass:file', config['keystorepassfile']]
727 + config['smartcardoptions'],
728 output=False, stderr_to_stdout=False)
729 if p.returncode != 0 or len(p.output) < 20:
730 msg = "Failed to get repo pubkey!"
731 if config['keystore'] == 'NONE':
732 msg += ' Is your crypto smartcard plugged in?'
733 logging.critical(msg)
736 repo_pubkey_fingerprint = cert_fingerprint(pubkey)
737 return hexlify(pubkey)
740 def make_index(apps, sortedids, apks, repodir, archive, categories):
741 """Make a repo index.
743 :param apps: fully populated apps list
744 :param apks: full populated apks list
745 :param repodir: the repo directory
746 :param archive: True if this is the archive repo, False if it's the
748 :param categories: list of categories
753 def addElement(name, value, doc, parent):
754 el = doc.createElement(name)
755 el.appendChild(doc.createTextNode(value))
756 parent.appendChild(el)
758 def addElementNonEmpty(name, value, doc, parent):
761 addElement(name, value, doc, parent)
763 def addElementCDATA(name, value, doc, parent):
764 el = doc.createElement(name)
765 el.appendChild(doc.createCDATASection(value))
766 parent.appendChild(el)
768 root = doc.createElement("fdroid")
769 doc.appendChild(root)
771 repoel = doc.createElement("repo")
773 mirrorcheckfailed = False
774 for mirror in config.get('mirrors', []):
775 base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
776 if config.get('nonstandardwebroot') is not True and base != 'fdroid':
777 logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
778 mirrorcheckfailed = True
779 if mirrorcheckfailed:
783 repoel.setAttribute("name", config['archive_name'])
784 if config['repo_maxage'] != 0:
785 repoel.setAttribute("maxage", str(config['repo_maxage']))
786 repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
787 repoel.setAttribute("url", config['archive_url'])
788 addElement('description', config['archive_description'], doc, repoel)
789 urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
790 for mirror in config.get('mirrors', []):
791 addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
794 repoel.setAttribute("name", config['repo_name'])
795 if config['repo_maxage'] != 0:
796 repoel.setAttribute("maxage", str(config['repo_maxage']))
797 repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
798 repoel.setAttribute("url", config['repo_url'])
799 addElement('description', config['repo_description'], doc, repoel)
800 urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
801 for mirror in config.get('mirrors', []):
802 addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
804 repoel.setAttribute("version", "15")
805 repoel.setAttribute("timestamp", str(int(time.time())))
808 if not options.nosign:
809 if 'repo_keyalias' not in config:
811 logging.critical("'repo_keyalias' not found in config.py!")
812 if 'keystore' not in config:
814 logging.critical("'keystore' not found in config.py!")
815 if 'keystorepass' not in config and 'keystorepassfile' not in config:
817 logging.critical("'keystorepass' not found in config.py!")
818 if 'keypass' not in config and 'keypassfile' not in config:
820 logging.critical("'keypass' not found in config.py!")
821 if not os.path.exists(config['keystore']):
823 logging.critical("'" + config['keystore'] + "' does not exist!")
825 logging.warning("`fdroid update` requires a signing key, you can create one using:")
826 logging.warning("\tfdroid update --create-key")
829 repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
830 root.appendChild(repoel)
832 for appid in sortedids:
835 if app.Disabled is not None:
838 # Get a list of the apks for this app...
841 if apk['id'] == appid:
844 if len(apklist) == 0:
847 apel = doc.createElement("application")
848 apel.setAttribute("id", app.id)
849 root.appendChild(apel)
851 addElement('id', app.id, doc, apel)
853 addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
855 addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
856 addElement('name', app.Name, doc, apel)
857 addElement('summary', app.Summary, doc, apel)
859 addElement('icon', app.icon, doc, apel)
863 return ("fdroid.app:" + appid, apps[appid].Name)
864 raise MetaDataException("Cannot resolve app id " + appid)
867 metadata.description_html(app.Description, linkres),
869 addElement('license', app.License, doc, apel)
871 addElement('categories', ','.join(app.Categories), doc, apel)
872 # We put the first (primary) category in LAST, which will have
873 # the desired effect of making clients that only understand one
874 # category see that one.
875 addElement('category', app.Categories[0], doc, apel)
876 addElement('web', app.WebSite, doc, apel)
877 addElement('source', app.SourceCode, doc, apel)
878 addElement('tracker', app.IssueTracker, doc, apel)
879 addElementNonEmpty('changelog', app.Changelog, doc, apel)
880 addElementNonEmpty('author', app.AuthorName, doc, apel)
881 addElementNonEmpty('email', app.AuthorEmail, doc, apel)
882 addElementNonEmpty('donate', app.Donate, doc, apel)
883 addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
884 addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
885 addElementNonEmpty('flattr', app.FlattrID, doc, apel)
887 # These elements actually refer to the current version (i.e. which
888 # one is recommended. They are historically mis-named, and need
889 # changing, but stay like this for now to support existing clients.
890 addElement('marketversion', app.CurrentVersion, doc, apel)
891 addElement('marketvercode', app.CurrentVersionCode, doc, apel)
894 af = app.AntiFeatures
896 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
898 pv = app.Provides.split(',')
899 addElementNonEmpty('provides', ','.join(pv), doc, apel)
901 addElement('requirements', 'root', doc, apel)
903 # Sort the apk list into version order, just so the web site
904 # doesn't have to do any work by default...
905 apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
907 # Check for duplicates - they will make the client unhappy...
908 for i in range(len(apklist) - 1):
909 if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
910 logging.critical("duplicate versions: '%s' - '%s'" % (
911 apklist[i]['apkname'], apklist[i + 1]['apkname']))
914 current_version_code = 0
915 current_version_file = None
917 # find the APK for the "Current Version"
918 if current_version_code < apk['versioncode']:
919 current_version_code = apk['versioncode']
920 if current_version_code < int(app.CurrentVersionCode):
921 current_version_file = apk['apkname']
923 apkel = doc.createElement("package")
924 apel.appendChild(apkel)
925 addElement('version', apk['version'], doc, apkel)
926 addElement('versioncode', str(apk['versioncode']), doc, apkel)
927 addElement('apkname', apk['apkname'], doc, apkel)
929 addElement('srcname', apk['srcname'], doc, apkel)
930 for hash_type in ['sha256']:
931 if hash_type not in apk:
933 hashel = doc.createElement("hash")
934 hashel.setAttribute("type", hash_type)
935 hashel.appendChild(doc.createTextNode(apk[hash_type]))
936 apkel.appendChild(hashel)
937 addElement('sig', apk['sig'], doc, apkel)
938 addElement('size', str(apk['size']), doc, apkel)
939 addElement('sdkver', str(apk['sdkversion']), doc, apkel)
940 if 'maxSdkVersion' in apk:
941 addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
943 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
944 addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
945 if 'nativecode' in apk:
946 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
947 addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
949 if current_version_file is not None \
950 and config['make_current_version_link'] \
951 and repodir == 'repo': # only create these
952 namefield = config['current_version_name_source']
953 sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
954 apklinkname = sanitized_name + '.apk'
955 current_version_path = os.path.join(repodir, current_version_file)
956 if os.path.islink(apklinkname):
957 os.remove(apklinkname)
958 os.symlink(current_version_path, apklinkname)
959 # also symlink gpg signature, if it exists
960 for extension in ('.asc', '.sig'):
961 sigfile_path = current_version_path + extension
962 if os.path.exists(sigfile_path):
963 siglinkname = apklinkname + extension
964 if os.path.islink(siglinkname):
965 os.remove(siglinkname)
966 os.symlink(sigfile_path, siglinkname)
969 output = doc.toprettyxml(encoding='utf-8')
971 output = doc.toxml(encoding='utf-8')
973 with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
976 if 'repo_keyalias' in config:
979 logging.info("Creating unsigned index in preparation for signing")
981 logging.info("Creating signed index with this key (SHA256):")
982 logging.info("%s" % repo_pubkey_fingerprint)
984 # Create a jar of the index...
985 jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
986 p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
987 if p.returncode != 0:
988 logging.critical("Failed to create {0}".format(jar_output))
992 signed = os.path.join(repodir, 'index.jar')
994 # Remove old signed index if not signing
995 if os.path.exists(signed):
998 args = [config['jarsigner'], '-keystore', config['keystore'],
999 '-storepass:file', config['keystorepassfile'],
1000 '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1001 signed, config['repo_keyalias']]
1002 if config['keystore'] == 'NONE':
1003 args += config['smartcardoptions']
1004 else: # smardcards never use -keypass
1005 args += ['-keypass:file', config['keypassfile']]
1006 p = FDroidPopen(args)
1007 if p.returncode != 0:
1008 logging.critical("Failed to sign index")
1011 # Copy the repo icon into the repo directory...
1012 icon_dir = os.path.join(repodir, 'icons')
1013 iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1014 shutil.copyfile(config['repo_icon'], iconfilename)
1016 # Write a category list in the repo to allow quick access...
1018 for cat in categories:
1019 catdata += cat + '\n'
1020 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1024 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1026 for appid, app in apps.items():
1028 if app.ArchivePolicy:
1029 keepversions = int(app.ArchivePolicy[:-9])
1031 keepversions = defaultkeepversions
1033 def filter_apk_list_sorted(apk_list):
1035 for apk in apk_list:
1036 if apk['id'] == appid:
1039 # Sort the apk list by version code. First is highest/newest.
1040 return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1042 def move_file(from_dir, to_dir, filename, ignore_missing):
1043 from_path = os.path.join(from_dir, filename)
1044 if ignore_missing and not os.path.exists(from_path):
1046 to_path = os.path.join(to_dir, filename)
1047 shutil.move(from_path, to_path)
1049 logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1050 .format(appid, len(apks), keepversions, len(archapks)))
1052 if len(apks) > keepversions:
1053 apklist = filter_apk_list_sorted(apks)
1054 # Move back the ones we don't want.
1055 for apk in apklist[keepversions:]:
1056 logging.info("Moving " + apk['apkname'] + " to archive")
1057 move_file(repodir, archivedir, apk['apkname'], False)
1058 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1059 for density in all_screen_densities:
1060 repo_icon_dir = get_icon_dir(repodir, density)
1061 archive_icon_dir = get_icon_dir(archivedir, density)
1062 if density not in apk['icons']:
1064 move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1065 if 'srcname' in apk:
1066 move_file(repodir, archivedir, apk['srcname'], False)
1067 archapks.append(apk)
1069 elif len(apks) < keepversions and len(archapks) > 0:
1070 required = keepversions - len(apks)
1071 archapklist = filter_apk_list_sorted(archapks)
1072 # Move forward the ones we want again.
1073 for apk in archapklist[:required]:
1074 logging.info("Moving " + apk['apkname'] + " from archive")
1075 move_file(archivedir, repodir, apk['apkname'], False)
1076 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1077 for density in all_screen_densities:
1078 repo_icon_dir = get_icon_dir(repodir, density)
1079 archive_icon_dir = get_icon_dir(archivedir, density)
1080 if density not in apk['icons']:
1082 move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1083 if 'srcname' in apk:
1084 move_file(archivedir, repodir, apk['srcname'], False)
1085 archapks.remove(apk)
1089 def add_apks_to_per_app_repos(repodir, apks):
1090 apks_per_app = dict()
1092 apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1093 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1094 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1095 apks_per_app[apk['id']] = apk
1097 if not os.path.exists(apk['per_app_icons']):
1098 logging.info('Adding new repo for only ' + apk['id'])
1099 os.makedirs(apk['per_app_icons'])
1101 apkpath = os.path.join(repodir, apk['apkname'])
1102 shutil.copy(apkpath, apk['per_app_repo'])
1103 apksigpath = apkpath + '.sig'
1104 if os.path.exists(apksigpath):
1105 shutil.copy(apksigpath, apk['per_app_repo'])
1106 apkascpath = apkpath + '.asc'
1107 if os.path.exists(apkascpath):
1108 shutil.copy(apkascpath, apk['per_app_repo'])
1117 global config, options
1119 # Parse command line...
1120 parser = ArgumentParser()
1121 common.setup_global_opts(parser)
1122 parser.add_argument("--create-key", action="store_true", default=False,
1123 help="Create a repo signing key in a keystore")
1124 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1125 help="Create skeleton metadata files that are missing")
1126 parser.add_argument("--delete-unknown", action="store_true", default=False,
1127 help="Delete APKs without metadata from the repo")
1128 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1129 help="Report on build data status")
1130 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1131 help="Interactively ask about things that need updating.")
1132 parser.add_argument("-I", "--icons", action="store_true", default=False,
1133 help="Resize all the icons exceeding the max pixel size and exit")
1134 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1135 help="Specify editor to use in interactive mode. Default " +
1136 "is /etc/alternatives/editor")
1137 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1138 help="Update the wiki")
1139 parser.add_argument("--pretty", action="store_true", default=False,
1140 help="Produce human-readable index.xml")
1141 parser.add_argument("--clean", action="store_true", default=False,
1142 help="Clean update - don't uses caches, reprocess all apks")
1143 parser.add_argument("--nosign", action="store_true", default=False,
1144 help="When configured for signed indexes, create only unsigned indexes at this stage")
1145 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1146 help="Use date from apk instead of current time for newly added apks")
1147 options = parser.parse_args()
1149 config = common.read_config(options)
1151 if not ('jarsigner' in config and 'keytool' in config):
1152 logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1156 if config['archive_older'] != 0:
1157 repodirs.append('archive')
1158 if not os.path.exists('archive'):
1162 resize_all_icons(repodirs)
1165 # check that icons exist now, rather than fail at the end of `fdroid update`
1166 for k in ['repo_icon', 'archive_icon']:
1168 if not os.path.exists(config[k]):
1169 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1172 # if the user asks to create a keystore, do it now, reusing whatever it can
1173 if options.create_key:
1174 if os.path.exists(config['keystore']):
1175 logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1176 logging.critical("\t'" + config['keystore'] + "'")
1179 if 'repo_keyalias' not in config:
1180 config['repo_keyalias'] = socket.getfqdn()
1181 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1182 if 'keydname' not in config:
1183 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1184 common.write_to_config(config, 'keydname', config['keydname'])
1185 if 'keystore' not in config:
1186 config['keystore'] = common.default_config.keystore
1187 common.write_to_config(config, 'keystore', config['keystore'])
1189 password = common.genpassword()
1190 if 'keystorepass' not in config:
1191 config['keystorepass'] = password
1192 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1193 if 'keypass' not in config:
1194 config['keypass'] = password
1195 common.write_to_config(config, 'keypass', config['keypass'])
1196 common.genkeystore(config)
1199 apps = metadata.read_metadata()
1201 # Generate a list of categories...
1203 for app in apps.values():
1204 categories.update(app.Categories)
1206 # Read known apks data (will be updated and written back when we've finished)
1207 knownapks = common.KnownApks()
1209 # Gather information about all the apk files in the repo directory, using
1210 # cached data if possible.
1211 apkcachefile = os.path.join('tmp', 'apkcache')
1212 if not options.clean and os.path.exists(apkcachefile):
1213 with open(apkcachefile, 'rb') as cf:
1214 apkcache = pickle.load(cf, encoding='utf-8')
1218 delete_disabled_builds(apps, apkcache, repodirs)
1220 # Scan all apks in the main repo
1221 apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1223 # Generate warnings for apk's with no metadata (or create skeleton
1224 # metadata files, if requested on the command line)
1227 if apk['id'] not in apps:
1228 if options.create_metadata:
1229 if 'name' not in apk:
1230 logging.error(apk['id'] + ' does not have a name! Skipping...')
1232 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
1233 f.write("License:Unknown\n")
1234 f.write("Web Site:\n")
1235 f.write("Source Code:\n")
1236 f.write("Issue Tracker:\n")
1237 f.write("Changelog:\n")
1238 f.write("Summary:" + apk['name'] + "\n")
1239 f.write("Description:\n")
1240 f.write(apk['name'] + "\n")
1243 logging.info("Generated skeleton metadata for " + apk['id'])
1246 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1247 if options.delete_unknown:
1248 logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1249 rmf = os.path.join(repodirs[0], apk['apkname'])
1250 if not os.path.exists(rmf):
1251 logging.error("Could not find {0} to remove it".format(rmf))
1255 logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1257 # update the metadata with the newly created ones included
1259 apps = metadata.read_metadata()
1261 # Scan the archive repo for apks as well
1262 if len(repodirs) > 1:
1263 archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1269 # Some information from the apks needs to be applied up to the application
1270 # level. When doing this, we use the info from the most recent version's apk.
1271 # We deal with figuring out when the app was added and last updated at the
1273 for appid, app in apps.items():
1275 for apk in apks + archapks:
1276 if apk['id'] == appid:
1277 if apk['versioncode'] > bestver:
1278 bestver = apk['versioncode']
1282 if not app.added or apk['added'] < app.added:
1283 app.added = apk['added']
1284 if not app.lastupdated or apk['added'] > app.lastupdated:
1285 app.lastupdated = apk['added']
1288 logging.debug("Don't know when " + appid + " was added")
1289 if not app.lastupdated:
1290 logging.debug("Don't know when " + appid + " was last updated")
1293 if app.Name is None:
1294 app.Name = app.AutoName or appid
1296 logging.debug("Application " + appid + " has no packages")
1298 if app.Name is None:
1299 app.Name = bestapk['name']
1300 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1301 if app.CurrentVersionCode is None:
1302 app.CurrentVersionCode = str(bestver)
1304 # Sort the app list by name, then the web site doesn't have to by default.
1305 # (we had to wait until we'd scanned the apks to do this, because mostly the
1306 # name comes from there!)
1307 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1309 # APKs are placed into multiple repos based on the app package, providing
1310 # per-app subscription feeds for nightly builds and things like it
1311 if config['per_app_repos']:
1312 add_apks_to_per_app_repos(repodirs[0], apks)
1313 for appid, app in apps.items():
1314 repodir = os.path.join(appid, 'fdroid', 'repo')
1316 appdict[appid] = app
1317 if os.path.isdir(repodir):
1318 make_index(appdict, [appid], apks, repodir, False, categories)
1320 logging.info('Skipping index generation for ' + appid)
1323 if len(repodirs) > 1:
1324 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1326 # Make the index for the main repo...
1327 make_index(apps, sortedids, apks, repodirs[0], False, categories)
1329 # If there's an archive repo, make the index for it. We already scanned it
1331 if len(repodirs) > 1:
1332 make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1334 if config['update_stats']:
1336 # Update known apks info...
1337 knownapks.writeifchanged()
1339 # Generate latest apps data for widget
1340 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1342 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1344 appid = line.rstrip()
1345 data += appid + "\t"
1347 data += app.Name + "\t"
1348 if app.icon is not None:
1349 data += app.icon + "\t"
1350 data += app.License + "\n"
1351 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1355 with open(apkcachefile, 'wb') as cf:
1356 pickle.dump(apkcache, cf)
1358 # Update the wiki...
1360 update_wiki(apps, sortedids, apks + archapks)
1362 logging.info("Finished.")
1364 if __name__ == "__main__":