3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
30 from datetime import datetime, timedelta
31 from xml.dom.minidom import Document
32 from argparse import ArgumentParser
34 from pyasn1.error import PyAsn1Error
35 from pyasn1.codec.der import decoder, encoder
36 from pyasn1_modules import rfc2315
37 from hashlib import md5
38 from binascii import hexlify, unhexlify
45 from common import FDroidPopen, SdkToolsPopen
46 from metadata import MetaDataException
48 screen_densities = ['640', '480', '320', '240', '160', '120']
50 all_screen_densities = ['0'] + screen_densities
53 def dpi_to_px(density):
54 return (int(density) * 48) / 160
58 return (int(px) * 160) / 48
61 def get_icon_dir(repodir, density):
63 return os.path.join(repodir, "icons")
64 return os.path.join(repodir, "icons-%s" % density)
67 def get_icon_dirs(repodir):
68 for density in screen_densities:
69 yield get_icon_dir(repodir, density)
72 def get_all_icon_dirs(repodir):
73 for density in all_screen_densities:
74 yield get_icon_dir(repodir, density)
77 def update_wiki(apps, sortedids, apks):
80 :param apps: fully populated list of all applications
81 :param apks: all apks, except...
83 logging.info("Updating wiki")
85 wikiredircat = 'App Redirects'
87 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
88 path=config['wiki_path'])
89 site.login(config['wiki_user'], config['wiki_password'])
91 generated_redirects = {}
93 for appid in sortedids:
98 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
100 for af in app.AntiFeatures:
101 wikidata += '{{AntiFeature|' + af + '}}\n'
106 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
109 time.strftime('%Y-%m-%d', app.added) if app.added else '',
110 time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
125 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
127 wikidata += app.Summary
128 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
130 wikidata += "=Description=\n"
131 wikidata += metadata.description_wiki(app.Description) + "\n"
133 wikidata += "=Maintainer Notes=\n"
134 if app.MaintainerNotes:
135 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
136 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
138 # Get a list of all packages for this application...
140 gotcurrentver = False
144 if apk['id'] == appid:
145 if str(apk['versioncode']) == app.CurrentVersionCode:
148 # Include ones we can't build, as a special case...
149 for build in app.builds:
151 if build.vercode == app.CurrentVersionCode:
153 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
154 apklist.append({'versioncode': int(build.vercode),
155 'version': build.version,
156 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
161 if apk['versioncode'] == int(build.vercode):
166 apklist.append({'versioncode': int(build.vercode),
167 'version': build.version,
168 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
170 if app.CurrentVersionCode == '0':
172 # Sort with most recent first...
173 apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
175 wikidata += "=Versions=\n"
176 if len(apklist) == 0:
177 wikidata += "We currently have no versions of this app available."
178 elif not gotcurrentver:
179 wikidata += "We don't have the current version of this app."
181 wikidata += "We have the current version of this app."
182 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
183 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
184 if len(app.NoSourceSince) > 0:
185 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
186 if len(app.CurrentVersion) > 0:
187 wikidata += "The current (recommended) version is " + app.CurrentVersion
188 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
191 wikidata += "==" + apk['version'] + "==\n"
193 if 'buildproblem' in apk:
194 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
197 wikidata += "This version is built and signed by "
199 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
201 wikidata += "the original developer.\n\n"
202 wikidata += "Version code: " + str(apk['versioncode']) + '\n'
204 wikidata += '\n[[Category:' + wikicat + ']]\n'
205 if len(app.NoSourceSince) > 0:
206 wikidata += '\n[[Category:Apps missing source code]]\n'
207 if validapks == 0 and not app.Disabled:
208 wikidata += '\n[[Category:Apps with no packages]]\n'
209 if cantupdate and not app.Disabled:
210 wikidata += "\n[[Category:Apps we cannot update]]\n"
211 if buildfails and not app.Disabled:
212 wikidata += "\n[[Category:Apps with failing builds]]\n"
213 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
214 wikidata += '\n[[Category:Apps to Update]]\n'
216 wikidata += '\n[[Category:Apps that are disabled]]\n'
217 if app.UpdateCheckMode == 'None' and not app.Disabled:
218 wikidata += '\n[[Category:Apps with no update check]]\n'
219 for appcat in app.Categories:
220 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
222 # We can't have underscores in the page name, even if they're in
223 # the package ID, because MediaWiki messes with them...
224 pagename = appid.replace('_', ' ')
226 # Drop a trailing newline, because mediawiki is going to drop it anyway
227 # and it we don't we'll think the page has changed when it hasn't...
228 if wikidata.endswith('\n'):
229 wikidata = wikidata[:-1]
231 generated_pages[pagename] = wikidata
233 # Make a redirect from the name to the ID too, unless there's
234 # already an existing page with the name and it isn't a redirect.
236 apppagename = app.Name.replace('_', ' ')
237 apppagename = apppagename.replace('{', '')
238 apppagename = apppagename.replace('}', ' ')
239 apppagename = apppagename.replace(':', ' ')
240 # Drop double spaces caused mostly by replacing ':' above
241 apppagename = apppagename.replace(' ', ' ')
242 for expagename in site.allpages(prefix=apppagename,
243 filterredir='nonredirects',
245 if expagename == apppagename:
247 # Another reason not to make the redirect page is if the app name
248 # is the same as it's ID, because that will overwrite the real page
249 # with an redirect to itself! (Although it seems like an odd
250 # scenario this happens a lot, e.g. where there is metadata but no
251 # builds or binaries to extract a name from.
252 if apppagename == pagename:
255 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
257 for tcat, genp in [(wikicat, generated_pages),
258 (wikiredircat, generated_redirects)]:
259 catpages = site.Pages['Category:' + tcat]
261 for page in catpages:
262 existingpages.append(page.name)
263 if page.name in genp:
264 pagetxt = page.edit()
265 if pagetxt != genp[page.name]:
266 logging.debug("Updating modified page " + page.name)
267 page.save(genp[page.name], summary='Auto-updated')
269 logging.debug("Page " + page.name + " is unchanged")
271 logging.warn("Deleting page " + page.name)
272 page.delete('No longer published')
273 for pagename, text in genp.items():
274 logging.debug("Checking " + pagename)
275 if pagename not in existingpages:
276 logging.debug("Creating page " + pagename)
278 newpage = site.Pages[pagename]
279 newpage.save(text, summary='Auto-created')
281 logging.error("...FAILED to create page '{0}'".format(pagename))
283 # Purge server cache to ensure counts are up to date
284 site.pages['Repository Maintenance'].purge()
287 def delete_disabled_builds(apps, apkcache, repodirs):
288 """Delete disabled build outputs.
290 :param apps: list of all applications, as per metadata.read_metadata
291 :param apkcache: current apk cache information
292 :param repodirs: the repo directories to process
294 for appid, app in apps.iteritems():
295 for build in app.builds:
296 if not build.disable:
298 apkfilename = appid + '_' + str(build.vercode) + '.apk'
299 iconfilename = "%s.%s.png" % (
302 for repodir in repodirs:
304 os.path.join(repodir, apkfilename),
305 os.path.join(repodir, apkfilename + '.asc'),
306 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
308 for density in all_screen_densities:
309 repo_dir = get_icon_dir(repodir, density)
310 files.append(os.path.join(repo_dir, iconfilename))
313 if os.path.exists(f):
314 logging.info("Deleting disabled build output " + f)
316 if apkfilename in apkcache:
317 del apkcache[apkfilename]
320 def resize_icon(iconpath, density):
322 if not os.path.isfile(iconpath):
326 im = Image.open(iconpath)
327 size = dpi_to_px(density)
329 if any(length > size for length in im.size):
331 im.thumbnail((size, size), Image.ANTIALIAS)
332 logging.debug("%s was too large at %s - new size is %s" % (
333 iconpath, oldsize, im.size))
334 im.save(iconpath, "PNG")
336 except Exception as e:
337 logging.error("Failed resizing {0} - {1}".format(iconpath, e))
340 def resize_all_icons(repodirs):
341 """Resize all icons that exceed the max size
343 :param repodirs: the repo directories to process
345 for repodir in repodirs:
346 for density in screen_densities:
347 icon_dir = get_icon_dir(repodir, density)
348 icon_glob = os.path.join(icon_dir, '*.png')
349 for iconpath in glob.glob(icon_glob):
350 resize_icon(iconpath, density)
353 # A signature block file with a .DSA, .RSA, or .EC extension
354 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
358 """ Get the signing certificate of an apk. To get the same md5 has that
359 Android gets, we encode the .RSA certificate in a specific format and pass
360 it hex-encoded to the md5 digest algorithm.
362 :param apkpath: path to the apk
363 :returns: A string containing the md5 of the signature of the apk or None
364 if an error occurred.
369 # verify the jar signature is correct
370 args = [config['jarsigner'], '-verify', apkpath]
371 p = FDroidPopen(args)
372 if p.returncode != 0:
373 logging.critical(apkpath + " has a bad signature!")
376 with zipfile.ZipFile(apkpath, 'r') as apk:
378 certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
381 logging.error("Found no signing certificates on %s" % apkpath)
384 logging.error("Found multiple signing certificates on %s" % apkpath)
387 cert = apk.read(certs[0])
389 content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
390 if content.getComponentByName('contentType') != rfc2315.signedData:
391 logging.error("Unexpected format.")
394 content = decoder.decode(content.getComponentByName('content'),
395 asn1Spec=rfc2315.SignedData())[0]
397 certificates = content.getComponentByName('certificates')
399 logging.error("Certificates not found.")
402 cert_encoded = encoder.encode(certificates)[4:]
404 return md5(cert_encoded.encode('hex')).hexdigest()
407 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
408 """Scan the apks in the given repo directory.
410 This also extracts the icons.
412 :param apps: list of all applications, as per metadata.read_metadata
413 :param apkcache: current apk cache information
414 :param repodir: repo directory to scan
415 :param knownapks: known apks info
416 :param use_date_from_apk: use date from APK (instead of current date)
418 :returns: (apks, cachechanged) where apks is a list of apk information,
419 and cachechanged is True if the apkcache got changed.
424 for icon_dir in get_all_icon_dirs(repodir):
425 if os.path.exists(icon_dir):
427 shutil.rmtree(icon_dir)
428 os.makedirs(icon_dir)
430 os.makedirs(icon_dir)
433 name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
434 vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
435 vername_pat = re.compile(".*versionName='([^']*)'.*")
436 label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
437 icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
438 icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
439 sdkversion_pat = re.compile(".*'([0-9]*)'.*")
440 string_pat = re.compile(".*'([^']*)'.*")
441 for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
443 apkfilename = apkfile[len(repodir) + 1:]
444 if ' ' in apkfilename:
445 logging.critical("Spaces in filenames are not allowed.")
448 # Calculate the sha256...
449 sha = hashlib.sha256()
450 with open(apkfile, 'rb') as f:
456 shasum = sha.hexdigest()
459 if apkfilename in apkcache:
460 apk = apkcache[apkfilename]
461 if apk['sha256'] == shasum:
462 logging.debug("Reading " + apkfilename + " from cache")
465 logging.debug("Ignoring stale cache data for " + apkfilename)
468 logging.debug("Processing " + apkfilename)
470 apk['apkname'] = apkfilename
471 apk['sha256'] = shasum
472 srcfilename = apkfilename[:-4] + "_src.tar.gz"
473 if os.path.exists(os.path.join(repodir, srcfilename)):
474 apk['srcname'] = srcfilename
475 apk['size'] = os.path.getsize(apkfile)
476 apk['permissions'] = set()
477 apk['features'] = set()
478 apk['icons_src'] = {}
480 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
481 if p.returncode != 0:
482 if options.delete_unknown:
483 if os.path.exists(apkfile):
484 logging.error("Failed to get apk information, deleting " + apkfile)
487 logging.error("Could not find {0} to remove it".format(apkfile))
489 logging.error("Failed to get apk information, skipping " + apkfile)
491 for line in p.output.splitlines():
492 if line.startswith("package:"):
494 apk['id'] = re.match(name_pat, line).group(1)
495 apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
496 apk['version'] = re.match(vername_pat, line).group(1)
497 except Exception as e:
498 logging.error("Package matching failed: " + str(e))
499 logging.info("Line was: " + line)
501 elif line.startswith("application:"):
502 apk['name'] = re.match(label_pat, line).group(1)
503 # Keep path to non-dpi icon in case we need it
504 match = re.match(icon_pat_nodpi, line)
506 apk['icons_src']['-1'] = match.group(1)
507 elif line.startswith("launchable-activity:"):
508 # Only use launchable-activity as fallback to application
510 apk['name'] = re.match(label_pat, line).group(1)
511 if '-1' not in apk['icons_src']:
512 match = re.match(icon_pat_nodpi, line)
514 apk['icons_src']['-1'] = match.group(1)
515 elif line.startswith("application-icon-"):
516 match = re.match(icon_pat, line)
518 density = match.group(1)
519 path = match.group(2)
520 apk['icons_src'][density] = path
521 elif line.startswith("sdkVersion:"):
522 m = re.match(sdkversion_pat, line)
524 logging.error(line.replace('sdkVersion:', '')
525 + ' is not a valid minSdkVersion!')
527 apk['sdkversion'] = m.group(1)
528 elif line.startswith("maxSdkVersion:"):
529 apk['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
530 elif line.startswith("native-code:"):
531 apk['nativecode'] = []
532 for arch in line[13:].split(' '):
533 apk['nativecode'].append(arch[1:-1])
534 elif line.startswith("uses-permission:"):
535 perm = re.match(string_pat, line).group(1)
536 if perm.startswith("android.permission."):
538 apk['permissions'].add(perm)
539 elif line.startswith("uses-feature:"):
540 perm = re.match(string_pat, line).group(1)
541 # Filter out this, it's only added with the latest SDK tools and
542 # causes problems for lots of apps.
543 if perm != "android.hardware.screen.portrait" \
544 and perm != "android.hardware.screen.landscape":
545 if perm.startswith("android.feature."):
547 apk['features'].add(perm)
549 if 'sdkversion' not in apk:
550 logging.warn("No SDK version information found in {0}".format(apkfile))
551 apk['sdkversion'] = 0
553 # Check for debuggable apks...
554 if common.isApkDebuggable(apkfile, config):
555 logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
557 # Get the signature (or md5 of, to be precise)...
558 logging.debug('Getting signature of {0}'.format(apkfile))
559 apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
561 logging.critical("Failed to get apk signature")
564 apkzip = zipfile.ZipFile(apkfile, 'r')
566 # if an APK has files newer than the system time, suggest updating
567 # the system clock. This is useful for offline systems, used for
568 # signing, which do not have another source of clock sync info. It
569 # has to be more than 24 hours newer because ZIP/APK files do not
570 # store timezone info
571 manifest = apkzip.getinfo('AndroidManifest.xml')
572 if manifest.date_time[1] == 0: # month can't be zero
573 logging.debug('AndroidManifest.xml has no date')
575 dt_obj = datetime(*manifest.date_time)
576 checkdt = dt_obj - timedelta(1)
577 if datetime.today() < checkdt:
578 logging.warn('System clock is older than manifest in: '
580 + '\nSet clock to that time using:\n'
581 + 'sudo date -s "' + str(dt_obj) + '"')
583 iconfilename = "%s.%s.png" % (
587 # Extract the icon file...
589 for density in screen_densities:
590 if density not in apk['icons_src']:
591 empty_densities.append(density)
593 iconsrc = apk['icons_src'][density]
594 icon_dir = get_icon_dir(repodir, density)
595 icondest = os.path.join(icon_dir, iconfilename)
598 with open(icondest, 'wb') as f:
599 f.write(apkzip.read(iconsrc))
600 apk['icons'][density] = iconfilename
603 logging.warn("Error retrieving icon file")
604 del apk['icons'][density]
605 del apk['icons_src'][density]
606 empty_densities.append(density)
608 if '-1' in apk['icons_src']:
609 iconsrc = apk['icons_src']['-1']
610 iconpath = os.path.join(
611 get_icon_dir(repodir, '0'), iconfilename)
612 with open(iconpath, 'wb') as f:
613 f.write(apkzip.read(iconsrc))
615 im = Image.open(iconpath)
616 dpi = px_to_dpi(im.size[0])
617 for density in screen_densities:
618 if density in apk['icons']:
620 if density == screen_densities[-1] or dpi >= int(density):
621 apk['icons'][density] = iconfilename
622 shutil.move(iconpath,
623 os.path.join(get_icon_dir(repodir, density), iconfilename))
624 empty_densities.remove(density)
626 except Exception as e:
627 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
630 apk['icon'] = iconfilename
634 # First try resizing down to not lose quality
636 for density in screen_densities:
637 if density not in empty_densities:
638 last_density = density
640 if last_density is None:
642 logging.debug("Density %s not available, resizing down from %s"
643 % (density, last_density))
645 last_iconpath = os.path.join(
646 get_icon_dir(repodir, last_density), iconfilename)
647 iconpath = os.path.join(
648 get_icon_dir(repodir, density), iconfilename)
650 im = Image.open(last_iconpath)
652 logging.warn("Invalid image file at %s" % last_iconpath)
655 size = dpi_to_px(density)
657 im.thumbnail((size, size), Image.ANTIALIAS)
658 im.save(iconpath, "PNG")
659 empty_densities.remove(density)
661 # Then just copy from the highest resolution available
663 for density in reversed(screen_densities):
664 if density not in empty_densities:
665 last_density = density
667 if last_density is None:
669 logging.debug("Density %s not available, copying from lower density %s"
670 % (density, last_density))
673 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
674 os.path.join(get_icon_dir(repodir, density), iconfilename))
676 empty_densities.remove(density)
678 for density in screen_densities:
679 icon_dir = get_icon_dir(repodir, density)
680 icondest = os.path.join(icon_dir, iconfilename)
681 resize_icon(icondest, density)
683 # Copy from icons-mdpi to icons since mdpi is the baseline density
684 baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
685 if os.path.isfile(baseline):
686 apk['icons']['0'] = iconfilename
687 shutil.copyfile(baseline,
688 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
690 # Record in known apks, getting the added date at the same time..
691 added = knownapks.recordapk(apk['apkname'], apk['id'])
693 if use_date_from_apk and manifest.date_time[1] != 0:
694 added = datetime(*manifest.date_time).timetuple()
695 logging.debug("Using date from APK")
699 apkcache[apkfilename] = apk
704 return apks, cachechanged
707 repo_pubkey_fingerprint = None
710 # Generate a certificate fingerprint the same way keytool does it
711 # (but with slightly different formatting)
712 def cert_fingerprint(data):
713 digest = hashlib.sha256(data).digest()
715 ret.append(' '.join("%02X" % ord(b) for b in digest))
719 def extract_pubkey():
720 global repo_pubkey_fingerprint
721 if 'repo_pubkey' in config:
722 pubkey = unhexlify(config['repo_pubkey'])
724 p = FDroidPopen([config['keytool'], '-exportcert',
725 '-alias', config['repo_keyalias'],
726 '-keystore', config['keystore'],
727 '-storepass:file', config['keystorepassfile']]
728 + config['smartcardoptions'],
729 output=False, stderr_to_stdout=False)
730 if p.returncode != 0 or len(p.output) < 20:
731 msg = "Failed to get repo pubkey!"
732 if config['keystore'] == 'NONE':
733 msg += ' Is your crypto smartcard plugged in?'
734 logging.critical(msg)
737 repo_pubkey_fingerprint = cert_fingerprint(pubkey)
738 return hexlify(pubkey)
741 def make_index(apps, sortedids, apks, repodir, archive, categories):
742 """Make a repo index.
744 :param apps: fully populated apps list
745 :param apks: full populated apks list
746 :param repodir: the repo directory
747 :param archive: True if this is the archive repo, False if it's the
749 :param categories: list of categories
754 def addElement(name, value, doc, parent):
755 el = doc.createElement(name)
756 el.appendChild(doc.createTextNode(value))
757 parent.appendChild(el)
759 def addElementNonEmpty(name, value, doc, parent):
762 addElement(name, value, doc, parent)
764 def addElementCDATA(name, value, doc, parent):
765 el = doc.createElement(name)
766 el.appendChild(doc.createCDATASection(value))
767 parent.appendChild(el)
769 root = doc.createElement("fdroid")
770 doc.appendChild(root)
772 repoel = doc.createElement("repo")
774 mirrorcheckfailed = False
775 for mirror in config.get('mirrors', []):
776 base = os.path.basename(urlparse.urlparse(mirror).path.rstrip('/'))
777 if config.get('nonstandardwebroot') is not True and base != 'fdroid':
778 logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
779 mirrorcheckfailed = True
780 if mirrorcheckfailed:
784 repoel.setAttribute("name", config['archive_name'])
785 if config['repo_maxage'] != 0:
786 repoel.setAttribute("maxage", str(config['repo_maxage']))
787 repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
788 repoel.setAttribute("url", config['archive_url'])
789 addElement('description', config['archive_description'], doc, repoel)
790 urlbasepath = os.path.basename(urlparse.urlparse(config['archive_url']).path)
791 for mirror in config.get('mirrors', []):
792 addElement('mirror', urlparse.urljoin(mirror, urlbasepath), doc, repoel)
795 repoel.setAttribute("name", config['repo_name'])
796 if config['repo_maxage'] != 0:
797 repoel.setAttribute("maxage", str(config['repo_maxage']))
798 repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
799 repoel.setAttribute("url", config['repo_url'])
800 addElement('description', config['repo_description'], doc, repoel)
801 urlbasepath = os.path.basename(urlparse.urlparse(config['repo_url']).path)
802 for mirror in config.get('mirrors', []):
803 addElement('mirror', urlparse.urljoin(mirror, urlbasepath), doc, repoel)
805 repoel.setAttribute("version", "15")
806 repoel.setAttribute("timestamp", str(int(time.time())))
809 if not options.nosign:
810 if 'repo_keyalias' not in config:
812 logging.critical("'repo_keyalias' not found in config.py!")
813 if 'keystore' not in config:
815 logging.critical("'keystore' not found in config.py!")
816 if 'keystorepass' not in config and 'keystorepassfile' not in config:
818 logging.critical("'keystorepass' not found in config.py!")
819 if 'keypass' not in config and 'keypassfile' not in config:
821 logging.critical("'keypass' not found in config.py!")
822 if not os.path.exists(config['keystore']):
824 logging.critical("'" + config['keystore'] + "' does not exist!")
826 logging.warning("`fdroid update` requires a signing key, you can create one using:")
827 logging.warning("\tfdroid update --create-key")
830 repoel.setAttribute("pubkey", extract_pubkey())
831 root.appendChild(repoel)
833 for appid in sortedids:
836 if app.Disabled is not None:
839 # Get a list of the apks for this app...
842 if apk['id'] == appid:
845 if len(apklist) == 0:
848 apel = doc.createElement("application")
849 apel.setAttribute("id", app.id)
850 root.appendChild(apel)
852 addElement('id', app.id, doc, apel)
854 addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
856 addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
857 addElement('name', app.Name, doc, apel)
858 addElement('summary', app.Summary, doc, apel)
860 addElement('icon', app.icon, doc, apel)
864 return ("fdroid.app:" + appid, apps[appid].Name)
865 raise MetaDataException("Cannot resolve app id " + appid)
868 metadata.description_html(app.Description, linkres),
870 addElement('license', app.License, doc, apel)
872 addElement('categories', ','.join(app.Categories), doc, apel)
873 # We put the first (primary) category in LAST, which will have
874 # the desired effect of making clients that only understand one
875 # category see that one.
876 addElement('category', app.Categories[0], doc, apel)
877 addElement('web', app.WebSite, doc, apel)
878 addElement('source', app.SourceCode, doc, apel)
879 addElement('tracker', app.IssueTracker, doc, apel)
880 addElementNonEmpty('changelog', app.Changelog, doc, apel)
881 addElementNonEmpty('author', app.AuthorName, doc, apel)
882 addElementNonEmpty('email', app.AuthorEmail, doc, apel)
883 addElementNonEmpty('donate', app.Donate, doc, apel)
884 addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
885 addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
886 addElementNonEmpty('flattr', app.FlattrID, doc, apel)
888 # These elements actually refer to the current version (i.e. which
889 # one is recommended. They are historically mis-named, and need
890 # changing, but stay like this for now to support existing clients.
891 addElement('marketversion', app.CurrentVersion, doc, apel)
892 addElement('marketvercode', app.CurrentVersionCode, doc, apel)
895 af = app.AntiFeatures
897 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
899 pv = app.Provides.split(',')
900 addElementNonEmpty('provides', ','.join(pv), doc, apel)
902 addElement('requirements', 'root', doc, apel)
904 # Sort the apk list into version order, just so the web site
905 # doesn't have to do any work by default...
906 apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
908 # Check for duplicates - they will make the client unhappy...
909 for i in range(len(apklist) - 1):
910 if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
911 logging.critical("duplicate versions: '%s' - '%s'" % (
912 apklist[i]['apkname'], apklist[i + 1]['apkname']))
915 current_version_code = 0
916 current_version_file = None
918 # find the APK for the "Current Version"
919 if current_version_code < apk['versioncode']:
920 current_version_code = apk['versioncode']
921 if current_version_code < int(app.CurrentVersionCode):
922 current_version_file = apk['apkname']
924 apkel = doc.createElement("package")
925 apel.appendChild(apkel)
926 addElement('version', apk['version'], doc, apkel)
927 addElement('versioncode', str(apk['versioncode']), doc, apkel)
928 addElement('apkname', apk['apkname'], doc, apkel)
930 addElement('srcname', apk['srcname'], doc, apkel)
931 for hash_type in ['sha256']:
932 if hash_type not in apk:
934 hashel = doc.createElement("hash")
935 hashel.setAttribute("type", hash_type)
936 hashel.appendChild(doc.createTextNode(apk[hash_type]))
937 apkel.appendChild(hashel)
938 addElement('sig', apk['sig'], doc, apkel)
939 addElement('size', str(apk['size']), doc, apkel)
940 addElement('sdkver', str(apk['sdkversion']), doc, apkel)
941 if 'maxsdkversion' in apk:
942 addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
944 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
945 addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
946 if 'nativecode' in apk:
947 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
948 addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
950 if current_version_file is not None \
951 and config['make_current_version_link'] \
952 and repodir == 'repo': # only create these
953 namefield = config['current_version_name_source']
954 sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
955 apklinkname = sanitized_name + '.apk'
956 current_version_path = os.path.join(repodir, current_version_file)
957 if os.path.islink(apklinkname):
958 os.remove(apklinkname)
959 os.symlink(current_version_path, apklinkname)
960 # also symlink gpg signature, if it exists
961 for extension in ('.asc', '.sig'):
962 sigfile_path = current_version_path + extension
963 if os.path.exists(sigfile_path):
964 siglinkname = apklinkname + extension
965 if os.path.islink(siglinkname):
966 os.remove(siglinkname)
967 os.symlink(sigfile_path, siglinkname)
970 output = doc.toprettyxml()
974 with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
977 if 'repo_keyalias' in config:
980 logging.info("Creating unsigned index in preparation for signing")
982 logging.info("Creating signed index with this key (SHA256):")
983 logging.info("%s" % repo_pubkey_fingerprint)
985 # Create a jar of the index...
986 jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
987 p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
988 if p.returncode != 0:
989 logging.critical("Failed to create {0}".format(jar_output))
993 signed = os.path.join(repodir, 'index.jar')
995 # Remove old signed index if not signing
996 if os.path.exists(signed):
999 args = [config['jarsigner'], '-keystore', config['keystore'],
1000 '-storepass:file', config['keystorepassfile'],
1001 '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1002 signed, config['repo_keyalias']]
1003 if config['keystore'] == 'NONE':
1004 args += config['smartcardoptions']
1005 else: # smardcards never use -keypass
1006 args += ['-keypass:file', config['keypassfile']]
1007 p = FDroidPopen(args)
1008 if p.returncode != 0:
1009 logging.critical("Failed to sign index")
1012 # Copy the repo icon into the repo directory...
1013 icon_dir = os.path.join(repodir, 'icons')
1014 iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1015 shutil.copyfile(config['repo_icon'], iconfilename)
1017 # Write a category list in the repo to allow quick access...
1019 for cat in categories:
1020 catdata += cat + '\n'
1021 with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
1025 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1027 for appid, app in apps.iteritems():
1029 if app.ArchivePolicy:
1030 keepversions = int(app.ArchivePolicy[:-9])
1032 keepversions = defaultkeepversions
1034 def filter_apk_list_sorted(apk_list):
1036 for apk in apk_list:
1037 if apk['id'] == appid:
1040 # Sort the apk list by version code. First is highest/newest.
1041 return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1043 def move_file(from_dir, to_dir, filename, ignore_missing):
1044 from_path = os.path.join(from_dir, filename)
1045 if ignore_missing and not os.path.exists(from_path):
1047 to_path = os.path.join(to_dir, filename)
1048 shutil.move(from_path, to_path)
1050 if len(apks) > keepversions:
1051 apklist = filter_apk_list_sorted(apks)
1052 # Move back the ones we don't want.
1053 for apk in apklist[keepversions:]:
1054 logging.info("Moving " + apk['apkname'] + " to archive")
1055 move_file(repodir, archivedir, apk['apkname'], False)
1056 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1057 for density in all_screen_densities:
1058 repo_icon_dir = get_icon_dir(repodir, density)
1059 archive_icon_dir = get_icon_dir(archivedir, density)
1060 if density not in apk['icons']:
1062 move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1063 if 'srcname' in apk:
1064 move_file(repodir, archivedir, apk['srcname'], False)
1065 archapks.append(apk)
1067 elif len(apks) < keepversions and len(archapks) > 0:
1068 required = keepversions - len(apks)
1069 archapklist = filter_apk_list_sorted(archapks)
1070 # Move forward the ones we want again.
1071 for apk in archapklist[:required]:
1072 logging.info("Moving " + apk['apkname'] + " from archive")
1073 move_file(archivedir, repodir, apk['apkname'], False)
1074 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1075 for density in all_screen_densities:
1076 repo_icon_dir = get_icon_dir(repodir, density)
1077 archive_icon_dir = get_icon_dir(archivedir, density)
1078 if density not in apk['icons']:
1080 move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1081 if 'srcname' in apk:
1082 move_file(archivedir, repodir, apk['srcname'], False)
1083 archapks.remove(apk)
1087 def add_apks_to_per_app_repos(repodir, apks):
1088 apks_per_app = dict()
1090 apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1091 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1092 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1093 apks_per_app[apk['id']] = apk
1095 if not os.path.exists(apk['per_app_icons']):
1096 logging.info('Adding new repo for only ' + apk['id'])
1097 os.makedirs(apk['per_app_icons'])
1099 apkpath = os.path.join(repodir, apk['apkname'])
1100 shutil.copy(apkpath, apk['per_app_repo'])
1101 apksigpath = apkpath + '.sig'
1102 if os.path.exists(apksigpath):
1103 shutil.copy(apksigpath, apk['per_app_repo'])
1104 apkascpath = apkpath + '.asc'
1105 if os.path.exists(apkascpath):
1106 shutil.copy(apkascpath, apk['per_app_repo'])
1115 global config, options
1117 # Parse command line...
1118 parser = ArgumentParser()
1119 common.setup_global_opts(parser)
1120 parser.add_argument("--create-key", action="store_true", default=False,
1121 help="Create a repo signing key in a keystore")
1122 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1123 help="Create skeleton metadata files that are missing")
1124 parser.add_argument("--delete-unknown", action="store_true", default=False,
1125 help="Delete APKs without metadata from the repo")
1126 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1127 help="Report on build data status")
1128 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1129 help="Interactively ask about things that need updating.")
1130 parser.add_argument("-I", "--icons", action="store_true", default=False,
1131 help="Resize all the icons exceeding the max pixel size and exit")
1132 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1133 help="Specify editor to use in interactive mode. Default " +
1134 "is /etc/alternatives/editor")
1135 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1136 help="Update the wiki")
1137 parser.add_argument("--pretty", action="store_true", default=False,
1138 help="Produce human-readable index.xml")
1139 parser.add_argument("--clean", action="store_true", default=False,
1140 help="Clean update - don't uses caches, reprocess all apks")
1141 parser.add_argument("--nosign", action="store_true", default=False,
1142 help="When configured for signed indexes, create only unsigned indexes at this stage")
1143 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1144 help="Use date from apk instead of current time for newly added apks")
1145 options = parser.parse_args()
1147 config = common.read_config(options)
1149 if not ('jarsigner' in config and 'keytool' in config):
1150 logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1154 if config['archive_older'] != 0:
1155 repodirs.append('archive')
1156 if not os.path.exists('archive'):
1160 resize_all_icons(repodirs)
1163 # check that icons exist now, rather than fail at the end of `fdroid update`
1164 for k in ['repo_icon', 'archive_icon']:
1166 if not os.path.exists(config[k]):
1167 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1170 # if the user asks to create a keystore, do it now, reusing whatever it can
1171 if options.create_key:
1172 if os.path.exists(config['keystore']):
1173 logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1174 logging.critical("\t'" + config['keystore'] + "'")
1177 if 'repo_keyalias' not in config:
1178 config['repo_keyalias'] = socket.getfqdn()
1179 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1180 if 'keydname' not in config:
1181 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1182 common.write_to_config(config, 'keydname', config['keydname'])
1183 if 'keystore' not in config:
1184 config['keystore'] = common.default_config.keystore
1185 common.write_to_config(config, 'keystore', config['keystore'])
1187 password = common.genpassword()
1188 if 'keystorepass' not in config:
1189 config['keystorepass'] = password
1190 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1191 if 'keypass' not in config:
1192 config['keypass'] = password
1193 common.write_to_config(config, 'keypass', config['keypass'])
1194 common.genkeystore(config)
1197 apps = metadata.read_metadata()
1199 # Generate a list of categories...
1201 for app in apps.itervalues():
1202 categories.update(app.Categories)
1204 # Read known apks data (will be updated and written back when we've finished)
1205 knownapks = common.KnownApks()
1207 # Gather information about all the apk files in the repo directory, using
1208 # cached data if possible.
1209 apkcachefile = os.path.join('tmp', 'apkcache')
1210 if not options.clean and os.path.exists(apkcachefile):
1211 with open(apkcachefile, 'rb') as cf:
1212 apkcache = pickle.load(cf)
1216 delete_disabled_builds(apps, apkcache, repodirs)
1218 # Scan all apks in the main repo
1219 apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1221 # Generate warnings for apk's with no metadata (or create skeleton
1222 # metadata files, if requested on the command line)
1225 if apk['id'] not in apps:
1226 if options.create_metadata:
1227 if 'name' not in apk:
1228 logging.error(apk['id'] + ' does not have a name! Skipping...')
1230 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
1231 f.write("License:Unknown\n")
1232 f.write("Web Site:\n")
1233 f.write("Source Code:\n")
1234 f.write("Issue Tracker:\n")
1235 f.write("Changelog:\n")
1236 f.write("Summary:" + apk['name'] + "\n")
1237 f.write("Description:\n")
1238 f.write(apk['name'] + "\n")
1241 logging.info("Generated skeleton metadata for " + apk['id'])
1244 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1245 if options.delete_unknown:
1246 logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1247 rmf = os.path.join(repodirs[0], apk['apkname'])
1248 if not os.path.exists(rmf):
1249 logging.error("Could not find {0} to remove it".format(rmf))
1253 logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1255 # update the metadata with the newly created ones included
1257 apps = metadata.read_metadata()
1259 # Scan the archive repo for apks as well
1260 if len(repodirs) > 1:
1261 archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1267 # Some information from the apks needs to be applied up to the application
1268 # level. When doing this, we use the info from the most recent version's apk.
1269 # We deal with figuring out when the app was added and last updated at the
1271 for appid, app in apps.iteritems():
1273 for apk in apks + archapks:
1274 if apk['id'] == appid:
1275 if apk['versioncode'] > bestver:
1276 bestver = apk['versioncode']
1280 if not app.added or apk['added'] < app.added:
1281 app.added = apk['added']
1282 if not app.lastupdated or apk['added'] > app.lastupdated:
1283 app.lastupdated = apk['added']
1286 logging.debug("Don't know when " + appid + " was added")
1287 if not app.lastupdated:
1288 logging.debug("Don't know when " + appid + " was last updated")
1291 if app.Name is None:
1292 app.Name = app.AutoName or appid
1294 logging.debug("Application " + appid + " has no packages")
1296 if app.Name is None:
1297 app.Name = bestapk['name']
1298 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1299 if app.CurrentVersionCode is None:
1300 app.CurrentVersionCode = str(bestver)
1302 # Sort the app list by name, then the web site doesn't have to by default.
1303 # (we had to wait until we'd scanned the apks to do this, because mostly the
1304 # name comes from there!)
1305 sortedids = sorted(apps.iterkeys(), key=lambda appid: apps[appid].Name.upper())
1307 # APKs are placed into multiple repos based on the app package, providing
1308 # per-app subscription feeds for nightly builds and things like it
1309 if config['per_app_repos']:
1310 add_apks_to_per_app_repos(repodirs[0], apks)
1311 for appid, app in apps.iteritems():
1312 repodir = os.path.join(appid, 'fdroid', 'repo')
1314 appdict[appid] = app
1315 if os.path.isdir(repodir):
1316 make_index(appdict, [appid], apks, repodir, False, categories)
1318 logging.info('Skipping index generation for ' + appid)
1321 if len(repodirs) > 1:
1322 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1324 # Make the index for the main repo...
1325 make_index(apps, sortedids, apks, repodirs[0], False, categories)
1327 # If there's an archive repo, make the index for it. We already scanned it
1329 if len(repodirs) > 1:
1330 make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1332 if config['update_stats']:
1334 # Update known apks info...
1335 knownapks.writeifchanged()
1337 # Generate latest apps data for widget
1338 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1340 for line in file(os.path.join('stats', 'latestapps.txt')):
1341 appid = line.rstrip()
1342 data += appid + "\t"
1344 data += app.Name + "\t"
1345 if app.icon is not None:
1346 data += app.icon + "\t"
1347 data += app.License + "\n"
1348 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
1352 with open(apkcachefile, 'wb') as cf:
1353 pickle.dump(apkcache, cf)
1355 # Update the wiki...
1357 update_wiki(apps, sortedids, apks + archapks)
1359 logging.info("Finished.")
1361 if __name__ == "__main__":