3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
32 from datetime import datetime, timedelta
33 from xml.dom.minidom import Document
34 from argparse import ArgumentParser
38 from pyasn1.error import PyAsn1Error
39 from pyasn1.codec.der import decoder, encoder
40 from pyasn1_modules import rfc2315
41 from binascii import hexlify, unhexlify
47 from . import metadata
48 from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
49 from .metadata import MetaDataException
53 screen_densities = ['640', '480', '320', '240', '160', '120']
55 all_screen_densities = ['0'] + screen_densities
57 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
58 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
61 def dpi_to_px(density):
62 return (int(density) * 48) / 160
66 return (int(px) * 160) / 48
69 def get_icon_dir(repodir, density):
71 return os.path.join(repodir, "icons")
72 return os.path.join(repodir, "icons-%s" % density)
75 def get_icon_dirs(repodir):
76 for density in screen_densities:
77 yield get_icon_dir(repodir, density)
80 def get_all_icon_dirs(repodir):
81 for density in all_screen_densities:
82 yield get_icon_dir(repodir, density)
85 def update_wiki(apps, sortedids, apks):
88 :param apps: fully populated list of all applications
89 :param apks: all apks, except...
91 logging.info("Updating wiki")
93 wikiredircat = 'App Redirects'
95 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
96 path=config['wiki_path'])
97 site.login(config['wiki_user'], config['wiki_password'])
99 generated_redirects = {}
101 for appid in sortedids:
106 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
108 for af in app.AntiFeatures:
109 wikidata += '{{AntiFeature|' + af + '}}\n'
114 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
117 time.strftime('%Y-%m-%d', app.added) if app.added else '',
118 time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
133 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
135 wikidata += app.Summary
136 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
138 wikidata += "=Description=\n"
139 wikidata += metadata.description_wiki(app.Description) + "\n"
141 wikidata += "=Maintainer Notes=\n"
142 if app.MaintainerNotes:
143 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
144 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
146 # Get a list of all packages for this application...
148 gotcurrentver = False
152 if apk['id'] == appid:
153 if str(apk['versioncode']) == app.CurrentVersionCode:
156 # Include ones we can't build, as a special case...
157 for build in app.builds:
159 if build.vercode == app.CurrentVersionCode:
161 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
162 apklist.append({'versioncode': int(build.vercode),
163 'version': build.version,
164 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
169 if apk['versioncode'] == int(build.vercode):
174 apklist.append({'versioncode': int(build.vercode),
175 'version': build.version,
176 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
178 if app.CurrentVersionCode == '0':
180 # Sort with most recent first...
181 apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
183 wikidata += "=Versions=\n"
184 if len(apklist) == 0:
185 wikidata += "We currently have no versions of this app available."
186 elif not gotcurrentver:
187 wikidata += "We don't have the current version of this app."
189 wikidata += "We have the current version of this app."
190 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
191 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
192 if len(app.NoSourceSince) > 0:
193 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
194 if len(app.CurrentVersion) > 0:
195 wikidata += "The current (recommended) version is " + app.CurrentVersion
196 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
199 wikidata += "==" + apk['version'] + "==\n"
201 if 'buildproblem' in apk:
202 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
205 wikidata += "This version is built and signed by "
207 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
209 wikidata += "the original developer.\n\n"
210 wikidata += "Version code: " + str(apk['versioncode']) + '\n'
212 wikidata += '\n[[Category:' + wikicat + ']]\n'
213 if len(app.NoSourceSince) > 0:
214 wikidata += '\n[[Category:Apps missing source code]]\n'
215 if validapks == 0 and not app.Disabled:
216 wikidata += '\n[[Category:Apps with no packages]]\n'
217 if cantupdate and not app.Disabled:
218 wikidata += "\n[[Category:Apps we cannot update]]\n"
219 if buildfails and not app.Disabled:
220 wikidata += "\n[[Category:Apps with failing builds]]\n"
221 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
222 wikidata += '\n[[Category:Apps to Update]]\n'
224 wikidata += '\n[[Category:Apps that are disabled]]\n'
225 if app.UpdateCheckMode == 'None' and not app.Disabled:
226 wikidata += '\n[[Category:Apps with no update check]]\n'
227 for appcat in app.Categories:
228 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
230 # We can't have underscores in the page name, even if they're in
231 # the package ID, because MediaWiki messes with them...
232 pagename = appid.replace('_', ' ')
234 # Drop a trailing newline, because mediawiki is going to drop it anyway
235 # and it we don't we'll think the page has changed when it hasn't...
236 if wikidata.endswith('\n'):
237 wikidata = wikidata[:-1]
239 generated_pages[pagename] = wikidata
241 # Make a redirect from the name to the ID too, unless there's
242 # already an existing page with the name and it isn't a redirect.
244 apppagename = app.Name.replace('_', ' ')
245 apppagename = apppagename.replace('{', '')
246 apppagename = apppagename.replace('}', ' ')
247 apppagename = apppagename.replace(':', ' ')
248 apppagename = apppagename.replace('[', ' ')
249 apppagename = apppagename.replace(']', ' ')
250 # Drop double spaces caused mostly by replacing ':' above
251 apppagename = apppagename.replace(' ', ' ')
252 for expagename in site.allpages(prefix=apppagename,
253 filterredir='nonredirects',
255 if expagename == apppagename:
257 # Another reason not to make the redirect page is if the app name
258 # is the same as it's ID, because that will overwrite the real page
259 # with an redirect to itself! (Although it seems like an odd
260 # scenario this happens a lot, e.g. where there is metadata but no
261 # builds or binaries to extract a name from.
262 if apppagename == pagename:
265 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
267 for tcat, genp in [(wikicat, generated_pages),
268 (wikiredircat, generated_redirects)]:
269 catpages = site.Pages['Category:' + tcat]
271 for page in catpages:
272 existingpages.append(page.name)
273 if page.name in genp:
274 pagetxt = page.edit()
275 if pagetxt != genp[page.name]:
276 logging.debug("Updating modified page " + page.name)
277 page.save(genp[page.name], summary='Auto-updated')
279 logging.debug("Page " + page.name + " is unchanged")
281 logging.warn("Deleting page " + page.name)
282 page.delete('No longer published')
283 for pagename, text in genp.items():
284 logging.debug("Checking " + pagename)
285 if pagename not in existingpages:
286 logging.debug("Creating page " + pagename)
288 newpage = site.Pages[pagename]
289 newpage.save(text, summary='Auto-created')
291 logging.error("...FAILED to create page '{0}'".format(pagename))
293 # Purge server cache to ensure counts are up to date
294 site.pages['Repository Maintenance'].purge()
297 def delete_disabled_builds(apps, apkcache, repodirs):
298 """Delete disabled build outputs.
300 :param apps: list of all applications, as per metadata.read_metadata
301 :param apkcache: current apk cache information
302 :param repodirs: the repo directories to process
304 for appid, app in apps.items():
305 for build in app.builds:
306 if not build.disable:
308 apkfilename = appid + '_' + str(build.vercode) + '.apk'
309 iconfilename = "%s.%s.png" % (
312 for repodir in repodirs:
314 os.path.join(repodir, apkfilename),
315 os.path.join(repodir, apkfilename + '.asc'),
316 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
318 for density in all_screen_densities:
319 repo_dir = get_icon_dir(repodir, density)
320 files.append(os.path.join(repo_dir, iconfilename))
323 if os.path.exists(f):
324 logging.info("Deleting disabled build output " + f)
326 if apkfilename in apkcache:
327 del apkcache[apkfilename]
330 def resize_icon(iconpath, density):
332 if not os.path.isfile(iconpath):
337 fp = open(iconpath, 'rb')
339 size = dpi_to_px(density)
341 if any(length > size for length in im.size):
343 im.thumbnail((size, size), Image.ANTIALIAS)
344 logging.debug("%s was too large at %s - new size is %s" % (
345 iconpath, oldsize, im.size))
346 im.save(iconpath, "PNG")
348 except Exception as e:
349 logging.error("Failed resizing {0} - {1}".format(iconpath, e))
356 def resize_all_icons(repodirs):
357 """Resize all icons that exceed the max size
359 :param repodirs: the repo directories to process
361 for repodir in repodirs:
362 for density in screen_densities:
363 icon_dir = get_icon_dir(repodir, density)
364 icon_glob = os.path.join(icon_dir, '*.png')
365 for iconpath in glob.glob(icon_glob):
366 resize_icon(iconpath, density)
369 # A signature block file with a .DSA, .RSA, or .EC extension
370 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
374 """ Get the signing certificate of an apk. To get the same md5 has that
375 Android gets, we encode the .RSA certificate in a specific format and pass
376 it hex-encoded to the md5 digest algorithm.
378 :param apkpath: path to the apk
379 :returns: A string containing the md5 of the signature of the apk or None
380 if an error occurred.
385 # verify the jar signature is correct
386 args = [config['jarsigner'], '-verify', apkpath]
387 p = FDroidPopen(args)
388 if p.returncode != 0:
389 logging.critical(apkpath + " has a bad signature!")
392 with zipfile.ZipFile(apkpath, 'r') as apk:
394 certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
397 logging.error("Found no signing certificates on %s" % apkpath)
400 logging.error("Found multiple signing certificates on %s" % apkpath)
403 cert = apk.read(certs[0])
405 content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
406 if content.getComponentByName('contentType') != rfc2315.signedData:
407 logging.error("Unexpected format.")
410 content = decoder.decode(content.getComponentByName('content'),
411 asn1Spec=rfc2315.SignedData())[0]
413 certificates = content.getComponentByName('certificates')
415 logging.error("Certificates not found.")
418 cert_encoded = encoder.encode(certificates)[4:]
420 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
423 def get_icon_bytes(apkzip, iconsrc):
424 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
426 return apkzip.read(iconsrc)
428 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
431 def sha256sum(filename):
432 '''Calculate the sha256 of the given file'''
433 sha = hashlib.sha256()
434 with open(filename, 'rb') as f:
440 return sha.hexdigest()
443 def insert_obbs(repodir, apps, apks):
444 """Scans the .obb files in a given repo directory and adds them to the
445 relevant APK instances. OBB files have versionCodes like APK
446 files, and they are loosely associated. If there is an OBB file
447 present, then any APK with the same or higher versionCode will use
448 that OBB file. There are two OBB types: main and patch, each APK
449 can only have only have one of each.
451 https://developer.android.com/google/play/expansion-files.html
453 :param repodir: repo directory to scan
454 :param apps: list of current, valid apps
455 :param apks: current information on all APKs
459 def obbWarnDelete(f, msg):
460 logging.warning(msg + f)
461 if options.delete_unknown:
462 logging.error("Deleting unknown file: " + f)
466 java_Integer_MIN_VALUE = -pow(2, 31)
467 for f in glob.glob(os.path.join(repodir, '*.obb')):
468 obbfile = os.path.basename(f)
469 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
470 chunks = obbfile.split('.')
471 if chunks[0] != 'main' and chunks[0] != 'patch':
472 obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
474 if not re.match(r'^-?[0-9]+$', chunks[1]):
475 obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
477 versioncode = int(chunks[1])
478 packagename = ".".join(chunks[2:-1])
480 highestVersionCode = java_Integer_MIN_VALUE
481 if packagename not in apps.keys():
482 obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
485 if packagename == apk['id'] and apk['versioncode'] > highestVersionCode:
486 highestVersionCode = apk['versioncode']
487 if versioncode > highestVersionCode:
488 obbWarnDelete(f, 'OBB file has newer versioncode(' + str(versioncode)
489 + ') than any APK: ')
491 obbsha256 = sha256sum(f)
492 obbs.append((packagename, versioncode, obbfile, obbsha256))
495 for (packagename, versioncode, obbfile, obbsha256) in sorted(obbs, reverse=True):
496 if versioncode <= apk['versioncode'] and packagename == apk['id']:
497 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
498 apk['obbMainFile'] = obbfile
499 apk['obbMainFileSha256'] = obbsha256
500 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
501 apk['obbPatchFile'] = obbfile
502 apk['obbPatchFileSha256'] = obbsha256
503 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
507 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
508 """Scan the apks in the given repo directory.
510 This also extracts the icons.
512 :param apps: list of all applications, as per metadata.read_metadata
513 :param apkcache: current apk cache information
514 :param repodir: repo directory to scan
515 :param knownapks: known apks info
516 :param use_date_from_apk: use date from APK (instead of current date)
518 :returns: (apks, cachechanged) where apks is a list of apk information,
519 and cachechanged is True if the apkcache got changed.
524 for icon_dir in get_all_icon_dirs(repodir):
525 if os.path.exists(icon_dir):
527 shutil.rmtree(icon_dir)
528 os.makedirs(icon_dir)
530 os.makedirs(icon_dir)
533 name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
534 vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
535 vername_pat = re.compile(".*versionName='([^']*)'.*")
536 label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
537 icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
538 icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
539 sdkversion_pat = re.compile(".*'([0-9]*)'.*")
540 permission_pat = re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
541 feature_pat = re.compile(".*name='([^']*)'.*")
542 for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
544 apkfilename = apkfile[len(repodir) + 1:]
545 if ' ' in apkfilename:
546 logging.critical("Spaces in filenames are not allowed.")
549 shasum = sha256sum(apkfile)
552 if apkfilename in apkcache:
553 apk = apkcache[apkfilename]
554 if apk['sha256'] == shasum:
555 logging.debug("Reading " + apkfilename + " from cache")
558 logging.debug("Ignoring stale cache data for " + apkfilename)
561 logging.debug("Processing " + apkfilename)
563 apk['apkname'] = apkfilename
564 apk['sha256'] = shasum
565 srcfilename = apkfilename[:-4] + "_src.tar.gz"
566 if os.path.exists(os.path.join(repodir, srcfilename)):
567 apk['srcname'] = srcfilename
568 apk['size'] = os.path.getsize(apkfile)
569 apk['uses-permission'] = set()
570 apk['uses-permission-sdk-23'] = set()
571 apk['features'] = set()
572 apk['icons_src'] = {}
574 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
575 if p.returncode != 0:
576 if options.delete_unknown:
577 if os.path.exists(apkfile):
578 logging.error("Failed to get apk information, deleting " + apkfile)
581 logging.error("Could not find {0} to remove it".format(apkfile))
583 logging.error("Failed to get apk information, skipping " + apkfile)
585 for line in p.output.splitlines():
586 if line.startswith("package:"):
588 apk['id'] = re.match(name_pat, line).group(1)
589 apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
590 apk['version'] = re.match(vername_pat, line).group(1)
591 except Exception as e:
592 logging.error("Package matching failed: " + str(e))
593 logging.info("Line was: " + line)
595 elif line.startswith("application:"):
596 apk['name'] = re.match(label_pat, line).group(1)
597 # Keep path to non-dpi icon in case we need it
598 match = re.match(icon_pat_nodpi, line)
600 apk['icons_src']['-1'] = match.group(1)
601 elif line.startswith("launchable-activity:"):
602 # Only use launchable-activity as fallback to application
604 apk['name'] = re.match(label_pat, line).group(1)
605 if '-1' not in apk['icons_src']:
606 match = re.match(icon_pat_nodpi, line)
608 apk['icons_src']['-1'] = match.group(1)
609 elif line.startswith("application-icon-"):
610 match = re.match(icon_pat, line)
612 density = match.group(1)
613 path = match.group(2)
614 apk['icons_src'][density] = path
615 elif line.startswith("sdkVersion:"):
616 m = re.match(sdkversion_pat, line)
618 logging.error(line.replace('sdkVersion:', '')
619 + ' is not a valid minSdkVersion!')
621 apk['minSdkVersion'] = m.group(1)
622 # if target not set, default to min
623 if 'targetSdkVersion' not in apk:
624 apk['targetSdkVersion'] = m.group(1)
625 elif line.startswith("targetSdkVersion:"):
626 m = re.match(sdkversion_pat, line)
628 logging.error(line.replace('targetSdkVersion:', '')
629 + ' is not a valid targetSdkVersion!')
631 apk['targetSdkVersion'] = m.group(1)
632 elif line.startswith("maxSdkVersion:"):
633 apk['maxSdkVersion'] = re.match(sdkversion_pat, line).group(1)
634 elif line.startswith("native-code:"):
635 apk['nativecode'] = []
636 for arch in line[13:].split(' '):
637 apk['nativecode'].append(arch[1:-1])
638 elif line.startswith('uses-permission:'):
639 perm_match = re.match(permission_pat, line).groupdict()
641 permission = UsesPermission(
643 perm_match['maxSdkVersion']
646 apk['uses-permission'].add(permission)
647 elif line.startswith('uses-permission-sdk-23:'):
648 perm_match = re.match(permission_pat, line).groupdict()
650 permission_sdk_23 = UsesPermissionSdk23(
652 perm_match['maxSdkVersion']
655 apk['uses-permission-sdk-23'].add(permission_sdk_23)
657 elif line.startswith('uses-feature:'):
658 feature = re.match(feature_pat, line).group(1)
659 # Filter out this, it's only added with the latest SDK tools and
660 # causes problems for lots of apps.
661 if feature != "android.hardware.screen.portrait" \
662 and feature != "android.hardware.screen.landscape":
663 if feature.startswith("android.feature."):
664 feature = feature[16:]
665 apk['features'].add(feature)
667 if 'minSdkVersion' not in apk:
668 logging.warn("No SDK version information found in {0}".format(apkfile))
669 apk['minSdkVersion'] = 1
671 # Check for debuggable apks...
672 if common.isApkDebuggable(apkfile, config):
673 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
675 # Get the signature (or md5 of, to be precise)...
676 logging.debug('Getting signature of {0}'.format(apkfile))
677 apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
679 logging.critical("Failed to get apk signature")
682 apkzip = zipfile.ZipFile(apkfile, 'r')
684 # if an APK has files newer than the system time, suggest updating
685 # the system clock. This is useful for offline systems, used for
686 # signing, which do not have another source of clock sync info. It
687 # has to be more than 24 hours newer because ZIP/APK files do not
688 # store timezone info
689 manifest = apkzip.getinfo('AndroidManifest.xml')
690 if manifest.date_time[1] == 0: # month can't be zero
691 logging.debug('AndroidManifest.xml has no date')
693 dt_obj = datetime(*manifest.date_time)
694 checkdt = dt_obj - timedelta(1)
695 if datetime.today() < checkdt:
696 logging.warn('System clock is older than manifest in: '
698 + '\nSet clock to that time using:\n'
699 + 'sudo date -s "' + str(dt_obj) + '"')
701 iconfilename = "%s.%s.png" % (
705 # Extract the icon file...
707 for density in screen_densities:
708 if density not in apk['icons_src']:
709 empty_densities.append(density)
711 iconsrc = apk['icons_src'][density]
712 icon_dir = get_icon_dir(repodir, density)
713 icondest = os.path.join(icon_dir, iconfilename)
716 with open(icondest, 'wb') as f:
717 f.write(get_icon_bytes(apkzip, iconsrc))
718 apk['icons'][density] = iconfilename
721 logging.warn("Error retrieving icon file")
722 del apk['icons'][density]
723 del apk['icons_src'][density]
724 empty_densities.append(density)
726 if '-1' in apk['icons_src']:
727 iconsrc = apk['icons_src']['-1']
728 iconpath = os.path.join(
729 get_icon_dir(repodir, '0'), iconfilename)
730 with open(iconpath, 'wb') as f:
731 f.write(get_icon_bytes(apkzip, iconsrc))
733 im = Image.open(iconpath)
734 dpi = px_to_dpi(im.size[0])
735 for density in screen_densities:
736 if density in apk['icons']:
738 if density == screen_densities[-1] or dpi >= int(density):
739 apk['icons'][density] = iconfilename
740 shutil.move(iconpath,
741 os.path.join(get_icon_dir(repodir, density), iconfilename))
742 empty_densities.remove(density)
744 except Exception as e:
745 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
748 apk['icon'] = iconfilename
752 # First try resizing down to not lose quality
754 for density in screen_densities:
755 if density not in empty_densities:
756 last_density = density
758 if last_density is None:
760 logging.debug("Density %s not available, resizing down from %s"
761 % (density, last_density))
763 last_iconpath = os.path.join(
764 get_icon_dir(repodir, last_density), iconfilename)
765 iconpath = os.path.join(
766 get_icon_dir(repodir, density), iconfilename)
769 fp = open(last_iconpath, 'rb')
772 size = dpi_to_px(density)
774 im.thumbnail((size, size), Image.ANTIALIAS)
775 im.save(iconpath, "PNG")
776 empty_densities.remove(density)
778 logging.warning("Invalid image file at %s" % last_iconpath)
783 # Then just copy from the highest resolution available
785 for density in reversed(screen_densities):
786 if density not in empty_densities:
787 last_density = density
789 if last_density is None:
791 logging.debug("Density %s not available, copying from lower density %s"
792 % (density, last_density))
795 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
796 os.path.join(get_icon_dir(repodir, density), iconfilename))
798 empty_densities.remove(density)
800 for density in screen_densities:
801 icon_dir = get_icon_dir(repodir, density)
802 icondest = os.path.join(icon_dir, iconfilename)
803 resize_icon(icondest, density)
805 # Copy from icons-mdpi to icons since mdpi is the baseline density
806 baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
807 if os.path.isfile(baseline):
808 apk['icons']['0'] = iconfilename
809 shutil.copyfile(baseline,
810 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
812 if use_date_from_apk and manifest.date_time[1] != 0:
813 default_date_param = datetime(*manifest.date_time).utctimetuple()
815 default_date_param = None
817 # Record in known apks, getting the added date at the same time..
818 added = knownapks.recordapk(apk['apkname'], apk['id'], default_date=default_date_param)
822 apkcache[apkfilename] = apk
827 return apks, cachechanged
830 repo_pubkey_fingerprint = None
833 # Generate a certificate fingerprint the same way keytool does it
834 # (but with slightly different formatting)
835 def cert_fingerprint(data):
836 digest = hashlib.sha256(data).digest()
838 ret.append(' '.join("%02X" % b for b in bytearray(digest)))
842 def extract_pubkey():
843 global repo_pubkey_fingerprint
844 if 'repo_pubkey' in config:
845 pubkey = unhexlify(config['repo_pubkey'])
847 p = FDroidPopenBytes([config['keytool'], '-exportcert',
848 '-alias', config['repo_keyalias'],
849 '-keystore', config['keystore'],
850 '-storepass:file', config['keystorepassfile']]
851 + config['smartcardoptions'],
852 output=False, stderr_to_stdout=False)
853 if p.returncode != 0 or len(p.output) < 20:
854 msg = "Failed to get repo pubkey!"
855 if config['keystore'] == 'NONE':
856 msg += ' Is your crypto smartcard plugged in?'
857 logging.critical(msg)
860 repo_pubkey_fingerprint = cert_fingerprint(pubkey)
861 return hexlify(pubkey)
864 def make_index(apps, sortedids, apks, repodir, archive, categories):
865 """Make a repo index.
867 :param apps: fully populated apps list
868 :param apks: full populated apks list
869 :param repodir: the repo directory
870 :param archive: True if this is the archive repo, False if it's the
872 :param categories: list of categories
877 def addElement(name, value, doc, parent):
878 el = doc.createElement(name)
879 el.appendChild(doc.createTextNode(value))
880 parent.appendChild(el)
882 def addElementNonEmpty(name, value, doc, parent):
885 addElement(name, value, doc, parent)
887 def addElementCDATA(name, value, doc, parent):
888 el = doc.createElement(name)
889 el.appendChild(doc.createCDATASection(value))
890 parent.appendChild(el)
892 root = doc.createElement("fdroid")
893 doc.appendChild(root)
895 repoel = doc.createElement("repo")
897 mirrorcheckfailed = False
898 for mirror in config.get('mirrors', []):
899 base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
900 if config.get('nonstandardwebroot') is not True and base != 'fdroid':
901 logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
902 mirrorcheckfailed = True
903 if mirrorcheckfailed:
907 repoel.setAttribute("name", config['archive_name'])
908 if config['repo_maxage'] != 0:
909 repoel.setAttribute("maxage", str(config['repo_maxage']))
910 repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
911 repoel.setAttribute("url", config['archive_url'])
912 addElement('description', config['archive_description'], doc, repoel)
913 urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
914 for mirror in config.get('mirrors', []):
915 addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
918 repoel.setAttribute("name", config['repo_name'])
919 if config['repo_maxage'] != 0:
920 repoel.setAttribute("maxage", str(config['repo_maxage']))
921 repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
922 repoel.setAttribute("url", config['repo_url'])
923 addElement('description', config['repo_description'], doc, repoel)
924 urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
925 for mirror in config.get('mirrors', []):
926 addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
928 repoel.setAttribute("version", str(METADATA_VERSION))
929 repoel.setAttribute("timestamp", str(int(time.time())))
932 if not options.nosign:
933 if 'repo_keyalias' not in config:
935 logging.critical("'repo_keyalias' not found in config.py!")
936 if 'keystore' not in config:
938 logging.critical("'keystore' not found in config.py!")
939 if 'keystorepass' not in config and 'keystorepassfile' not in config:
941 logging.critical("'keystorepass' not found in config.py!")
942 if 'keypass' not in config and 'keypassfile' not in config:
944 logging.critical("'keypass' not found in config.py!")
945 if not os.path.exists(config['keystore']):
947 logging.critical("'" + config['keystore'] + "' does not exist!")
949 logging.warning("`fdroid update` requires a signing key, you can create one using:")
950 logging.warning("\tfdroid update --create-key")
953 repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
954 root.appendChild(repoel)
956 for command in ('install', 'delete'):
958 key = command + '_list'
960 if isinstance(config[key], str):
961 packageNames = [config[key]]
962 elif all(isinstance(item, str) for item in config[key]):
963 packageNames = config[key]
965 raise TypeError('only accepts strings, lists, and tuples')
966 for packageName in packageNames:
967 element = doc.createElement(command)
968 root.appendChild(element)
969 element.setAttribute('packageName', packageName)
971 for appid in sortedids:
974 if app.Disabled is not None:
977 # Get a list of the apks for this app...
980 if apk['id'] == appid:
983 if len(apklist) == 0:
986 apel = doc.createElement("application")
987 apel.setAttribute("id", app.id)
988 root.appendChild(apel)
990 addElement('id', app.id, doc, apel)
992 addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
994 addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
995 addElement('name', app.Name, doc, apel)
996 addElement('summary', app.Summary, doc, apel)
998 addElement('icon', app.icon, doc, apel)
1002 return ("fdroid.app:" + appid, apps[appid].Name)
1003 raise MetaDataException("Cannot resolve app id " + appid)
1006 metadata.description_html(app.Description, linkres),
1008 addElement('license', app.License, doc, apel)
1010 addElement('categories', ','.join(app.Categories), doc, apel)
1011 # We put the first (primary) category in LAST, which will have
1012 # the desired effect of making clients that only understand one
1013 # category see that one.
1014 addElement('category', app.Categories[0], doc, apel)
1015 addElement('web', app.WebSite, doc, apel)
1016 addElement('source', app.SourceCode, doc, apel)
1017 addElement('tracker', app.IssueTracker, doc, apel)
1018 addElementNonEmpty('changelog', app.Changelog, doc, apel)
1019 addElementNonEmpty('author', app.AuthorName, doc, apel)
1020 addElementNonEmpty('email', app.AuthorEmail, doc, apel)
1021 addElementNonEmpty('donate', app.Donate, doc, apel)
1022 addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
1023 addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
1024 addElementNonEmpty('flattr', app.FlattrID, doc, apel)
1026 # These elements actually refer to the current version (i.e. which
1027 # one is recommended. They are historically mis-named, and need
1028 # changing, but stay like this for now to support existing clients.
1029 addElement('marketversion', app.CurrentVersion, doc, apel)
1030 addElement('marketvercode', app.CurrentVersionCode, doc, apel)
1032 if app.AntiFeatures:
1033 af = app.AntiFeatures
1035 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
1037 pv = app.Provides.split(',')
1038 addElementNonEmpty('provides', ','.join(pv), doc, apel)
1039 if app.RequiresRoot:
1040 addElement('requirements', 'root', doc, apel)
1042 # Sort the apk list into version order, just so the web site
1043 # doesn't have to do any work by default...
1044 apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
1046 # Check for duplicates - they will make the client unhappy...
1047 for i in range(len(apklist) - 1):
1048 if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
1049 logging.critical("duplicate versions: '%s' - '%s'" % (
1050 apklist[i]['apkname'], apklist[i + 1]['apkname']))
1053 current_version_code = 0
1054 current_version_file = None
1056 # find the APK for the "Current Version"
1057 if current_version_code < apk['versioncode']:
1058 current_version_code = apk['versioncode']
1059 if current_version_code < int(app.CurrentVersionCode):
1060 current_version_file = apk['apkname']
1062 apkel = doc.createElement("package")
1063 apel.appendChild(apkel)
1064 addElement('version', apk['version'], doc, apkel)
1065 addElement('versioncode', str(apk['versioncode']), doc, apkel)
1066 addElement('apkname', apk['apkname'], doc, apkel)
1067 if 'srcname' in apk:
1068 addElement('srcname', apk['srcname'], doc, apkel)
1069 for hash_type in ['sha256']:
1070 if hash_type not in apk:
1072 hashel = doc.createElement("hash")
1073 hashel.setAttribute("type", hash_type)
1074 hashel.appendChild(doc.createTextNode(apk[hash_type]))
1075 apkel.appendChild(hashel)
1076 addElement('sig', apk['sig'], doc, apkel)
1077 addElement('size', str(apk['size']), doc, apkel)
1078 addElement('sdkver', str(apk['minSdkVersion']), doc, apkel)
1079 if 'targetSdkVersion' in apk:
1080 addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
1081 if 'maxSdkVersion' in apk:
1082 addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
1083 addElementNonEmpty('obbMainFile', apk.get('obbMainFile'), doc, apkel)
1084 addElementNonEmpty('obbMainFileSha256', apk.get('obbMainFileSha256'), doc, apkel)
1085 addElementNonEmpty('obbPatchFile', apk.get('obbPatchFile'), doc, apkel)
1086 addElementNonEmpty('obbPatchFileSha256', apk.get('obbPatchFileSha256'), doc, apkel)
1088 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
1090 # TODO: remove old permission format
1091 old_permissions = set()
1092 for perm in apk['uses-permission']:
1093 perm_name = perm.name
1094 if perm_name.startswith("android.permission."):
1095 perm_name = perm_name[19:]
1096 old_permissions.add(perm_name)
1097 addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel)
1099 for permission in apk['uses-permission']:
1100 permel = doc.createElement('uses-permission')
1101 permel.setAttribute('name', permission.name)
1102 if permission.maxSdkVersion is not None:
1103 permel.setAttribute('maxSdkVersion', permission.maxSdkVersion)
1104 apkel.appendChild(permel)
1105 for permission_sdk_23 in apk['uses-permission-sdk-23']:
1106 permel = doc.createElement('uses-permission-sdk-23')
1107 permel.setAttribute('name', permission_sdk_23.name)
1108 if permission_sdk_23.maxSdkVersion is not None:
1109 permel.setAttribute('maxSdkVersion', permission_sdk_23.maxSdkVersion)
1110 apkel.appendChild(permel)
1111 if 'nativecode' in apk:
1112 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
1113 addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
1115 if current_version_file is not None \
1116 and config['make_current_version_link'] \
1117 and repodir == 'repo': # only create these
1118 namefield = config['current_version_name_source']
1119 sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
1120 apklinkname = sanitized_name + '.apk'
1121 current_version_path = os.path.join(repodir, current_version_file)
1122 if os.path.islink(apklinkname):
1123 os.remove(apklinkname)
1124 os.symlink(current_version_path, apklinkname)
1125 # also symlink gpg signature, if it exists
1126 for extension in ('.asc', '.sig'):
1127 sigfile_path = current_version_path + extension
1128 if os.path.exists(sigfile_path):
1129 siglinkname = apklinkname + extension
1130 if os.path.islink(siglinkname):
1131 os.remove(siglinkname)
1132 os.symlink(sigfile_path, siglinkname)
1135 output = doc.toprettyxml(encoding='utf-8')
1137 output = doc.toxml(encoding='utf-8')
1139 with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
1142 if 'repo_keyalias' in config:
1145 logging.info("Creating unsigned index in preparation for signing")
1147 logging.info("Creating signed index with this key (SHA256):")
1148 logging.info("%s" % repo_pubkey_fingerprint)
1150 # Create a jar of the index...
1151 jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
1152 p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
1153 if p.returncode != 0:
1154 logging.critical("Failed to create {0}".format(jar_output))
1158 signed = os.path.join(repodir, 'index.jar')
1160 # Remove old signed index if not signing
1161 if os.path.exists(signed):
1164 args = [config['jarsigner'], '-keystore', config['keystore'],
1165 '-storepass:file', config['keystorepassfile'],
1166 '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1167 signed, config['repo_keyalias']]
1168 if config['keystore'] == 'NONE':
1169 args += config['smartcardoptions']
1170 else: # smardcards never use -keypass
1171 args += ['-keypass:file', config['keypassfile']]
1172 p = FDroidPopen(args)
1173 if p.returncode != 0:
1174 logging.critical("Failed to sign index")
1177 # Copy the repo icon into the repo directory...
1178 icon_dir = os.path.join(repodir, 'icons')
1179 iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1180 shutil.copyfile(config['repo_icon'], iconfilename)
1182 # Write a category list in the repo to allow quick access...
1184 for cat in categories:
1185 catdata += cat + '\n'
1186 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1190 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1192 for appid, app in apps.items():
1194 if app.ArchivePolicy:
1195 keepversions = int(app.ArchivePolicy[:-9])
1197 keepversions = defaultkeepversions
1199 def filter_apk_list_sorted(apk_list):
1201 for apk in apk_list:
1202 if apk['id'] == appid:
1205 # Sort the apk list by version code. First is highest/newest.
1206 return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1208 def move_file(from_dir, to_dir, filename, ignore_missing):
1209 from_path = os.path.join(from_dir, filename)
1210 if ignore_missing and not os.path.exists(from_path):
1212 to_path = os.path.join(to_dir, filename)
1213 shutil.move(from_path, to_path)
1215 logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1216 .format(appid, len(apks), keepversions, len(archapks)))
1218 if len(apks) > keepversions:
1219 apklist = filter_apk_list_sorted(apks)
1220 # Move back the ones we don't want.
1221 for apk in apklist[keepversions:]:
1222 logging.info("Moving " + apk['apkname'] + " to archive")
1223 move_file(repodir, archivedir, apk['apkname'], False)
1224 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1225 for density in all_screen_densities:
1226 repo_icon_dir = get_icon_dir(repodir, density)
1227 archive_icon_dir = get_icon_dir(archivedir, density)
1228 if density not in apk['icons']:
1230 move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1231 if 'srcname' in apk:
1232 move_file(repodir, archivedir, apk['srcname'], False)
1233 archapks.append(apk)
1235 elif len(apks) < keepversions and len(archapks) > 0:
1236 required = keepversions - len(apks)
1237 archapklist = filter_apk_list_sorted(archapks)
1238 # Move forward the ones we want again.
1239 for apk in archapklist[:required]:
1240 logging.info("Moving " + apk['apkname'] + " from archive")
1241 move_file(archivedir, repodir, apk['apkname'], False)
1242 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1243 for density in all_screen_densities:
1244 repo_icon_dir = get_icon_dir(repodir, density)
1245 archive_icon_dir = get_icon_dir(archivedir, density)
1246 if density not in apk['icons']:
1248 move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1249 if 'srcname' in apk:
1250 move_file(archivedir, repodir, apk['srcname'], False)
1251 archapks.remove(apk)
1255 def add_apks_to_per_app_repos(repodir, apks):
1256 apks_per_app = dict()
1258 apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1259 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1260 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1261 apks_per_app[apk['id']] = apk
1263 if not os.path.exists(apk['per_app_icons']):
1264 logging.info('Adding new repo for only ' + apk['id'])
1265 os.makedirs(apk['per_app_icons'])
1267 apkpath = os.path.join(repodir, apk['apkname'])
1268 shutil.copy(apkpath, apk['per_app_repo'])
1269 apksigpath = apkpath + '.sig'
1270 if os.path.exists(apksigpath):
1271 shutil.copy(apksigpath, apk['per_app_repo'])
1272 apkascpath = apkpath + '.asc'
1273 if os.path.exists(apkascpath):
1274 shutil.copy(apkascpath, apk['per_app_repo'])
1283 global config, options
1285 # Parse command line...
1286 parser = ArgumentParser()
1287 common.setup_global_opts(parser)
1288 parser.add_argument("--create-key", action="store_true", default=False,
1289 help="Create a repo signing key in a keystore")
1290 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1291 help="Create skeleton metadata files that are missing")
1292 parser.add_argument("--delete-unknown", action="store_true", default=False,
1293 help="Delete APKs and/or OBBs without metadata from the repo")
1294 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1295 help="Report on build data status")
1296 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1297 help="Interactively ask about things that need updating.")
1298 parser.add_argument("-I", "--icons", action="store_true", default=False,
1299 help="Resize all the icons exceeding the max pixel size and exit")
1300 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1301 help="Specify editor to use in interactive mode. Default " +
1302 "is /etc/alternatives/editor")
1303 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1304 help="Update the wiki")
1305 parser.add_argument("--pretty", action="store_true", default=False,
1306 help="Produce human-readable index.xml")
1307 parser.add_argument("--clean", action="store_true", default=False,
1308 help="Clean update - don't uses caches, reprocess all apks")
1309 parser.add_argument("--nosign", action="store_true", default=False,
1310 help="When configured for signed indexes, create only unsigned indexes at this stage")
1311 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1312 help="Use date from apk instead of current time for newly added apks")
1313 options = parser.parse_args()
1315 config = common.read_config(options)
1317 if not ('jarsigner' in config and 'keytool' in config):
1318 logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1322 if config['archive_older'] != 0:
1323 repodirs.append('archive')
1324 if not os.path.exists('archive'):
1328 resize_all_icons(repodirs)
1331 # check that icons exist now, rather than fail at the end of `fdroid update`
1332 for k in ['repo_icon', 'archive_icon']:
1334 if not os.path.exists(config[k]):
1335 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1338 # if the user asks to create a keystore, do it now, reusing whatever it can
1339 if options.create_key:
1340 if os.path.exists(config['keystore']):
1341 logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1342 logging.critical("\t'" + config['keystore'] + "'")
1345 if 'repo_keyalias' not in config:
1346 config['repo_keyalias'] = socket.getfqdn()
1347 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1348 if 'keydname' not in config:
1349 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1350 common.write_to_config(config, 'keydname', config['keydname'])
1351 if 'keystore' not in config:
1352 config['keystore'] = common.default_config.keystore
1353 common.write_to_config(config, 'keystore', config['keystore'])
1355 password = common.genpassword()
1356 if 'keystorepass' not in config:
1357 config['keystorepass'] = password
1358 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1359 if 'keypass' not in config:
1360 config['keypass'] = password
1361 common.write_to_config(config, 'keypass', config['keypass'])
1362 common.genkeystore(config)
1365 apps = metadata.read_metadata()
1367 # Generate a list of categories...
1369 for app in apps.values():
1370 categories.update(app.Categories)
1372 # Read known apks data (will be updated and written back when we've finished)
1373 knownapks = common.KnownApks()
1375 # Gather information about all the apk files in the repo directory, using
1376 # cached data if possible.
1377 apkcachefile = os.path.join('tmp', 'apkcache')
1378 if not options.clean and os.path.exists(apkcachefile):
1379 with open(apkcachefile, 'rb') as cf:
1380 apkcache = pickle.load(cf, encoding='utf-8')
1381 if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
1386 delete_disabled_builds(apps, apkcache, repodirs)
1388 # Scan all apks in the main repo
1389 apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1391 # Generate warnings for apk's with no metadata (or create skeleton
1392 # metadata files, if requested on the command line)
1395 if apk['id'] not in apps:
1396 if options.create_metadata:
1397 if 'name' not in apk:
1398 logging.error(apk['id'] + ' does not have a name! Skipping...')
1400 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
1401 f.write("License:Unknown\n")
1402 f.write("Web Site:\n")
1403 f.write("Source Code:\n")
1404 f.write("Issue Tracker:\n")
1405 f.write("Changelog:\n")
1406 f.write("Summary:" + apk['name'] + "\n")
1407 f.write("Description:\n")
1408 f.write(apk['name'] + "\n")
1411 logging.info("Generated skeleton metadata for " + apk['id'])
1414 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1415 if options.delete_unknown:
1416 logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1417 rmf = os.path.join(repodirs[0], apk['apkname'])
1418 if not os.path.exists(rmf):
1419 logging.error("Could not find {0} to remove it".format(rmf))
1423 logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1425 # update the metadata with the newly created ones included
1427 apps = metadata.read_metadata()
1429 insert_obbs(repodirs[0], apps, apks)
1431 # Scan the archive repo for apks as well
1432 if len(repodirs) > 1:
1433 archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1439 # Some information from the apks needs to be applied up to the application
1440 # level. When doing this, we use the info from the most recent version's apk.
1441 # We deal with figuring out when the app was added and last updated at the
1443 for appid, app in apps.items():
1445 for apk in apks + archapks:
1446 if apk['id'] == appid:
1447 if apk['versioncode'] > bestver:
1448 bestver = apk['versioncode']
1452 if not app.added or apk['added'] < app.added:
1453 app.added = apk['added']
1454 if not app.lastupdated or apk['added'] > app.lastupdated:
1455 app.lastupdated = apk['added']
1458 logging.debug("Don't know when " + appid + " was added")
1459 if not app.lastupdated:
1460 logging.debug("Don't know when " + appid + " was last updated")
1463 if app.Name is None:
1464 app.Name = app.AutoName or appid
1466 logging.debug("Application " + appid + " has no packages")
1468 if app.Name is None:
1469 app.Name = bestapk['name']
1470 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1471 if app.CurrentVersionCode is None:
1472 app.CurrentVersionCode = str(bestver)
1474 # Sort the app list by name, then the web site doesn't have to by default.
1475 # (we had to wait until we'd scanned the apks to do this, because mostly the
1476 # name comes from there!)
1477 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1479 # APKs are placed into multiple repos based on the app package, providing
1480 # per-app subscription feeds for nightly builds and things like it
1481 if config['per_app_repos']:
1482 add_apks_to_per_app_repos(repodirs[0], apks)
1483 for appid, app in apps.items():
1484 repodir = os.path.join(appid, 'fdroid', 'repo')
1486 appdict[appid] = app
1487 if os.path.isdir(repodir):
1488 make_index(appdict, [appid], apks, repodir, False, categories)
1490 logging.info('Skipping index generation for ' + appid)
1493 if len(repodirs) > 1:
1494 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1496 # Make the index for the main repo...
1497 make_index(apps, sortedids, apks, repodirs[0], False, categories)
1499 # If there's an archive repo, make the index for it. We already scanned it
1501 if len(repodirs) > 1:
1502 make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1504 if config['update_stats']:
1506 # Update known apks info...
1507 knownapks.writeifchanged()
1509 # Generate latest apps data for widget
1510 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1512 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1514 appid = line.rstrip()
1515 data += appid + "\t"
1517 data += app.Name + "\t"
1518 if app.icon is not None:
1519 data += app.icon + "\t"
1520 data += app.License + "\n"
1521 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1525 apkcache["METADATA_VERSION"] = METADATA_VERSION
1526 with open(apkcachefile, 'wb') as cf:
1527 pickle.dump(apkcache, cf)
1529 # Update the wiki...
1531 update_wiki(apps, sortedids, apks + archapks)
1533 logging.info("Finished.")
1535 if __name__ == "__main__":