3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2016, Blue Jay Wireless
5 # Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
6 # Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
7 # Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU Affero General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU Affero General Public License for more details.
19 # You should have received a copy of the GNU Affero General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
32 from datetime import datetime, timedelta
33 from xml.dom.minidom import Document
34 from argparse import ArgumentParser
38 from pyasn1.error import PyAsn1Error
39 from pyasn1.codec.der import decoder, encoder
40 from pyasn1_modules import rfc2315
41 from binascii import hexlify, unhexlify
47 from . import metadata
48 from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
49 from .metadata import MetaDataException
53 screen_densities = ['640', '480', '320', '240', '160', '120']
55 all_screen_densities = ['0'] + screen_densities
57 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
58 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
61 def dpi_to_px(density):
62 return (int(density) * 48) / 160
66 return (int(px) * 160) / 48
69 def get_icon_dir(repodir, density):
71 return os.path.join(repodir, "icons")
72 return os.path.join(repodir, "icons-%s" % density)
75 def get_icon_dirs(repodir):
76 for density in screen_densities:
77 yield get_icon_dir(repodir, density)
80 def get_all_icon_dirs(repodir):
81 for density in all_screen_densities:
82 yield get_icon_dir(repodir, density)
85 def update_wiki(apps, sortedids, apks):
88 :param apps: fully populated list of all applications
89 :param apks: all apks, except...
91 logging.info("Updating wiki")
93 wikiredircat = 'App Redirects'
95 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
96 path=config['wiki_path'])
97 site.login(config['wiki_user'], config['wiki_password'])
99 generated_redirects = {}
101 for appid in sortedids:
106 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
108 for af in app.AntiFeatures:
109 wikidata += '{{AntiFeature|' + af + '}}\n'
114 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
117 time.strftime('%Y-%m-%d', app.added) if app.added else '',
118 time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
133 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
135 wikidata += app.Summary
136 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
138 wikidata += "=Description=\n"
139 wikidata += metadata.description_wiki(app.Description) + "\n"
141 wikidata += "=Maintainer Notes=\n"
142 if app.MaintainerNotes:
143 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
144 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
146 # Get a list of all packages for this application...
148 gotcurrentver = False
152 if apk['id'] == appid:
153 if str(apk['versioncode']) == app.CurrentVersionCode:
156 # Include ones we can't build, as a special case...
157 for build in app.builds:
159 if build.vercode == app.CurrentVersionCode:
161 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
162 apklist.append({'versioncode': int(build.vercode),
163 'version': build.version,
164 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
169 if apk['versioncode'] == int(build.vercode):
174 apklist.append({'versioncode': int(build.vercode),
175 'version': build.version,
176 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
178 if app.CurrentVersionCode == '0':
180 # Sort with most recent first...
181 apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
183 wikidata += "=Versions=\n"
184 if len(apklist) == 0:
185 wikidata += "We currently have no versions of this app available."
186 elif not gotcurrentver:
187 wikidata += "We don't have the current version of this app."
189 wikidata += "We have the current version of this app."
190 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
191 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
192 if len(app.NoSourceSince) > 0:
193 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
194 if len(app.CurrentVersion) > 0:
195 wikidata += "The current (recommended) version is " + app.CurrentVersion
196 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
199 wikidata += "==" + apk['version'] + "==\n"
201 if 'buildproblem' in apk:
202 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
205 wikidata += "This version is built and signed by "
207 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
209 wikidata += "the original developer.\n\n"
210 wikidata += "Version code: " + str(apk['versioncode']) + '\n'
212 wikidata += '\n[[Category:' + wikicat + ']]\n'
213 if len(app.NoSourceSince) > 0:
214 wikidata += '\n[[Category:Apps missing source code]]\n'
215 if validapks == 0 and not app.Disabled:
216 wikidata += '\n[[Category:Apps with no packages]]\n'
217 if cantupdate and not app.Disabled:
218 wikidata += "\n[[Category:Apps we cannot update]]\n"
219 if buildfails and not app.Disabled:
220 wikidata += "\n[[Category:Apps with failing builds]]\n"
221 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
222 wikidata += '\n[[Category:Apps to Update]]\n'
224 wikidata += '\n[[Category:Apps that are disabled]]\n'
225 if app.UpdateCheckMode == 'None' and not app.Disabled:
226 wikidata += '\n[[Category:Apps with no update check]]\n'
227 for appcat in app.Categories:
228 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
230 # We can't have underscores in the page name, even if they're in
231 # the package ID, because MediaWiki messes with them...
232 pagename = appid.replace('_', ' ')
234 # Drop a trailing newline, because mediawiki is going to drop it anyway
235 # and it we don't we'll think the page has changed when it hasn't...
236 if wikidata.endswith('\n'):
237 wikidata = wikidata[:-1]
239 generated_pages[pagename] = wikidata
241 # Make a redirect from the name to the ID too, unless there's
242 # already an existing page with the name and it isn't a redirect.
244 apppagename = app.Name.replace('_', ' ')
245 apppagename = apppagename.replace('{', '')
246 apppagename = apppagename.replace('}', ' ')
247 apppagename = apppagename.replace(':', ' ')
248 apppagename = apppagename.replace('[', ' ')
249 apppagename = apppagename.replace(']', ' ')
250 # Drop double spaces caused mostly by replacing ':' above
251 apppagename = apppagename.replace(' ', ' ')
252 for expagename in site.allpages(prefix=apppagename,
253 filterredir='nonredirects',
255 if expagename == apppagename:
257 # Another reason not to make the redirect page is if the app name
258 # is the same as it's ID, because that will overwrite the real page
259 # with an redirect to itself! (Although it seems like an odd
260 # scenario this happens a lot, e.g. where there is metadata but no
261 # builds or binaries to extract a name from.
262 if apppagename == pagename:
265 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
267 for tcat, genp in [(wikicat, generated_pages),
268 (wikiredircat, generated_redirects)]:
269 catpages = site.Pages['Category:' + tcat]
271 for page in catpages:
272 existingpages.append(page.name)
273 if page.name in genp:
274 pagetxt = page.edit()
275 if pagetxt != genp[page.name]:
276 logging.debug("Updating modified page " + page.name)
277 page.save(genp[page.name], summary='Auto-updated')
279 logging.debug("Page " + page.name + " is unchanged")
281 logging.warn("Deleting page " + page.name)
282 page.delete('No longer published')
283 for pagename, text in genp.items():
284 logging.debug("Checking " + pagename)
285 if pagename not in existingpages:
286 logging.debug("Creating page " + pagename)
288 newpage = site.Pages[pagename]
289 newpage.save(text, summary='Auto-created')
291 logging.error("...FAILED to create page '{0}'".format(pagename))
293 # Purge server cache to ensure counts are up to date
294 site.pages['Repository Maintenance'].purge()
297 def delete_disabled_builds(apps, apkcache, repodirs):
298 """Delete disabled build outputs.
300 :param apps: list of all applications, as per metadata.read_metadata
301 :param apkcache: current apk cache information
302 :param repodirs: the repo directories to process
304 for appid, app in apps.items():
305 for build in app.builds:
306 if not build.disable:
308 apkfilename = appid + '_' + str(build.vercode) + '.apk'
309 iconfilename = "%s.%s.png" % (
312 for repodir in repodirs:
314 os.path.join(repodir, apkfilename),
315 os.path.join(repodir, apkfilename + '.asc'),
316 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
318 for density in all_screen_densities:
319 repo_dir = get_icon_dir(repodir, density)
320 files.append(os.path.join(repo_dir, iconfilename))
323 if os.path.exists(f):
324 logging.info("Deleting disabled build output " + f)
326 if apkfilename in apkcache:
327 del apkcache[apkfilename]
330 def resize_icon(iconpath, density):
332 if not os.path.isfile(iconpath):
337 fp = open(iconpath, 'rb')
339 size = dpi_to_px(density)
341 if any(length > size for length in im.size):
343 im.thumbnail((size, size), Image.ANTIALIAS)
344 logging.debug("%s was too large at %s - new size is %s" % (
345 iconpath, oldsize, im.size))
346 im.save(iconpath, "PNG")
348 except Exception as e:
349 logging.error("Failed resizing {0} - {1}".format(iconpath, e))
356 def resize_all_icons(repodirs):
357 """Resize all icons that exceed the max size
359 :param repodirs: the repo directories to process
361 for repodir in repodirs:
362 for density in screen_densities:
363 icon_dir = get_icon_dir(repodir, density)
364 icon_glob = os.path.join(icon_dir, '*.png')
365 for iconpath in glob.glob(icon_glob):
366 resize_icon(iconpath, density)
369 # A signature block file with a .DSA, .RSA, or .EC extension
370 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
374 """ Get the signing certificate of an apk. To get the same md5 has that
375 Android gets, we encode the .RSA certificate in a specific format and pass
376 it hex-encoded to the md5 digest algorithm.
378 :param apkpath: path to the apk
379 :returns: A string containing the md5 of the signature of the apk or None
380 if an error occurred.
385 # verify the jar signature is correct
386 args = [config['jarsigner'], '-verify', apkpath]
387 p = FDroidPopen(args)
388 if p.returncode != 0:
389 logging.critical(apkpath + " has a bad signature!")
392 with zipfile.ZipFile(apkpath, 'r') as apk:
394 certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
397 logging.error("Found no signing certificates on %s" % apkpath)
400 logging.error("Found multiple signing certificates on %s" % apkpath)
403 cert = apk.read(certs[0])
405 content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
406 if content.getComponentByName('contentType') != rfc2315.signedData:
407 logging.error("Unexpected format.")
410 content = decoder.decode(content.getComponentByName('content'),
411 asn1Spec=rfc2315.SignedData())[0]
413 certificates = content.getComponentByName('certificates')
415 logging.error("Certificates not found.")
418 cert_encoded = encoder.encode(certificates)[4:]
420 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
423 def get_icon_bytes(apkzip, iconsrc):
424 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
426 return apkzip.read(iconsrc)
428 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
431 def sha256sum(filename):
432 '''Calculate the sha256 of the given file'''
433 sha = hashlib.sha256()
434 with open(filename, 'rb') as f:
440 return sha.hexdigest()
443 def insert_obbs(repodir, apps, apks):
444 """Scans the .obb files in a given repo directory and adds them to the
445 relevant APK instances. OBB files have versionCodes like APK
446 files, and they are loosely associated. If there is an OBB file
447 present, then any APK with the same or higher versionCode will use
448 that OBB file. There are two OBB types: main and patch, each APK
449 can only have only have one of each.
451 https://developer.android.com/google/play/expansion-files.html
453 :param repodir: repo directory to scan
454 :param apps: list of current, valid apps
455 :param apks: current information on all APKs
459 def obbWarnDelete(f, msg):
460 logging.warning(msg + f)
461 if options.delete_unknown:
462 logging.error("Deleting unknown file: " + f)
466 java_Integer_MIN_VALUE = -pow(2, 31)
467 for f in glob.glob(os.path.join(repodir, '*.obb')):
468 obbfile = os.path.basename(f)
469 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
470 chunks = obbfile.split('.')
471 if chunks[0] != 'main' and chunks[0] != 'patch':
472 obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
474 if not re.match(r'^-?[0-9]+$', chunks[1]):
475 obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
477 versioncode = int(chunks[1])
478 packagename = ".".join(chunks[2:-1])
480 highestVersionCode = java_Integer_MIN_VALUE
481 if packagename not in apps.keys():
482 obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
485 if packagename == apk['id'] and apk['versioncode'] > highestVersionCode:
486 highestVersionCode = apk['versioncode']
487 if versioncode > highestVersionCode:
488 obbWarnDelete(f, 'OBB file has newer versioncode(' + str(versioncode)
489 + ') than any APK: ')
491 obbsha256 = sha256sum(f)
492 obbs.append((packagename, versioncode, obbfile, obbsha256))
495 for (packagename, versioncode, obbfile, obbsha256) in sorted(obbs, reverse=True):
496 if versioncode <= apk['versioncode'] and packagename == apk['id']:
497 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
498 apk['obbMainFile'] = obbfile
499 apk['obbMainFileSha256'] = obbsha256
500 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
501 apk['obbPatchFile'] = obbfile
502 apk['obbPatchFileSha256'] = obbsha256
503 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
507 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
508 """Scan the apks in the given repo directory.
510 This also extracts the icons.
512 :param apps: list of all applications, as per metadata.read_metadata
513 :param apkcache: current apk cache information
514 :param repodir: repo directory to scan
515 :param knownapks: known apks info
516 :param use_date_from_apk: use date from APK (instead of current date)
518 :returns: (apks, cachechanged) where apks is a list of apk information,
519 and cachechanged is True if the apkcache got changed.
524 for icon_dir in get_all_icon_dirs(repodir):
525 if os.path.exists(icon_dir):
527 shutil.rmtree(icon_dir)
528 os.makedirs(icon_dir)
530 os.makedirs(icon_dir)
533 name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
534 vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
535 vername_pat = re.compile(".*versionName='([^']*)'.*")
536 label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
537 icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
538 icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
539 sdkversion_pat = re.compile(".*'([0-9]*)'.*")
540 permission_pat = re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
541 feature_pat = re.compile(".*name='([^']*)'.*")
542 for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
544 apkfilename = apkfile[len(repodir) + 1:]
545 if ' ' in apkfilename:
546 logging.critical("Spaces in filenames are not allowed.")
549 shasum = sha256sum(apkfile)
552 if apkfilename in apkcache:
553 apk = apkcache[apkfilename]
554 if apk['sha256'] == shasum:
555 logging.debug("Reading " + apkfilename + " from cache")
558 logging.debug("Ignoring stale cache data for " + apkfilename)
561 logging.debug("Processing " + apkfilename)
563 apk['apkname'] = apkfilename
564 apk['sha256'] = shasum
565 srcfilename = apkfilename[:-4] + "_src.tar.gz"
566 if os.path.exists(os.path.join(repodir, srcfilename)):
567 apk['srcname'] = srcfilename
568 apk['size'] = os.path.getsize(apkfile)
569 apk['uses-permission'] = set()
570 apk['uses-permission-sdk-23'] = set()
571 apk['features'] = set()
572 apk['icons_src'] = {}
574 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
575 if p.returncode != 0:
576 if options.delete_unknown:
577 if os.path.exists(apkfile):
578 logging.error("Failed to get apk information, deleting " + apkfile)
581 logging.error("Could not find {0} to remove it".format(apkfile))
583 logging.error("Failed to get apk information, skipping " + apkfile)
585 for line in p.output.splitlines():
586 if line.startswith("package:"):
588 apk['id'] = re.match(name_pat, line).group(1)
589 apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
590 apk['version'] = re.match(vername_pat, line).group(1)
591 except Exception as e:
592 logging.error("Package matching failed: " + str(e))
593 logging.info("Line was: " + line)
595 elif line.startswith("application:"):
596 apk['name'] = re.match(label_pat, line).group(1)
597 # Keep path to non-dpi icon in case we need it
598 match = re.match(icon_pat_nodpi, line)
600 apk['icons_src']['-1'] = match.group(1)
601 elif line.startswith("launchable-activity:"):
602 # Only use launchable-activity as fallback to application
604 apk['name'] = re.match(label_pat, line).group(1)
605 if '-1' not in apk['icons_src']:
606 match = re.match(icon_pat_nodpi, line)
608 apk['icons_src']['-1'] = match.group(1)
609 elif line.startswith("application-icon-"):
610 match = re.match(icon_pat, line)
612 density = match.group(1)
613 path = match.group(2)
614 apk['icons_src'][density] = path
615 elif line.startswith("sdkVersion:"):
616 m = re.match(sdkversion_pat, line)
618 logging.error(line.replace('sdkVersion:', '')
619 + ' is not a valid minSdkVersion!')
621 apk['minSdkVersion'] = m.group(1)
622 # if target not set, default to min
623 if 'targetSdkVersion' not in apk:
624 apk['targetSdkVersion'] = m.group(1)
625 elif line.startswith("targetSdkVersion:"):
626 m = re.match(sdkversion_pat, line)
628 logging.error(line.replace('targetSdkVersion:', '')
629 + ' is not a valid targetSdkVersion!')
631 apk['targetSdkVersion'] = m.group(1)
632 elif line.startswith("maxSdkVersion:"):
633 apk['maxSdkVersion'] = re.match(sdkversion_pat, line).group(1)
634 elif line.startswith("native-code:"):
635 apk['nativecode'] = []
636 for arch in line[13:].split(' '):
637 apk['nativecode'].append(arch[1:-1])
638 elif line.startswith('uses-permission:'):
639 perm_match = re.match(permission_pat, line).groupdict()
641 permission = UsesPermission(
643 perm_match['maxSdkVersion']
646 apk['uses-permission'].add(permission)
647 elif line.startswith('uses-permission-sdk-23:'):
648 perm_match = re.match(permission_pat, line).groupdict()
650 permission_sdk_23 = UsesPermissionSdk23(
652 perm_match['maxSdkVersion']
655 apk['uses-permission-sdk-23'].add(permission_sdk_23)
657 elif line.startswith('uses-feature:'):
658 feature = re.match(feature_pat, line).group(1)
659 # Filter out this, it's only added with the latest SDK tools and
660 # causes problems for lots of apps.
661 if feature != "android.hardware.screen.portrait" \
662 and feature != "android.hardware.screen.landscape":
663 if feature.startswith("android.feature."):
664 feature = feature[16:]
665 apk['features'].add(feature)
667 if 'minSdkVersion' not in apk:
668 logging.warn("No SDK version information found in {0}".format(apkfile))
669 apk['minSdkVersion'] = 1
671 # Check for debuggable apks...
672 if common.isApkDebuggable(apkfile, config):
673 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
675 # Get the signature (or md5 of, to be precise)...
676 logging.debug('Getting signature of {0}'.format(apkfile))
677 apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
679 logging.critical("Failed to get apk signature")
682 apkzip = zipfile.ZipFile(apkfile, 'r')
684 # if an APK has files newer than the system time, suggest updating
685 # the system clock. This is useful for offline systems, used for
686 # signing, which do not have another source of clock sync info. It
687 # has to be more than 24 hours newer because ZIP/APK files do not
688 # store timezone info
689 manifest = apkzip.getinfo('AndroidManifest.xml')
690 if manifest.date_time[1] == 0: # month can't be zero
691 logging.debug('AndroidManifest.xml has no date')
693 dt_obj = datetime(*manifest.date_time)
694 checkdt = dt_obj - timedelta(1)
695 if datetime.today() < checkdt:
696 logging.warn('System clock is older than manifest in: '
698 + '\nSet clock to that time using:\n'
699 + 'sudo date -s "' + str(dt_obj) + '"')
701 iconfilename = "%s.%s.png" % (
705 # Extract the icon file...
707 for density in screen_densities:
708 if density not in apk['icons_src']:
709 empty_densities.append(density)
711 iconsrc = apk['icons_src'][density]
712 icon_dir = get_icon_dir(repodir, density)
713 icondest = os.path.join(icon_dir, iconfilename)
716 with open(icondest, 'wb') as f:
717 f.write(get_icon_bytes(apkzip, iconsrc))
718 apk['icons'][density] = iconfilename
721 logging.warn("Error retrieving icon file")
722 del apk['icons'][density]
723 del apk['icons_src'][density]
724 empty_densities.append(density)
726 if '-1' in apk['icons_src']:
727 iconsrc = apk['icons_src']['-1']
728 iconpath = os.path.join(
729 get_icon_dir(repodir, '0'), iconfilename)
730 with open(iconpath, 'wb') as f:
731 f.write(get_icon_bytes(apkzip, iconsrc))
733 im = Image.open(iconpath)
734 dpi = px_to_dpi(im.size[0])
735 for density in screen_densities:
736 if density in apk['icons']:
738 if density == screen_densities[-1] or dpi >= int(density):
739 apk['icons'][density] = iconfilename
740 shutil.move(iconpath,
741 os.path.join(get_icon_dir(repodir, density), iconfilename))
742 empty_densities.remove(density)
744 except Exception as e:
745 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
748 apk['icon'] = iconfilename
752 # First try resizing down to not lose quality
754 for density in screen_densities:
755 if density not in empty_densities:
756 last_density = density
758 if last_density is None:
760 logging.debug("Density %s not available, resizing down from %s"
761 % (density, last_density))
763 last_iconpath = os.path.join(
764 get_icon_dir(repodir, last_density), iconfilename)
765 iconpath = os.path.join(
766 get_icon_dir(repodir, density), iconfilename)
769 fp = open(last_iconpath, 'rb')
772 size = dpi_to_px(density)
774 im.thumbnail((size, size), Image.ANTIALIAS)
775 im.save(iconpath, "PNG")
776 empty_densities.remove(density)
778 logging.warning("Invalid image file at %s" % last_iconpath)
783 # Then just copy from the highest resolution available
785 for density in reversed(screen_densities):
786 if density not in empty_densities:
787 last_density = density
789 if last_density is None:
791 logging.debug("Density %s not available, copying from lower density %s"
792 % (density, last_density))
795 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
796 os.path.join(get_icon_dir(repodir, density), iconfilename))
798 empty_densities.remove(density)
800 for density in screen_densities:
801 icon_dir = get_icon_dir(repodir, density)
802 icondest = os.path.join(icon_dir, iconfilename)
803 resize_icon(icondest, density)
805 # Copy from icons-mdpi to icons since mdpi is the baseline density
806 baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
807 if os.path.isfile(baseline):
808 apk['icons']['0'] = iconfilename
809 shutil.copyfile(baseline,
810 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
812 if use_date_from_apk and manifest.date_time[1] != 0:
813 default_date_param = datetime(*manifest.date_time).utctimetuple()
815 default_date_param = None
817 # Record in known apks, getting the added date at the same time..
818 added = knownapks.recordapk(apk['apkname'], apk['id'], default_date=default_date_param)
822 apkcache[apkfilename] = apk
827 return apks, cachechanged
830 repo_pubkey_fingerprint = None
833 # Generate a certificate fingerprint the same way keytool does it
834 # (but with slightly different formatting)
835 def cert_fingerprint(data):
836 digest = hashlib.sha256(data).digest()
838 ret.append(' '.join("%02X" % b for b in bytearray(digest)))
842 def extract_pubkey():
843 global repo_pubkey_fingerprint
844 if 'repo_pubkey' in config:
845 pubkey = unhexlify(config['repo_pubkey'])
847 p = FDroidPopenBytes([config['keytool'], '-exportcert',
848 '-alias', config['repo_keyalias'],
849 '-keystore', config['keystore'],
850 '-storepass:file', config['keystorepassfile']]
851 + config['smartcardoptions'],
852 output=False, stderr_to_stdout=False)
853 if p.returncode != 0 or len(p.output) < 20:
854 msg = "Failed to get repo pubkey!"
855 if config['keystore'] == 'NONE':
856 msg += ' Is your crypto smartcard plugged in?'
857 logging.critical(msg)
860 repo_pubkey_fingerprint = cert_fingerprint(pubkey)
861 return hexlify(pubkey)
864 def make_index(apps, sortedids, apks, repodir, archive, categories):
865 """Make a repo index.
867 :param apps: fully populated apps list
868 :param apks: full populated apks list
869 :param repodir: the repo directory
870 :param archive: True if this is the archive repo, False if it's the
872 :param categories: list of categories
877 def addElement(name, value, doc, parent):
878 el = doc.createElement(name)
879 el.appendChild(doc.createTextNode(value))
880 parent.appendChild(el)
882 def addElementNonEmpty(name, value, doc, parent):
885 addElement(name, value, doc, parent)
887 def addElementCDATA(name, value, doc, parent):
888 el = doc.createElement(name)
889 el.appendChild(doc.createCDATASection(value))
890 parent.appendChild(el)
892 root = doc.createElement("fdroid")
893 doc.appendChild(root)
895 repoel = doc.createElement("repo")
897 mirrorcheckfailed = False
899 for mirror in config.get('mirrors', []):
900 base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
901 if config.get('nonstandardwebroot') is not True and base != 'fdroid':
902 logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
903 mirrorcheckfailed = True
904 # must end with / or urljoin strips a whole path segment
905 if mirror.endswith('/'):
906 mirrors.append(mirror)
908 mirrors.append(mirror + '/')
909 if mirrorcheckfailed:
913 repoel.setAttribute("name", config['archive_name'])
914 if config['repo_maxage'] != 0:
915 repoel.setAttribute("maxage", str(config['repo_maxage']))
916 repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
917 repoel.setAttribute("url", config['archive_url'])
918 addElement('description', config['archive_description'], doc, repoel)
919 urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
920 for mirror in mirrors:
921 addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
924 repoel.setAttribute("name", config['repo_name'])
925 if config['repo_maxage'] != 0:
926 repoel.setAttribute("maxage", str(config['repo_maxage']))
927 repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
928 repoel.setAttribute("url", config['repo_url'])
929 addElement('description', config['repo_description'], doc, repoel)
930 urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
931 for mirror in mirrors:
932 addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
934 repoel.setAttribute("version", str(METADATA_VERSION))
935 repoel.setAttribute("timestamp", str(int(time.time())))
938 if not options.nosign:
939 if 'repo_keyalias' not in config:
941 logging.critical("'repo_keyalias' not found in config.py!")
942 if 'keystore' not in config:
944 logging.critical("'keystore' not found in config.py!")
945 if 'keystorepass' not in config and 'keystorepassfile' not in config:
947 logging.critical("'keystorepass' not found in config.py!")
948 if 'keypass' not in config and 'keypassfile' not in config:
950 logging.critical("'keypass' not found in config.py!")
951 if not os.path.exists(config['keystore']):
953 logging.critical("'" + config['keystore'] + "' does not exist!")
955 logging.warning("`fdroid update` requires a signing key, you can create one using:")
956 logging.warning("\tfdroid update --create-key")
959 repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
960 root.appendChild(repoel)
962 for command in ('install', 'uninstall'):
964 key = command + '_list'
966 if isinstance(config[key], str):
967 packageNames = [config[key]]
968 elif all(isinstance(item, str) for item in config[key]):
969 packageNames = config[key]
971 raise TypeError('only accepts strings, lists, and tuples')
972 for packageName in packageNames:
973 element = doc.createElement(command)
974 root.appendChild(element)
975 element.setAttribute('packageName', packageName)
977 for appid in sortedids:
980 if app.Disabled is not None:
983 # Get a list of the apks for this app...
986 if apk['id'] == appid:
989 if len(apklist) == 0:
992 apel = doc.createElement("application")
993 apel.setAttribute("id", app.id)
994 root.appendChild(apel)
996 addElement('id', app.id, doc, apel)
998 addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
1000 addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
1001 addElement('name', app.Name, doc, apel)
1002 addElement('summary', app.Summary, doc, apel)
1004 addElement('icon', app.icon, doc, apel)
1008 return ("fdroid.app:" + appid, apps[appid].Name)
1009 raise MetaDataException("Cannot resolve app id " + appid)
1012 metadata.description_html(app.Description, linkres),
1014 addElement('license', app.License, doc, apel)
1016 addElement('categories', ','.join(app.Categories), doc, apel)
1017 # We put the first (primary) category in LAST, which will have
1018 # the desired effect of making clients that only understand one
1019 # category see that one.
1020 addElement('category', app.Categories[0], doc, apel)
1021 addElement('web', app.WebSite, doc, apel)
1022 addElement('source', app.SourceCode, doc, apel)
1023 addElement('tracker', app.IssueTracker, doc, apel)
1024 addElementNonEmpty('changelog', app.Changelog, doc, apel)
1025 addElementNonEmpty('author', app.AuthorName, doc, apel)
1026 addElementNonEmpty('email', app.AuthorEmail, doc, apel)
1027 addElementNonEmpty('donate', app.Donate, doc, apel)
1028 addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
1029 addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
1030 addElementNonEmpty('flattr', app.FlattrID, doc, apel)
1032 # These elements actually refer to the current version (i.e. which
1033 # one is recommended. They are historically mis-named, and need
1034 # changing, but stay like this for now to support existing clients.
1035 addElement('marketversion', app.CurrentVersion, doc, apel)
1036 addElement('marketvercode', app.CurrentVersionCode, doc, apel)
1038 if app.AntiFeatures:
1039 af = app.AntiFeatures
1041 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
1043 pv = app.Provides.split(',')
1044 addElementNonEmpty('provides', ','.join(pv), doc, apel)
1045 if app.RequiresRoot:
1046 addElement('requirements', 'root', doc, apel)
1048 # Sort the apk list into version order, just so the web site
1049 # doesn't have to do any work by default...
1050 apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
1052 # Check for duplicates - they will make the client unhappy...
1053 for i in range(len(apklist) - 1):
1054 if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
1055 logging.critical("duplicate versions: '%s' - '%s'" % (
1056 apklist[i]['apkname'], apklist[i + 1]['apkname']))
1059 current_version_code = 0
1060 current_version_file = None
1062 # find the APK for the "Current Version"
1063 if current_version_code < apk['versioncode']:
1064 current_version_code = apk['versioncode']
1065 if current_version_code < int(app.CurrentVersionCode):
1066 current_version_file = apk['apkname']
1068 apkel = doc.createElement("package")
1069 apel.appendChild(apkel)
1070 addElement('version', apk['version'], doc, apkel)
1071 addElement('versioncode', str(apk['versioncode']), doc, apkel)
1072 addElement('apkname', apk['apkname'], doc, apkel)
1073 if 'srcname' in apk:
1074 addElement('srcname', apk['srcname'], doc, apkel)
1075 for hash_type in ['sha256']:
1076 if hash_type not in apk:
1078 hashel = doc.createElement("hash")
1079 hashel.setAttribute("type", hash_type)
1080 hashel.appendChild(doc.createTextNode(apk[hash_type]))
1081 apkel.appendChild(hashel)
1082 addElement('sig', apk['sig'], doc, apkel)
1083 addElement('size', str(apk['size']), doc, apkel)
1084 addElement('sdkver', str(apk['minSdkVersion']), doc, apkel)
1085 if 'targetSdkVersion' in apk:
1086 addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
1087 if 'maxSdkVersion' in apk:
1088 addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
1089 addElementNonEmpty('obbMainFile', apk.get('obbMainFile'), doc, apkel)
1090 addElementNonEmpty('obbMainFileSha256', apk.get('obbMainFileSha256'), doc, apkel)
1091 addElementNonEmpty('obbPatchFile', apk.get('obbPatchFile'), doc, apkel)
1092 addElementNonEmpty('obbPatchFileSha256', apk.get('obbPatchFileSha256'), doc, apkel)
1094 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
1096 # TODO: remove old permission format
1097 old_permissions = set()
1098 for perm in apk['uses-permission']:
1099 perm_name = perm.name
1100 if perm_name.startswith("android.permission."):
1101 perm_name = perm_name[19:]
1102 old_permissions.add(perm_name)
1103 addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel)
1105 for permission in apk['uses-permission']:
1106 permel = doc.createElement('uses-permission')
1107 permel.setAttribute('name', permission.name)
1108 if permission.maxSdkVersion is not None:
1109 permel.setAttribute('maxSdkVersion', permission.maxSdkVersion)
1110 apkel.appendChild(permel)
1111 for permission_sdk_23 in apk['uses-permission-sdk-23']:
1112 permel = doc.createElement('uses-permission-sdk-23')
1113 permel.setAttribute('name', permission_sdk_23.name)
1114 if permission_sdk_23.maxSdkVersion is not None:
1115 permel.setAttribute('maxSdkVersion', permission_sdk_23.maxSdkVersion)
1116 apkel.appendChild(permel)
1117 if 'nativecode' in apk:
1118 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
1119 addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
1121 if current_version_file is not None \
1122 and config['make_current_version_link'] \
1123 and repodir == 'repo': # only create these
1124 namefield = config['current_version_name_source']
1125 sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
1126 apklinkname = sanitized_name + '.apk'
1127 current_version_path = os.path.join(repodir, current_version_file)
1128 if os.path.islink(apklinkname):
1129 os.remove(apklinkname)
1130 os.symlink(current_version_path, apklinkname)
1131 # also symlink gpg signature, if it exists
1132 for extension in ('.asc', '.sig'):
1133 sigfile_path = current_version_path + extension
1134 if os.path.exists(sigfile_path):
1135 siglinkname = apklinkname + extension
1136 if os.path.islink(siglinkname):
1137 os.remove(siglinkname)
1138 os.symlink(sigfile_path, siglinkname)
1141 output = doc.toprettyxml(encoding='utf-8')
1143 output = doc.toxml(encoding='utf-8')
1145 with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
1148 if 'repo_keyalias' in config:
1151 logging.info("Creating unsigned index in preparation for signing")
1153 logging.info("Creating signed index with this key (SHA256):")
1154 logging.info("%s" % repo_pubkey_fingerprint)
1156 # Create a jar of the index...
1157 jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
1158 p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
1159 if p.returncode != 0:
1160 logging.critical("Failed to create {0}".format(jar_output))
1164 signed = os.path.join(repodir, 'index.jar')
1166 # Remove old signed index if not signing
1167 if os.path.exists(signed):
1170 args = [config['jarsigner'], '-keystore', config['keystore'],
1171 '-storepass:file', config['keystorepassfile'],
1172 '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1173 signed, config['repo_keyalias']]
1174 if config['keystore'] == 'NONE':
1175 args += config['smartcardoptions']
1176 else: # smardcards never use -keypass
1177 args += ['-keypass:file', config['keypassfile']]
1178 p = FDroidPopen(args)
1179 if p.returncode != 0:
1180 logging.critical("Failed to sign index")
1183 # Copy the repo icon into the repo directory...
1184 icon_dir = os.path.join(repodir, 'icons')
1185 iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1186 shutil.copyfile(config['repo_icon'], iconfilename)
1188 # Write a category list in the repo to allow quick access...
1190 for cat in categories:
1191 catdata += cat + '\n'
1192 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1196 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1198 for appid, app in apps.items():
1200 if app.ArchivePolicy:
1201 keepversions = int(app.ArchivePolicy[:-9])
1203 keepversions = defaultkeepversions
1205 def filter_apk_list_sorted(apk_list):
1207 for apk in apk_list:
1208 if apk['id'] == appid:
1211 # Sort the apk list by version code. First is highest/newest.
1212 return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1214 def move_file(from_dir, to_dir, filename, ignore_missing):
1215 from_path = os.path.join(from_dir, filename)
1216 if ignore_missing and not os.path.exists(from_path):
1218 to_path = os.path.join(to_dir, filename)
1219 shutil.move(from_path, to_path)
1221 logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1222 .format(appid, len(apks), keepversions, len(archapks)))
1224 if len(apks) > keepversions:
1225 apklist = filter_apk_list_sorted(apks)
1226 # Move back the ones we don't want.
1227 for apk in apklist[keepversions:]:
1228 logging.info("Moving " + apk['apkname'] + " to archive")
1229 move_file(repodir, archivedir, apk['apkname'], False)
1230 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1231 for density in all_screen_densities:
1232 repo_icon_dir = get_icon_dir(repodir, density)
1233 archive_icon_dir = get_icon_dir(archivedir, density)
1234 if density not in apk['icons']:
1236 move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1237 if 'srcname' in apk:
1238 move_file(repodir, archivedir, apk['srcname'], False)
1239 archapks.append(apk)
1241 elif len(apks) < keepversions and len(archapks) > 0:
1242 required = keepversions - len(apks)
1243 archapklist = filter_apk_list_sorted(archapks)
1244 # Move forward the ones we want again.
1245 for apk in archapklist[:required]:
1246 logging.info("Moving " + apk['apkname'] + " from archive")
1247 move_file(archivedir, repodir, apk['apkname'], False)
1248 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1249 for density in all_screen_densities:
1250 repo_icon_dir = get_icon_dir(repodir, density)
1251 archive_icon_dir = get_icon_dir(archivedir, density)
1252 if density not in apk['icons']:
1254 move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1255 if 'srcname' in apk:
1256 move_file(archivedir, repodir, apk['srcname'], False)
1257 archapks.remove(apk)
1261 def add_apks_to_per_app_repos(repodir, apks):
1262 apks_per_app = dict()
1264 apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1265 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1266 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1267 apks_per_app[apk['id']] = apk
1269 if not os.path.exists(apk['per_app_icons']):
1270 logging.info('Adding new repo for only ' + apk['id'])
1271 os.makedirs(apk['per_app_icons'])
1273 apkpath = os.path.join(repodir, apk['apkname'])
1274 shutil.copy(apkpath, apk['per_app_repo'])
1275 apksigpath = apkpath + '.sig'
1276 if os.path.exists(apksigpath):
1277 shutil.copy(apksigpath, apk['per_app_repo'])
1278 apkascpath = apkpath + '.asc'
1279 if os.path.exists(apkascpath):
1280 shutil.copy(apkascpath, apk['per_app_repo'])
1289 global config, options
1291 # Parse command line...
1292 parser = ArgumentParser()
1293 common.setup_global_opts(parser)
1294 parser.add_argument("--create-key", action="store_true", default=False,
1295 help="Create a repo signing key in a keystore")
1296 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1297 help="Create skeleton metadata files that are missing")
1298 parser.add_argument("--delete-unknown", action="store_true", default=False,
1299 help="Delete APKs and/or OBBs without metadata from the repo")
1300 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1301 help="Report on build data status")
1302 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1303 help="Interactively ask about things that need updating.")
1304 parser.add_argument("-I", "--icons", action="store_true", default=False,
1305 help="Resize all the icons exceeding the max pixel size and exit")
1306 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1307 help="Specify editor to use in interactive mode. Default " +
1308 "is /etc/alternatives/editor")
1309 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1310 help="Update the wiki")
1311 parser.add_argument("--pretty", action="store_true", default=False,
1312 help="Produce human-readable index.xml")
1313 parser.add_argument("--clean", action="store_true", default=False,
1314 help="Clean update - don't uses caches, reprocess all apks")
1315 parser.add_argument("--nosign", action="store_true", default=False,
1316 help="When configured for signed indexes, create only unsigned indexes at this stage")
1317 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1318 help="Use date from apk instead of current time for newly added apks")
1319 metadata.add_metadata_arguments(parser)
1320 options = parser.parse_args()
1321 metadata.warnings_action = options.W
1323 config = common.read_config(options)
1325 if not ('jarsigner' in config and 'keytool' in config):
1326 logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1330 if config['archive_older'] != 0:
1331 repodirs.append('archive')
1332 if not os.path.exists('archive'):
1336 resize_all_icons(repodirs)
1339 # check that icons exist now, rather than fail at the end of `fdroid update`
1340 for k in ['repo_icon', 'archive_icon']:
1342 if not os.path.exists(config[k]):
1343 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1346 # if the user asks to create a keystore, do it now, reusing whatever it can
1347 if options.create_key:
1348 if os.path.exists(config['keystore']):
1349 logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1350 logging.critical("\t'" + config['keystore'] + "'")
1353 if 'repo_keyalias' not in config:
1354 config['repo_keyalias'] = socket.getfqdn()
1355 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1356 if 'keydname' not in config:
1357 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1358 common.write_to_config(config, 'keydname', config['keydname'])
1359 if 'keystore' not in config:
1360 config['keystore'] = common.default_config.keystore
1361 common.write_to_config(config, 'keystore', config['keystore'])
1363 password = common.genpassword()
1364 if 'keystorepass' not in config:
1365 config['keystorepass'] = password
1366 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1367 if 'keypass' not in config:
1368 config['keypass'] = password
1369 common.write_to_config(config, 'keypass', config['keypass'])
1370 common.genkeystore(config)
1373 apps = metadata.read_metadata()
1375 # Generate a list of categories...
1377 for app in apps.values():
1378 categories.update(app.Categories)
1380 # Read known apks data (will be updated and written back when we've finished)
1381 knownapks = common.KnownApks()
1383 # Gather information about all the apk files in the repo directory, using
1384 # cached data if possible.
1385 apkcachefile = os.path.join('tmp', 'apkcache')
1386 if not options.clean and os.path.exists(apkcachefile):
1387 with open(apkcachefile, 'rb') as cf:
1388 apkcache = pickle.load(cf, encoding='utf-8')
1389 if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
1394 delete_disabled_builds(apps, apkcache, repodirs)
1396 # Scan all apks in the main repo
1397 apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1399 # Generate warnings for apk's with no metadata (or create skeleton
1400 # metadata files, if requested on the command line)
1403 if apk['id'] not in apps:
1404 if options.create_metadata:
1405 if 'name' not in apk:
1406 logging.error(apk['id'] + ' does not have a name! Skipping...')
1408 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
1409 f.write("License:Unknown\n")
1410 f.write("Web Site:\n")
1411 f.write("Source Code:\n")
1412 f.write("Issue Tracker:\n")
1413 f.write("Changelog:\n")
1414 f.write("Summary:" + apk['name'] + "\n")
1415 f.write("Description:\n")
1416 f.write(apk['name'] + "\n")
1419 logging.info("Generated skeleton metadata for " + apk['id'])
1422 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1423 if options.delete_unknown:
1424 logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1425 rmf = os.path.join(repodirs[0], apk['apkname'])
1426 if not os.path.exists(rmf):
1427 logging.error("Could not find {0} to remove it".format(rmf))
1431 logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1433 # update the metadata with the newly created ones included
1435 apps = metadata.read_metadata()
1437 insert_obbs(repodirs[0], apps, apks)
1439 # Scan the archive repo for apks as well
1440 if len(repodirs) > 1:
1441 archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1447 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
1448 UNSET_VERSION_CODE = -0x100000000
1450 # Some information from the apks needs to be applied up to the application
1451 # level. When doing this, we use the info from the most recent version's apk.
1452 # We deal with figuring out when the app was added and last updated at the
1454 for appid, app in apps.items():
1455 bestver = UNSET_VERSION_CODE
1456 for apk in apks + archapks:
1457 if apk['id'] == appid:
1458 if apk['versioncode'] > bestver:
1459 bestver = apk['versioncode']
1463 if not app.added or apk['added'] < app.added:
1464 app.added = apk['added']
1465 if not app.lastupdated or apk['added'] > app.lastupdated:
1466 app.lastupdated = apk['added']
1469 logging.debug("Don't know when " + appid + " was added")
1470 if not app.lastupdated:
1471 logging.debug("Don't know when " + appid + " was last updated")
1473 if bestver == UNSET_VERSION_CODE:
1474 if app.Name is None:
1475 app.Name = app.AutoName or appid
1477 logging.debug("Application " + appid + " has no packages")
1479 if app.Name is None:
1480 app.Name = bestapk['name']
1481 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1482 if app.CurrentVersionCode is None:
1483 app.CurrentVersionCode = str(bestver)
1485 # Sort the app list by name, then the web site doesn't have to by default.
1486 # (we had to wait until we'd scanned the apks to do this, because mostly the
1487 # name comes from there!)
1488 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1490 # APKs are placed into multiple repos based on the app package, providing
1491 # per-app subscription feeds for nightly builds and things like it
1492 if config['per_app_repos']:
1493 add_apks_to_per_app_repos(repodirs[0], apks)
1494 for appid, app in apps.items():
1495 repodir = os.path.join(appid, 'fdroid', 'repo')
1497 appdict[appid] = app
1498 if os.path.isdir(repodir):
1499 make_index(appdict, [appid], apks, repodir, False, categories)
1501 logging.info('Skipping index generation for ' + appid)
1504 if len(repodirs) > 1:
1505 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1507 # Make the index for the main repo...
1508 make_index(apps, sortedids, apks, repodirs[0], False, categories)
1510 # If there's an archive repo, make the index for it. We already scanned it
1512 if len(repodirs) > 1:
1513 make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1515 if config['update_stats']:
1517 # Update known apks info...
1518 knownapks.writeifchanged()
1520 # Generate latest apps data for widget
1521 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1523 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1525 appid = line.rstrip()
1526 data += appid + "\t"
1528 data += app.Name + "\t"
1529 if app.icon is not None:
1530 data += app.icon + "\t"
1531 data += app.License + "\n"
1532 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1536 apkcache["METADATA_VERSION"] = METADATA_VERSION
1537 with open(apkcachefile, 'wb') as cf:
1538 pickle.dump(apkcache, cf)
1540 # Update the wiki...
1542 update_wiki(apps, sortedids, apks + archapks)
1544 logging.info("Finished.")
1546 if __name__ == "__main__":