3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
30 from datetime import datetime, timedelta
31 from xml.dom.minidom import Document
32 from argparse import ArgumentParser
36 from pyasn1.error import PyAsn1Error
37 from pyasn1.codec.der import decoder, encoder
38 from pyasn1_modules import rfc2315
39 from binascii import hexlify, unhexlify
45 from . import metadata
46 from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
47 from .metadata import MetaDataException
51 screen_densities = ['640', '480', '320', '240', '160', '120']
53 all_screen_densities = ['0'] + screen_densities
55 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
56 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
59 def dpi_to_px(density):
60 return (int(density) * 48) / 160
64 return (int(px) * 160) / 48
67 def get_icon_dir(repodir, density):
69 return os.path.join(repodir, "icons")
70 return os.path.join(repodir, "icons-%s" % density)
73 def get_icon_dirs(repodir):
74 for density in screen_densities:
75 yield get_icon_dir(repodir, density)
78 def get_all_icon_dirs(repodir):
79 for density in all_screen_densities:
80 yield get_icon_dir(repodir, density)
83 def update_wiki(apps, sortedids, apks):
86 :param apps: fully populated list of all applications
87 :param apks: all apks, except...
89 logging.info("Updating wiki")
91 wikiredircat = 'App Redirects'
93 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
94 path=config['wiki_path'])
95 site.login(config['wiki_user'], config['wiki_password'])
97 generated_redirects = {}
99 for appid in sortedids:
104 wikidata += '{{Disabled|' + app.Disabled + '}}\n'
106 for af in app.AntiFeatures:
107 wikidata += '{{AntiFeature|' + af + '}}\n'
112 wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
115 time.strftime('%Y-%m-%d', app.added) if app.added else '',
116 time.strftime('%Y-%m-%d', app.lastupdated) if app.lastupdated else '',
131 wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))
133 wikidata += app.Summary
134 wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"
136 wikidata += "=Description=\n"
137 wikidata += metadata.description_wiki(app.Description) + "\n"
139 wikidata += "=Maintainer Notes=\n"
140 if app.MaintainerNotes:
141 wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
142 wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)
144 # Get a list of all packages for this application...
146 gotcurrentver = False
150 if apk['id'] == appid:
151 if str(apk['versioncode']) == app.CurrentVersionCode:
154 # Include ones we can't build, as a special case...
155 for build in app.builds:
157 if build.vercode == app.CurrentVersionCode:
159 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
160 apklist.append({'versioncode': int(build.vercode),
161 'version': build.version,
162 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
167 if apk['versioncode'] == int(build.vercode):
172 apklist.append({'versioncode': int(build.vercode),
173 'version': build.version,
174 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
176 if app.CurrentVersionCode == '0':
178 # Sort with most recent first...
179 apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
181 wikidata += "=Versions=\n"
182 if len(apklist) == 0:
183 wikidata += "We currently have no versions of this app available."
184 elif not gotcurrentver:
185 wikidata += "We don't have the current version of this app."
187 wikidata += "We have the current version of this app."
188 wikidata += " (Check mode: " + app.UpdateCheckMode + ") "
189 wikidata += " (Auto-update mode: " + app.AutoUpdateMode + ")\n\n"
190 if len(app.NoSourceSince) > 0:
191 wikidata += "This application has partially or entirely been missing source code since version " + app.NoSourceSince + ".\n\n"
192 if len(app.CurrentVersion) > 0:
193 wikidata += "The current (recommended) version is " + app.CurrentVersion
194 wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
197 wikidata += "==" + apk['version'] + "==\n"
199 if 'buildproblem' in apk:
200 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
203 wikidata += "This version is built and signed by "
205 wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
207 wikidata += "the original developer.\n\n"
208 wikidata += "Version code: " + str(apk['versioncode']) + '\n'
210 wikidata += '\n[[Category:' + wikicat + ']]\n'
211 if len(app.NoSourceSince) > 0:
212 wikidata += '\n[[Category:Apps missing source code]]\n'
213 if validapks == 0 and not app.Disabled:
214 wikidata += '\n[[Category:Apps with no packages]]\n'
215 if cantupdate and not app.Disabled:
216 wikidata += "\n[[Category:Apps we cannot update]]\n"
217 if buildfails and not app.Disabled:
218 wikidata += "\n[[Category:Apps with failing builds]]\n"
219 elif not gotcurrentver and not cantupdate and not app.Disabled and app.UpdateCheckMode != "Static":
220 wikidata += '\n[[Category:Apps to Update]]\n'
222 wikidata += '\n[[Category:Apps that are disabled]]\n'
223 if app.UpdateCheckMode == 'None' and not app.Disabled:
224 wikidata += '\n[[Category:Apps with no update check]]\n'
225 for appcat in app.Categories:
226 wikidata += '\n[[Category:{0}]]\n'.format(appcat)
228 # We can't have underscores in the page name, even if they're in
229 # the package ID, because MediaWiki messes with them...
230 pagename = appid.replace('_', ' ')
232 # Drop a trailing newline, because mediawiki is going to drop it anyway
233 # and it we don't we'll think the page has changed when it hasn't...
234 if wikidata.endswith('\n'):
235 wikidata = wikidata[:-1]
237 generated_pages[pagename] = wikidata
239 # Make a redirect from the name to the ID too, unless there's
240 # already an existing page with the name and it isn't a redirect.
242 apppagename = app.Name.replace('_', ' ')
243 apppagename = apppagename.replace('{', '')
244 apppagename = apppagename.replace('}', ' ')
245 apppagename = apppagename.replace(':', ' ')
246 apppagename = apppagename.replace('[', ' ')
247 apppagename = apppagename.replace(']', ' ')
248 # Drop double spaces caused mostly by replacing ':' above
249 apppagename = apppagename.replace(' ', ' ')
250 for expagename in site.allpages(prefix=apppagename,
251 filterredir='nonredirects',
253 if expagename == apppagename:
255 # Another reason not to make the redirect page is if the app name
256 # is the same as it's ID, because that will overwrite the real page
257 # with an redirect to itself! (Although it seems like an odd
258 # scenario this happens a lot, e.g. where there is metadata but no
259 # builds or binaries to extract a name from.
260 if apppagename == pagename:
263 generated_redirects[apppagename] = "#REDIRECT [[" + pagename + "]]\n[[Category:" + wikiredircat + "]]"
265 for tcat, genp in [(wikicat, generated_pages),
266 (wikiredircat, generated_redirects)]:
267 catpages = site.Pages['Category:' + tcat]
269 for page in catpages:
270 existingpages.append(page.name)
271 if page.name in genp:
272 pagetxt = page.edit()
273 if pagetxt != genp[page.name]:
274 logging.debug("Updating modified page " + page.name)
275 page.save(genp[page.name], summary='Auto-updated')
277 logging.debug("Page " + page.name + " is unchanged")
279 logging.warn("Deleting page " + page.name)
280 page.delete('No longer published')
281 for pagename, text in genp.items():
282 logging.debug("Checking " + pagename)
283 if pagename not in existingpages:
284 logging.debug("Creating page " + pagename)
286 newpage = site.Pages[pagename]
287 newpage.save(text, summary='Auto-created')
289 logging.error("...FAILED to create page '{0}'".format(pagename))
291 # Purge server cache to ensure counts are up to date
292 site.pages['Repository Maintenance'].purge()
295 def delete_disabled_builds(apps, apkcache, repodirs):
296 """Delete disabled build outputs.
298 :param apps: list of all applications, as per metadata.read_metadata
299 :param apkcache: current apk cache information
300 :param repodirs: the repo directories to process
302 for appid, app in apps.items():
303 for build in app.builds:
304 if not build.disable:
306 apkfilename = appid + '_' + str(build.vercode) + '.apk'
307 iconfilename = "%s.%s.png" % (
310 for repodir in repodirs:
312 os.path.join(repodir, apkfilename),
313 os.path.join(repodir, apkfilename + '.asc'),
314 os.path.join(repodir, apkfilename[:-4] + "_src.tar.gz"),
316 for density in all_screen_densities:
317 repo_dir = get_icon_dir(repodir, density)
318 files.append(os.path.join(repo_dir, iconfilename))
321 if os.path.exists(f):
322 logging.info("Deleting disabled build output " + f)
324 if apkfilename in apkcache:
325 del apkcache[apkfilename]
328 def resize_icon(iconpath, density):
330 if not os.path.isfile(iconpath):
335 fp = open(iconpath, 'rb')
337 size = dpi_to_px(density)
339 if any(length > size for length in im.size):
341 im.thumbnail((size, size), Image.ANTIALIAS)
342 logging.debug("%s was too large at %s - new size is %s" % (
343 iconpath, oldsize, im.size))
344 im.save(iconpath, "PNG")
346 except Exception as e:
347 logging.error("Failed resizing {0} - {1}".format(iconpath, e))
354 def resize_all_icons(repodirs):
355 """Resize all icons that exceed the max size
357 :param repodirs: the repo directories to process
359 for repodir in repodirs:
360 for density in screen_densities:
361 icon_dir = get_icon_dir(repodir, density)
362 icon_glob = os.path.join(icon_dir, '*.png')
363 for iconpath in glob.glob(icon_glob):
364 resize_icon(iconpath, density)
367 # A signature block file with a .DSA, .RSA, or .EC extension
368 cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
372 """ Get the signing certificate of an apk. To get the same md5 has that
373 Android gets, we encode the .RSA certificate in a specific format and pass
374 it hex-encoded to the md5 digest algorithm.
376 :param apkpath: path to the apk
377 :returns: A string containing the md5 of the signature of the apk or None
378 if an error occurred.
383 # verify the jar signature is correct
384 args = [config['jarsigner'], '-verify', apkpath]
385 p = FDroidPopen(args)
386 if p.returncode != 0:
387 logging.critical(apkpath + " has a bad signature!")
390 with zipfile.ZipFile(apkpath, 'r') as apk:
392 certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
395 logging.error("Found no signing certificates on %s" % apkpath)
398 logging.error("Found multiple signing certificates on %s" % apkpath)
401 cert = apk.read(certs[0])
403 content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
404 if content.getComponentByName('contentType') != rfc2315.signedData:
405 logging.error("Unexpected format.")
408 content = decoder.decode(content.getComponentByName('content'),
409 asn1Spec=rfc2315.SignedData())[0]
411 certificates = content.getComponentByName('certificates')
413 logging.error("Certificates not found.")
416 cert_encoded = encoder.encode(certificates)[4:]
418 return hashlib.md5(hexlify(cert_encoded)).hexdigest()
421 def get_icon_bytes(apkzip, iconsrc):
422 '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
424 return apkzip.read(iconsrc)
426 return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
429 def sha256sum(filename):
430 '''Calculate the sha256 of the given file'''
431 sha = hashlib.sha256()
432 with open(filename, 'rb') as f:
438 return sha.hexdigest()
441 def insert_obbs(repodir, apps, apks):
442 """Scans the .obb files in a given repo directory and adds them to the
443 relevant APK instances. OBB files have versionCodes like APK
444 files, and they are loosely associated. If there is an OBB file
445 present, then any APK with the same or higher versionCode will use
446 that OBB file. There are two OBB types: main and patch, each APK
447 can only have only have one of each.
449 https://developer.android.com/google/play/expansion-files.html
451 :param repodir: repo directory to scan
452 :param apps: list of current, valid apps
453 :param apks: current information on all APKs
457 def obbWarnDelete(f, msg):
458 logging.warning(msg + f)
459 if options.delete_unknown:
460 logging.error("Deleting unknown file: " + f)
464 java_Integer_MIN_VALUE = -pow(2, 31)
465 for f in glob.glob(os.path.join(repodir, '*.obb')):
466 obbfile = os.path.basename(f)
467 # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
468 chunks = obbfile.split('.')
469 if chunks[0] != 'main' and chunks[0] != 'patch':
470 obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
472 if not re.match(r'^-?[0-9]+$', chunks[1]):
473 obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
475 versioncode = int(chunks[1])
476 packagename = ".".join(chunks[2:-1])
478 highestVersionCode = java_Integer_MIN_VALUE
479 if packagename not in apps.keys():
480 obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
483 if packagename == apk['id'] and apk['versioncode'] > highestVersionCode:
484 highestVersionCode = apk['versioncode']
485 if versioncode > highestVersionCode:
486 obbWarnDelete(f, 'OBB file has newer versioncode(' + str(versioncode)
487 + ') than any APK: ')
489 obbsha256 = sha256sum(f)
490 obbs.append((packagename, versioncode, obbfile, obbsha256))
493 for (packagename, versioncode, obbfile, obbsha256) in sorted(obbs, reverse=True):
494 if versioncode <= apk['versioncode'] and packagename == apk['id']:
495 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
496 apk['obbMainFile'] = obbfile
497 apk['obbMainFileSha256'] = obbsha256
498 elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
499 apk['obbPatchFile'] = obbfile
500 apk['obbPatchFileSha256'] = obbsha256
501 if 'obbMainFile' in apk and 'obbPatchFile' in apk:
505 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
506 """Scan the apks in the given repo directory.
508 This also extracts the icons.
510 :param apps: list of all applications, as per metadata.read_metadata
511 :param apkcache: current apk cache information
512 :param repodir: repo directory to scan
513 :param knownapks: known apks info
514 :param use_date_from_apk: use date from APK (instead of current date)
516 :returns: (apks, cachechanged) where apks is a list of apk information,
517 and cachechanged is True if the apkcache got changed.
522 for icon_dir in get_all_icon_dirs(repodir):
523 if os.path.exists(icon_dir):
525 shutil.rmtree(icon_dir)
526 os.makedirs(icon_dir)
528 os.makedirs(icon_dir)
531 name_pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
532 vercode_pat = re.compile(".*versionCode='([0-9]*)'.*")
533 vername_pat = re.compile(".*versionName='([^']*)'.*")
534 label_pat = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
535 icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
536 icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
537 sdkversion_pat = re.compile(".*'([0-9]*)'.*")
538 permission_pat = re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
539 feature_pat = re.compile(".*name='([^']*)'.*")
540 for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
542 apkfilename = apkfile[len(repodir) + 1:]
543 if ' ' in apkfilename:
544 logging.critical("Spaces in filenames are not allowed.")
547 shasum = sha256sum(apkfile)
550 if apkfilename in apkcache:
551 apk = apkcache[apkfilename]
552 if apk['sha256'] == shasum:
553 logging.debug("Reading " + apkfilename + " from cache")
556 logging.debug("Ignoring stale cache data for " + apkfilename)
559 logging.debug("Processing " + apkfilename)
561 apk['apkname'] = apkfilename
562 apk['sha256'] = shasum
563 srcfilename = apkfilename[:-4] + "_src.tar.gz"
564 if os.path.exists(os.path.join(repodir, srcfilename)):
565 apk['srcname'] = srcfilename
566 apk['size'] = os.path.getsize(apkfile)
567 apk['uses-permission'] = set()
568 apk['uses-permission-sdk-23'] = set()
569 apk['features'] = set()
570 apk['icons_src'] = {}
572 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
573 if p.returncode != 0:
574 if options.delete_unknown:
575 if os.path.exists(apkfile):
576 logging.error("Failed to get apk information, deleting " + apkfile)
579 logging.error("Could not find {0} to remove it".format(apkfile))
581 logging.error("Failed to get apk information, skipping " + apkfile)
583 for line in p.output.splitlines():
584 if line.startswith("package:"):
586 apk['id'] = re.match(name_pat, line).group(1)
587 apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
588 apk['version'] = re.match(vername_pat, line).group(1)
589 except Exception as e:
590 logging.error("Package matching failed: " + str(e))
591 logging.info("Line was: " + line)
593 elif line.startswith("application:"):
594 apk['name'] = re.match(label_pat, line).group(1)
595 # Keep path to non-dpi icon in case we need it
596 match = re.match(icon_pat_nodpi, line)
598 apk['icons_src']['-1'] = match.group(1)
599 elif line.startswith("launchable-activity:"):
600 # Only use launchable-activity as fallback to application
602 apk['name'] = re.match(label_pat, line).group(1)
603 if '-1' not in apk['icons_src']:
604 match = re.match(icon_pat_nodpi, line)
606 apk['icons_src']['-1'] = match.group(1)
607 elif line.startswith("application-icon-"):
608 match = re.match(icon_pat, line)
610 density = match.group(1)
611 path = match.group(2)
612 apk['icons_src'][density] = path
613 elif line.startswith("sdkVersion:"):
614 m = re.match(sdkversion_pat, line)
616 logging.error(line.replace('sdkVersion:', '')
617 + ' is not a valid minSdkVersion!')
619 apk['minSdkVersion'] = m.group(1)
620 # if target not set, default to min
621 if 'targetSdkVersion' not in apk:
622 apk['targetSdkVersion'] = m.group(1)
623 elif line.startswith("targetSdkVersion:"):
624 m = re.match(sdkversion_pat, line)
626 logging.error(line.replace('targetSdkVersion:', '')
627 + ' is not a valid targetSdkVersion!')
629 apk['targetSdkVersion'] = m.group(1)
630 elif line.startswith("maxSdkVersion:"):
631 apk['maxSdkVersion'] = re.match(sdkversion_pat, line).group(1)
632 elif line.startswith("native-code:"):
633 apk['nativecode'] = []
634 for arch in line[13:].split(' '):
635 apk['nativecode'].append(arch[1:-1])
636 elif line.startswith('uses-permission:'):
637 perm_match = re.match(permission_pat, line).groupdict()
639 permission = UsesPermission(
641 perm_match['maxSdkVersion']
644 apk['uses-permission'].add(permission)
645 elif line.startswith('uses-permission-sdk-23:'):
646 perm_match = re.match(permission_pat, line).groupdict()
648 permission_sdk_23 = UsesPermissionSdk23(
650 perm_match['maxSdkVersion']
653 apk['uses-permission-sdk-23'].add(permission_sdk_23)
655 elif line.startswith('uses-feature:'):
656 feature = re.match(feature_pat, line).group(1)
657 # Filter out this, it's only added with the latest SDK tools and
658 # causes problems for lots of apps.
659 if feature != "android.hardware.screen.portrait" \
660 and feature != "android.hardware.screen.landscape":
661 if feature.startswith("android.feature."):
662 feature = feature[16:]
663 apk['features'].add(feature)
665 if 'minSdkVersion' not in apk:
666 logging.warn("No SDK version information found in {0}".format(apkfile))
667 apk['minSdkVersion'] = 1
669 # Check for debuggable apks...
670 if common.isApkDebuggable(apkfile, config):
671 logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
673 # Get the signature (or md5 of, to be precise)...
674 logging.debug('Getting signature of {0}'.format(apkfile))
675 apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
677 logging.critical("Failed to get apk signature")
680 apkzip = zipfile.ZipFile(apkfile, 'r')
682 # if an APK has files newer than the system time, suggest updating
683 # the system clock. This is useful for offline systems, used for
684 # signing, which do not have another source of clock sync info. It
685 # has to be more than 24 hours newer because ZIP/APK files do not
686 # store timezone info
687 manifest = apkzip.getinfo('AndroidManifest.xml')
688 if manifest.date_time[1] == 0: # month can't be zero
689 logging.debug('AndroidManifest.xml has no date')
691 dt_obj = datetime(*manifest.date_time)
692 checkdt = dt_obj - timedelta(1)
693 if datetime.today() < checkdt:
694 logging.warn('System clock is older than manifest in: '
696 + '\nSet clock to that time using:\n'
697 + 'sudo date -s "' + str(dt_obj) + '"')
699 iconfilename = "%s.%s.png" % (
703 # Extract the icon file...
705 for density in screen_densities:
706 if density not in apk['icons_src']:
707 empty_densities.append(density)
709 iconsrc = apk['icons_src'][density]
710 icon_dir = get_icon_dir(repodir, density)
711 icondest = os.path.join(icon_dir, iconfilename)
714 with open(icondest, 'wb') as f:
715 f.write(get_icon_bytes(apkzip, iconsrc))
716 apk['icons'][density] = iconfilename
719 logging.warn("Error retrieving icon file")
720 del apk['icons'][density]
721 del apk['icons_src'][density]
722 empty_densities.append(density)
724 if '-1' in apk['icons_src']:
725 iconsrc = apk['icons_src']['-1']
726 iconpath = os.path.join(
727 get_icon_dir(repodir, '0'), iconfilename)
728 with open(iconpath, 'wb') as f:
729 f.write(get_icon_bytes(apkzip, iconsrc))
731 im = Image.open(iconpath)
732 dpi = px_to_dpi(im.size[0])
733 for density in screen_densities:
734 if density in apk['icons']:
736 if density == screen_densities[-1] or dpi >= int(density):
737 apk['icons'][density] = iconfilename
738 shutil.move(iconpath,
739 os.path.join(get_icon_dir(repodir, density), iconfilename))
740 empty_densities.remove(density)
742 except Exception as e:
743 logging.warn("Failed reading {0} - {1}".format(iconpath, e))
746 apk['icon'] = iconfilename
750 # First try resizing down to not lose quality
752 for density in screen_densities:
753 if density not in empty_densities:
754 last_density = density
756 if last_density is None:
758 logging.debug("Density %s not available, resizing down from %s"
759 % (density, last_density))
761 last_iconpath = os.path.join(
762 get_icon_dir(repodir, last_density), iconfilename)
763 iconpath = os.path.join(
764 get_icon_dir(repodir, density), iconfilename)
767 fp = open(last_iconpath, 'rb')
770 size = dpi_to_px(density)
772 im.thumbnail((size, size), Image.ANTIALIAS)
773 im.save(iconpath, "PNG")
774 empty_densities.remove(density)
776 logging.warning("Invalid image file at %s" % last_iconpath)
781 # Then just copy from the highest resolution available
783 for density in reversed(screen_densities):
784 if density not in empty_densities:
785 last_density = density
787 if last_density is None:
789 logging.debug("Density %s not available, copying from lower density %s"
790 % (density, last_density))
793 os.path.join(get_icon_dir(repodir, last_density), iconfilename),
794 os.path.join(get_icon_dir(repodir, density), iconfilename))
796 empty_densities.remove(density)
798 for density in screen_densities:
799 icon_dir = get_icon_dir(repodir, density)
800 icondest = os.path.join(icon_dir, iconfilename)
801 resize_icon(icondest, density)
803 # Copy from icons-mdpi to icons since mdpi is the baseline density
804 baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
805 if os.path.isfile(baseline):
806 apk['icons']['0'] = iconfilename
807 shutil.copyfile(baseline,
808 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
810 if use_date_from_apk and manifest.date_time[1] != 0:
811 default_date_param = datetime(*manifest.date_time).utctimetuple()
813 default_date_param = None
815 # Record in known apks, getting the added date at the same time..
816 added = knownapks.recordapk(apk['apkname'], apk['id'], default_date=default_date_param)
820 apkcache[apkfilename] = apk
825 return apks, cachechanged
828 repo_pubkey_fingerprint = None
831 # Generate a certificate fingerprint the same way keytool does it
832 # (but with slightly different formatting)
833 def cert_fingerprint(data):
834 digest = hashlib.sha256(data).digest()
836 ret.append(' '.join("%02X" % b for b in bytearray(digest)))
840 def extract_pubkey():
841 global repo_pubkey_fingerprint
842 if 'repo_pubkey' in config:
843 pubkey = unhexlify(config['repo_pubkey'])
845 p = FDroidPopenBytes([config['keytool'], '-exportcert',
846 '-alias', config['repo_keyalias'],
847 '-keystore', config['keystore'],
848 '-storepass:file', config['keystorepassfile']]
849 + config['smartcardoptions'],
850 output=False, stderr_to_stdout=False)
851 if p.returncode != 0 or len(p.output) < 20:
852 msg = "Failed to get repo pubkey!"
853 if config['keystore'] == 'NONE':
854 msg += ' Is your crypto smartcard plugged in?'
855 logging.critical(msg)
858 repo_pubkey_fingerprint = cert_fingerprint(pubkey)
859 return hexlify(pubkey)
862 def make_index(apps, sortedids, apks, repodir, archive, categories):
863 """Make a repo index.
865 :param apps: fully populated apps list
866 :param apks: full populated apks list
867 :param repodir: the repo directory
868 :param archive: True if this is the archive repo, False if it's the
870 :param categories: list of categories
875 def addElement(name, value, doc, parent):
876 el = doc.createElement(name)
877 el.appendChild(doc.createTextNode(value))
878 parent.appendChild(el)
880 def addElementNonEmpty(name, value, doc, parent):
883 addElement(name, value, doc, parent)
885 def addElementCDATA(name, value, doc, parent):
886 el = doc.createElement(name)
887 el.appendChild(doc.createCDATASection(value))
888 parent.appendChild(el)
890 root = doc.createElement("fdroid")
891 doc.appendChild(root)
893 repoel = doc.createElement("repo")
895 mirrorcheckfailed = False
896 for mirror in config.get('mirrors', []):
897 base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
898 if config.get('nonstandardwebroot') is not True and base != 'fdroid':
899 logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
900 mirrorcheckfailed = True
901 if mirrorcheckfailed:
905 repoel.setAttribute("name", config['archive_name'])
906 if config['repo_maxage'] != 0:
907 repoel.setAttribute("maxage", str(config['repo_maxage']))
908 repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
909 repoel.setAttribute("url", config['archive_url'])
910 addElement('description', config['archive_description'], doc, repoel)
911 urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
912 for mirror in config.get('mirrors', []):
913 addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
916 repoel.setAttribute("name", config['repo_name'])
917 if config['repo_maxage'] != 0:
918 repoel.setAttribute("maxage", str(config['repo_maxage']))
919 repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
920 repoel.setAttribute("url", config['repo_url'])
921 addElement('description', config['repo_description'], doc, repoel)
922 urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
923 for mirror in config.get('mirrors', []):
924 addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
926 repoel.setAttribute("version", str(METADATA_VERSION))
927 repoel.setAttribute("timestamp", str(int(time.time())))
930 if not options.nosign:
931 if 'repo_keyalias' not in config:
933 logging.critical("'repo_keyalias' not found in config.py!")
934 if 'keystore' not in config:
936 logging.critical("'keystore' not found in config.py!")
937 if 'keystorepass' not in config and 'keystorepassfile' not in config:
939 logging.critical("'keystorepass' not found in config.py!")
940 if 'keypass' not in config and 'keypassfile' not in config:
942 logging.critical("'keypass' not found in config.py!")
943 if not os.path.exists(config['keystore']):
945 logging.critical("'" + config['keystore'] + "' does not exist!")
947 logging.warning("`fdroid update` requires a signing key, you can create one using:")
948 logging.warning("\tfdroid update --create-key")
951 repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
952 root.appendChild(repoel)
954 for appid in sortedids:
957 if app.Disabled is not None:
960 # Get a list of the apks for this app...
963 if apk['id'] == appid:
966 if len(apklist) == 0:
969 apel = doc.createElement("application")
970 apel.setAttribute("id", app.id)
971 root.appendChild(apel)
973 addElement('id', app.id, doc, apel)
975 addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
977 addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastupdated), doc, apel)
978 addElement('name', app.Name, doc, apel)
979 addElement('summary', app.Summary, doc, apel)
981 addElement('icon', app.icon, doc, apel)
985 return ("fdroid.app:" + appid, apps[appid].Name)
986 raise MetaDataException("Cannot resolve app id " + appid)
989 metadata.description_html(app.Description, linkres),
991 addElement('license', app.License, doc, apel)
993 addElement('categories', ','.join(app.Categories), doc, apel)
994 # We put the first (primary) category in LAST, which will have
995 # the desired effect of making clients that only understand one
996 # category see that one.
997 addElement('category', app.Categories[0], doc, apel)
998 addElement('web', app.WebSite, doc, apel)
999 addElement('source', app.SourceCode, doc, apel)
1000 addElement('tracker', app.IssueTracker, doc, apel)
1001 addElementNonEmpty('changelog', app.Changelog, doc, apel)
1002 addElementNonEmpty('author', app.AuthorName, doc, apel)
1003 addElementNonEmpty('email', app.AuthorEmail, doc, apel)
1004 addElementNonEmpty('donate', app.Donate, doc, apel)
1005 addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel)
1006 addElementNonEmpty('litecoin', app.Litecoin, doc, apel)
1007 addElementNonEmpty('flattr', app.FlattrID, doc, apel)
1009 # These elements actually refer to the current version (i.e. which
1010 # one is recommended. They are historically mis-named, and need
1011 # changing, but stay like this for now to support existing clients.
1012 addElement('marketversion', app.CurrentVersion, doc, apel)
1013 addElement('marketvercode', app.CurrentVersionCode, doc, apel)
1015 if app.AntiFeatures:
1016 af = app.AntiFeatures
1018 addElementNonEmpty('antifeatures', ','.join(af), doc, apel)
1020 pv = app.Provides.split(',')
1021 addElementNonEmpty('provides', ','.join(pv), doc, apel)
1022 if app.RequiresRoot:
1023 addElement('requirements', 'root', doc, apel)
1025 # Sort the apk list into version order, just so the web site
1026 # doesn't have to do any work by default...
1027 apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
1029 # Check for duplicates - they will make the client unhappy...
1030 for i in range(len(apklist) - 1):
1031 if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
1032 logging.critical("duplicate versions: '%s' - '%s'" % (
1033 apklist[i]['apkname'], apklist[i + 1]['apkname']))
1036 current_version_code = 0
1037 current_version_file = None
1039 # find the APK for the "Current Version"
1040 if current_version_code < apk['versioncode']:
1041 current_version_code = apk['versioncode']
1042 if current_version_code < int(app.CurrentVersionCode):
1043 current_version_file = apk['apkname']
1045 apkel = doc.createElement("package")
1046 apel.appendChild(apkel)
1047 addElement('version', apk['version'], doc, apkel)
1048 addElement('versioncode', str(apk['versioncode']), doc, apkel)
1049 addElement('apkname', apk['apkname'], doc, apkel)
1050 if 'srcname' in apk:
1051 addElement('srcname', apk['srcname'], doc, apkel)
1052 for hash_type in ['sha256']:
1053 if hash_type not in apk:
1055 hashel = doc.createElement("hash")
1056 hashel.setAttribute("type", hash_type)
1057 hashel.appendChild(doc.createTextNode(apk[hash_type]))
1058 apkel.appendChild(hashel)
1059 addElement('sig', apk['sig'], doc, apkel)
1060 addElement('size', str(apk['size']), doc, apkel)
1061 addElement('sdkver', str(apk['minSdkVersion']), doc, apkel)
1062 if 'targetSdkVersion' in apk:
1063 addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
1064 if 'maxSdkVersion' in apk:
1065 addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
1066 addElementNonEmpty('obbMainFile', apk.get('obbMainFile'), doc, apkel)
1067 addElementNonEmpty('obbMainFileSha256', apk.get('obbMainFileSha256'), doc, apkel)
1068 addElementNonEmpty('obbPatchFile', apk.get('obbPatchFile'), doc, apkel)
1069 addElementNonEmpty('obbPatchFileSha256', apk.get('obbPatchFileSha256'), doc, apkel)
1071 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
1073 # TODO: remove old permission format
1074 old_permissions = set()
1075 for perm in apk['uses-permission']:
1076 perm_name = perm.name
1077 if perm_name.startswith("android.permission."):
1078 perm_name = perm_name[19:]
1079 old_permissions.add(perm_name)
1080 addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel)
1082 for permission in apk['uses-permission']:
1083 permel = doc.createElement('uses-permission')
1084 permel.setAttribute('name', permission.name)
1085 if permission.maxSdkVersion is not None:
1086 permel.setAttribute('maxSdkVersion', permission.maxSdkVersion)
1087 apkel.appendChild(permel)
1088 for permission_sdk_23 in apk['uses-permission-sdk-23']:
1089 permel = doc.createElement('uses-permission-sdk-23')
1090 permel.setAttribute('name', permission_sdk_23.name)
1091 if permission_sdk_23.maxSdkVersion is not None:
1092 permel.setAttribute('maxSdkVersion', permission_sdk_23.maxSdkVersion)
1093 apkel.appendChild(permel)
1094 if 'nativecode' in apk:
1095 addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
1096 addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
1098 if current_version_file is not None \
1099 and config['make_current_version_link'] \
1100 and repodir == 'repo': # only create these
1101 namefield = config['current_version_name_source']
1102 sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
1103 apklinkname = sanitized_name + '.apk'
1104 current_version_path = os.path.join(repodir, current_version_file)
1105 if os.path.islink(apklinkname):
1106 os.remove(apklinkname)
1107 os.symlink(current_version_path, apklinkname)
1108 # also symlink gpg signature, if it exists
1109 for extension in ('.asc', '.sig'):
1110 sigfile_path = current_version_path + extension
1111 if os.path.exists(sigfile_path):
1112 siglinkname = apklinkname + extension
1113 if os.path.islink(siglinkname):
1114 os.remove(siglinkname)
1115 os.symlink(sigfile_path, siglinkname)
1118 output = doc.toprettyxml(encoding='utf-8')
1120 output = doc.toxml(encoding='utf-8')
1122 with open(os.path.join(repodir, 'index.xml'), 'wb') as f:
1125 if 'repo_keyalias' in config:
1128 logging.info("Creating unsigned index in preparation for signing")
1130 logging.info("Creating signed index with this key (SHA256):")
1131 logging.info("%s" % repo_pubkey_fingerprint)
1133 # Create a jar of the index...
1134 jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar'
1135 p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir)
1136 if p.returncode != 0:
1137 logging.critical("Failed to create {0}".format(jar_output))
1141 signed = os.path.join(repodir, 'index.jar')
1143 # Remove old signed index if not signing
1144 if os.path.exists(signed):
1147 args = [config['jarsigner'], '-keystore', config['keystore'],
1148 '-storepass:file', config['keystorepassfile'],
1149 '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
1150 signed, config['repo_keyalias']]
1151 if config['keystore'] == 'NONE':
1152 args += config['smartcardoptions']
1153 else: # smardcards never use -keypass
1154 args += ['-keypass:file', config['keypassfile']]
1155 p = FDroidPopen(args)
1156 if p.returncode != 0:
1157 logging.critical("Failed to sign index")
1160 # Copy the repo icon into the repo directory...
1161 icon_dir = os.path.join(repodir, 'icons')
1162 iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
1163 shutil.copyfile(config['repo_icon'], iconfilename)
1165 # Write a category list in the repo to allow quick access...
1167 for cat in categories:
1168 catdata += cat + '\n'
1169 with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
1173 def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions):
1175 for appid, app in apps.items():
1177 if app.ArchivePolicy:
1178 keepversions = int(app.ArchivePolicy[:-9])
1180 keepversions = defaultkeepversions
1182 def filter_apk_list_sorted(apk_list):
1184 for apk in apk_list:
1185 if apk['id'] == appid:
1188 # Sort the apk list by version code. First is highest/newest.
1189 return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
1191 def move_file(from_dir, to_dir, filename, ignore_missing):
1192 from_path = os.path.join(from_dir, filename)
1193 if ignore_missing and not os.path.exists(from_path):
1195 to_path = os.path.join(to_dir, filename)
1196 shutil.move(from_path, to_path)
1198 logging.debug("Checking archiving for {0} - apks:{1}, keepversions:{2}, archapks:{3}"
1199 .format(appid, len(apks), keepversions, len(archapks)))
1201 if len(apks) > keepversions:
1202 apklist = filter_apk_list_sorted(apks)
1203 # Move back the ones we don't want.
1204 for apk in apklist[keepversions:]:
1205 logging.info("Moving " + apk['apkname'] + " to archive")
1206 move_file(repodir, archivedir, apk['apkname'], False)
1207 move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
1208 for density in all_screen_densities:
1209 repo_icon_dir = get_icon_dir(repodir, density)
1210 archive_icon_dir = get_icon_dir(archivedir, density)
1211 if density not in apk['icons']:
1213 move_file(repo_icon_dir, archive_icon_dir, apk['icons'][density], True)
1214 if 'srcname' in apk:
1215 move_file(repodir, archivedir, apk['srcname'], False)
1216 archapks.append(apk)
1218 elif len(apks) < keepversions and len(archapks) > 0:
1219 required = keepversions - len(apks)
1220 archapklist = filter_apk_list_sorted(archapks)
1221 # Move forward the ones we want again.
1222 for apk in archapklist[:required]:
1223 logging.info("Moving " + apk['apkname'] + " from archive")
1224 move_file(archivedir, repodir, apk['apkname'], False)
1225 move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
1226 for density in all_screen_densities:
1227 repo_icon_dir = get_icon_dir(repodir, density)
1228 archive_icon_dir = get_icon_dir(archivedir, density)
1229 if density not in apk['icons']:
1231 move_file(archive_icon_dir, repo_icon_dir, apk['icons'][density], True)
1232 if 'srcname' in apk:
1233 move_file(archivedir, repodir, apk['srcname'], False)
1234 archapks.remove(apk)
1238 def add_apks_to_per_app_repos(repodir, apks):
1239 apks_per_app = dict()
1241 apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
1242 apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
1243 apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
1244 apks_per_app[apk['id']] = apk
1246 if not os.path.exists(apk['per_app_icons']):
1247 logging.info('Adding new repo for only ' + apk['id'])
1248 os.makedirs(apk['per_app_icons'])
1250 apkpath = os.path.join(repodir, apk['apkname'])
1251 shutil.copy(apkpath, apk['per_app_repo'])
1252 apksigpath = apkpath + '.sig'
1253 if os.path.exists(apksigpath):
1254 shutil.copy(apksigpath, apk['per_app_repo'])
1255 apkascpath = apkpath + '.asc'
1256 if os.path.exists(apkascpath):
1257 shutil.copy(apkascpath, apk['per_app_repo'])
1266 global config, options
1268 # Parse command line...
1269 parser = ArgumentParser()
1270 common.setup_global_opts(parser)
1271 parser.add_argument("--create-key", action="store_true", default=False,
1272 help="Create a repo signing key in a keystore")
1273 parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
1274 help="Create skeleton metadata files that are missing")
1275 parser.add_argument("--delete-unknown", action="store_true", default=False,
1276 help="Delete APKs and/or OBBs without metadata from the repo")
1277 parser.add_argument("-b", "--buildreport", action="store_true", default=False,
1278 help="Report on build data status")
1279 parser.add_argument("-i", "--interactive", default=False, action="store_true",
1280 help="Interactively ask about things that need updating.")
1281 parser.add_argument("-I", "--icons", action="store_true", default=False,
1282 help="Resize all the icons exceeding the max pixel size and exit")
1283 parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
1284 help="Specify editor to use in interactive mode. Default " +
1285 "is /etc/alternatives/editor")
1286 parser.add_argument("-w", "--wiki", default=False, action="store_true",
1287 help="Update the wiki")
1288 parser.add_argument("--pretty", action="store_true", default=False,
1289 help="Produce human-readable index.xml")
1290 parser.add_argument("--clean", action="store_true", default=False,
1291 help="Clean update - don't uses caches, reprocess all apks")
1292 parser.add_argument("--nosign", action="store_true", default=False,
1293 help="When configured for signed indexes, create only unsigned indexes at this stage")
1294 parser.add_argument("--use-date-from-apk", action="store_true", default=False,
1295 help="Use date from apk instead of current time for newly added apks")
1296 options = parser.parse_args()
1298 config = common.read_config(options)
1300 if not ('jarsigner' in config and 'keytool' in config):
1301 logging.critical('Java JDK not found! Install in standard location or set java_paths!')
1305 if config['archive_older'] != 0:
1306 repodirs.append('archive')
1307 if not os.path.exists('archive'):
1311 resize_all_icons(repodirs)
1314 # check that icons exist now, rather than fail at the end of `fdroid update`
1315 for k in ['repo_icon', 'archive_icon']:
1317 if not os.path.exists(config[k]):
1318 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
1321 # if the user asks to create a keystore, do it now, reusing whatever it can
1322 if options.create_key:
1323 if os.path.exists(config['keystore']):
1324 logging.critical("Cowardily refusing to overwrite existing signing key setup!")
1325 logging.critical("\t'" + config['keystore'] + "'")
1328 if 'repo_keyalias' not in config:
1329 config['repo_keyalias'] = socket.getfqdn()
1330 common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
1331 if 'keydname' not in config:
1332 config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
1333 common.write_to_config(config, 'keydname', config['keydname'])
1334 if 'keystore' not in config:
1335 config['keystore'] = common.default_config.keystore
1336 common.write_to_config(config, 'keystore', config['keystore'])
1338 password = common.genpassword()
1339 if 'keystorepass' not in config:
1340 config['keystorepass'] = password
1341 common.write_to_config(config, 'keystorepass', config['keystorepass'])
1342 if 'keypass' not in config:
1343 config['keypass'] = password
1344 common.write_to_config(config, 'keypass', config['keypass'])
1345 common.genkeystore(config)
1348 apps = metadata.read_metadata()
1350 # Generate a list of categories...
1352 for app in apps.values():
1353 categories.update(app.Categories)
1355 # Read known apks data (will be updated and written back when we've finished)
1356 knownapks = common.KnownApks()
1358 # Gather information about all the apk files in the repo directory, using
1359 # cached data if possible.
1360 apkcachefile = os.path.join('tmp', 'apkcache')
1361 if not options.clean and os.path.exists(apkcachefile):
1362 with open(apkcachefile, 'rb') as cf:
1363 apkcache = pickle.load(cf, encoding='utf-8')
1364 if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
1369 delete_disabled_builds(apps, apkcache, repodirs)
1371 # Scan all apks in the main repo
1372 apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
1374 # Generate warnings for apk's with no metadata (or create skeleton
1375 # metadata files, if requested on the command line)
1378 if apk['id'] not in apps:
1379 if options.create_metadata:
1380 if 'name' not in apk:
1381 logging.error(apk['id'] + ' does not have a name! Skipping...')
1383 f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
1384 f.write("License:Unknown\n")
1385 f.write("Web Site:\n")
1386 f.write("Source Code:\n")
1387 f.write("Issue Tracker:\n")
1388 f.write("Changelog:\n")
1389 f.write("Summary:" + apk['name'] + "\n")
1390 f.write("Description:\n")
1391 f.write(apk['name'] + "\n")
1394 logging.info("Generated skeleton metadata for " + apk['id'])
1397 msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
1398 if options.delete_unknown:
1399 logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
1400 rmf = os.path.join(repodirs[0], apk['apkname'])
1401 if not os.path.exists(rmf):
1402 logging.error("Could not find {0} to remove it".format(rmf))
1406 logging.warn(msg + "\n\tUse `fdroid update -c` to create it.")
1408 # update the metadata with the newly created ones included
1410 apps = metadata.read_metadata()
1412 insert_obbs(repodirs[0], apps, apks)
1414 # Scan the archive repo for apks as well
1415 if len(repodirs) > 1:
1416 archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
1422 # Some information from the apks needs to be applied up to the application
1423 # level. When doing this, we use the info from the most recent version's apk.
1424 # We deal with figuring out when the app was added and last updated at the
1426 for appid, app in apps.items():
1428 for apk in apks + archapks:
1429 if apk['id'] == appid:
1430 if apk['versioncode'] > bestver:
1431 bestver = apk['versioncode']
1435 if not app.added or apk['added'] < app.added:
1436 app.added = apk['added']
1437 if not app.lastupdated or apk['added'] > app.lastupdated:
1438 app.lastupdated = apk['added']
1441 logging.debug("Don't know when " + appid + " was added")
1442 if not app.lastupdated:
1443 logging.debug("Don't know when " + appid + " was last updated")
1446 if app.Name is None:
1447 app.Name = app.AutoName or appid
1449 logging.debug("Application " + appid + " has no packages")
1451 if app.Name is None:
1452 app.Name = bestapk['name']
1453 app.icon = bestapk['icon'] if 'icon' in bestapk else None
1454 if app.CurrentVersionCode is None:
1455 app.CurrentVersionCode = str(bestver)
1457 # Sort the app list by name, then the web site doesn't have to by default.
1458 # (we had to wait until we'd scanned the apks to do this, because mostly the
1459 # name comes from there!)
1460 sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper())
1462 # APKs are placed into multiple repos based on the app package, providing
1463 # per-app subscription feeds for nightly builds and things like it
1464 if config['per_app_repos']:
1465 add_apks_to_per_app_repos(repodirs[0], apks)
1466 for appid, app in apps.items():
1467 repodir = os.path.join(appid, 'fdroid', 'repo')
1469 appdict[appid] = app
1470 if os.path.isdir(repodir):
1471 make_index(appdict, [appid], apks, repodir, False, categories)
1473 logging.info('Skipping index generation for ' + appid)
1476 if len(repodirs) > 1:
1477 archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
1479 # Make the index for the main repo...
1480 make_index(apps, sortedids, apks, repodirs[0], False, categories)
1482 # If there's an archive repo, make the index for it. We already scanned it
1484 if len(repodirs) > 1:
1485 make_index(apps, sortedids, archapks, repodirs[1], True, categories)
1487 if config['update_stats']:
1489 # Update known apks info...
1490 knownapks.writeifchanged()
1492 # Generate latest apps data for widget
1493 if os.path.exists(os.path.join('stats', 'latestapps.txt')):
1495 with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
1497 appid = line.rstrip()
1498 data += appid + "\t"
1500 data += app.Name + "\t"
1501 if app.icon is not None:
1502 data += app.icon + "\t"
1503 data += app.License + "\n"
1504 with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
1508 apkcache["METADATA_VERSION"] = METADATA_VERSION
1509 with open(apkcachefile, 'wb') as cf:
1510 pickle.dump(apkcache, cf)
1512 # Update the wiki...
1514 update_wiki(apps, sortedids, apks + archapks)
1516 logging.info("Finished.")
1518 if __name__ == "__main__":