From: Hans-Christoph Steiner Date: Mon, 7 Nov 2016 20:47:53 +0000 (+0100) Subject: allow metadata to be embedded in source repos via .fdroid.yml X-Git-Tag: 0.8~145^2~2 X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=commitdiff_plain;h=a4e4310803a463433eb7515c2038a8d3ea44edc4;p=fdroidserver.git allow metadata to be embedded in source repos via .fdroid.yml This allows a source repo to include a complete metadata file so that it can be built directly in place using `fdroid build`. If that app is then included in fdroiddata, it will first load the source repo type and URL from fdroiddata, then read .fdroid.yml if it exists, then include the rest of the metadata as specified in fdroiddata, so that fdroiddata has precedence over the metadata in the source code. This lets `fdroid build` apps without having a whole fdroiddata setup, but instead just directly in place in the source code. This also lets devs optionallu maintain the fdroid metadata as part of their app, rather than in fdroiddata without loosing any control. This should make it easier to spread around the maintenance load. --- diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 5aa2d7c6..281ad523 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -1100,9 +1100,10 @@ def main(): extlib_dir = os.path.join(build_dir, 'extlib') # Read all app and srclib metadata - allapps = metadata.read_metadata(xref=not options.onserver) - + pkgs = common.read_pkg_args(options.appid, True) + allapps = metadata.read_metadata(not options.onserver, pkgs) apps = common.read_app_args(options.appid, allapps, True) + for appid, app in list(apps.items()): if (app.Disabled and not options.force) or not app.RepoType or not app.builds: del apps[appid] @@ -1141,16 +1142,7 @@ def main(): # the source repo. We can reuse it on subsequent builds, if # there are any. if first: - if app.RepoType == 'srclib': - build_dir = os.path.join('build', 'srclib', app.Repo) - else: - build_dir = os.path.join('build', appid) - - # Set up vcs interface and make sure we have the latest code... - logging.debug("Getting {0} vcs interface for {1}" - .format(app.RepoType, app.Repo)) - vcs = common.getvcs(app.RepoType, app.Repo, build_dir) - + vcs, build_dir = common.setup_vcs(app) first = False logging.debug("Checking " + build.version) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index a023c57d..121f6edd 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -486,6 +486,27 @@ def getcvname(app): return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode) +def get_build_dir(app): + '''get the dir that this app will be built in''' + + if app.RepoType == 'srclib': + return os.path.join('build', 'srclib', app.Repo) + + return os.path.join('build', app.id) + + +def setup_vcs(app): + '''checkout code from VCS and return instance of vcs and the build dir''' + build_dir = get_build_dir(app) + + # Set up vcs interface and make sure we have the latest code... + logging.debug("Getting {0} vcs interface for {1}" + .format(app.RepoType, app.Repo)) + vcs = getvcs(app.RepoType, app.Repo, build_dir) + + return vcs, build_dir + + def getvcs(vcstype, remote, local): if vcstype == 'git': return vcs_git(remote, local) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index f8dc2b5c..986d240a 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -224,6 +224,21 @@ class App(): else: self.set_field(f, v) + def update(self, d): + '''Like dict.update()''' + for k, v in d.__dict__.items(): + if k == '_modified': + continue + elif k == 'builds': + for b in v: + build = Build() + del(b.__dict__['_modified']) + build.update_flags(b.__dict__) + self.builds.append(build) + elif v: + self.__dict__[k] = v + self._modified.add(k) + TYPE_UNKNOWN = 0 TYPE_OBSOLETE = 1 @@ -778,10 +793,12 @@ def read_srclibs(): srclibs[srclibname] = parse_srclib(metadatapath) -def read_metadata(xref=True): +def read_metadata(xref=True, check_vcs=[]): """ Read all metadata. Returns a list of 'app' objects (which are dictionaries as returned by the parse_txt_metadata function. + + check_vcs is the list of packageNames to check for .fdroid.yml in source """ # Always read the srclibs before the apps, since they can use a srlib as @@ -809,7 +826,7 @@ def read_metadata(xref=True): packageName, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath)) if packageName in apps: warn_or_exception("Found multiple metadata files for " + packageName) - app = parse_metadata(metadatapath) + app = parse_metadata(metadatapath, packageName in check_vcs) check_metadata(app) apps[app.id] = app @@ -962,7 +979,9 @@ def _decode_bool(s): warn_or_exception("Invalid bool '%s'" % s) -def parse_metadata(metadatapath): +def parse_metadata(metadatapath, check_vcs=False): + '''parse metadata file, optionally checking the git repo for metadata first''' + _, ext = fdroidserver.common.get_extension(metadatapath) accepted = fdroidserver.common.config['accepted_formats'] if ext not in accepted: @@ -971,7 +990,11 @@ def parse_metadata(metadatapath): app = App() app.metadatapath = metadatapath - app.id, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath)) + name, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath)) + if name == '.fdroid': + check_vcs = False + else: + app.id = name with open(metadatapath, 'r', encoding='utf-8') as mf: if ext == 'txt': @@ -985,7 +1008,28 @@ def parse_metadata(metadatapath): else: warn_or_exception('Unknown metadata format: %s' % metadatapath) + if check_vcs and app.Repo: + build_dir = fdroidserver.common.get_build_dir(app) + metadata_in_repo = os.path.join(build_dir, '.fdroid.yml') + if not os.path.isfile(metadata_in_repo): + vcs, build_dir = fdroidserver.common.setup_vcs(app) + vcs.gotorevision('HEAD') # HEAD since we can't know where else to go + if os.path.isfile(metadata_in_repo): + logging.debug('Including metadata from ' + metadata_in_repo) + app.update(parse_metadata(metadata_in_repo)) + post_metadata_parse(app) + + if not app.id: + if app.builds: + build = app.builds[-1] + if build.subdir: + root_dir = build.subdir + else: + root_dir = '.' + paths = fdroidserver.common.manifest_paths(root_dir, build.gradle) + _, _, app.id = fdroidserver.common.parse_androidmanifests(paths, app) + return app diff --git a/tests/import.TestCase b/tests/import.TestCase index 8eef5acb..3cfdb92e 100755 --- a/tests/import.TestCase +++ b/tests/import.TestCase @@ -25,6 +25,7 @@ class ImportTest(unittest.TestCase): '''fdroid import''' def test_import_gitlab(self): + os.chdir(os.path.dirname(__file__)) # FDroidPopen needs some config to work config = dict() fdroidserver.common.fill_config_defaults(config) diff --git a/tests/metadata/info.guardianproject.checkey.txt b/tests/metadata/info.guardianproject.checkey.txt new file mode 100644 index 00000000..43faaade --- /dev/null +++ b/tests/metadata/info.guardianproject.checkey.txt @@ -0,0 +1,21 @@ +Categories:Development,GuardianProject +License:GPLv3 +Web Site:https://dev.guardianproject.info/projects/checkey +Source Code:https://github.com/guardianproject/checkey +Issue Tracker:https://dev.guardianproject.info/projects/checkey/issues +Bitcoin:1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk + +Auto Name:Checkey +Summary:Info on local apps +Description: +Checkey is a utility for getting information about the APKs that are installed +on your device. Starting with a list of all of the apps that you have +installed on your device, it will show you the APK signature with a single +touch, and provides links to virustotal.com and androidobservatory.org to +easily access the profiles of that APK. It will also let you export the +signing certificate and generate ApkSignaturePin pin files for use with the +TrustedIntents library. +. + + +Current Version Code:9999999 diff --git a/tests/metadata/org.fdroid.ci.test.app.txt b/tests/metadata/org.fdroid.ci.test.app.txt new file mode 100644 index 00000000..466b7af6 --- /dev/null +++ b/tests/metadata/org.fdroid.ci.test.app.txt @@ -0,0 +1,2 @@ +Repo Type:git +Repo:https://gitlab.com/eighthave/ci-test-app diff --git a/tests/metadata/raw.template.txt b/tests/metadata/raw.template.txt new file mode 100644 index 00000000..ec2fe286 --- /dev/null +++ b/tests/metadata/raw.template.txt @@ -0,0 +1,9 @@ +License:Unknown +Web Site: +Source Code: +Issue Tracker: +Changelog: +Summary:Template +Description: +Template +. diff --git a/tests/run-tests b/tests/run-tests index a4474daa..354e6782 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -100,6 +100,8 @@ echo_header "test python getsig replacement" cd $WORKSPACE/tests/getsig ./make.sh + +cd $WORKSPACE/tests for testcase in $WORKSPACE/tests/*.TestCase; do $testcase done @@ -138,6 +140,19 @@ $fdroid readmeta $fdroid update +#------------------------------------------------------------------------------# +echo_header 'run `fdroid build` in fresh git checkout from import.TestCase' + +cd $WORKSPACE/tests/tmp/importer +if [ -d $ANDROID_HOME/platforms/android-23 ]; then + echo "build_tools = '`ls -1 $ANDROID_HOME/build-tools/ | sort -n | tail -1`'" > config.py + echo "force_build_tools = True" >> config.py + $fdroid build --verbose org.fdroid.ci.test.app:300 +else + echo 'WARNING: Skipping `fdroid build` test since android-23 is missing!' +fi + + #------------------------------------------------------------------------------# echo_header "copy tests/repo, generate java/gpg keys, update, and gpgsign"