From 30b2f5a48ac14932dc212417a075fc0d4a388d5f Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 29 Nov 2017 22:03:26 +0100 Subject: [PATCH] build: sort `fdroid build --all` by most recently changed first --- fdroidserver/build.py | 2 +- fdroidserver/metadata.py | 48 ++++++++++++++++++++++++++-------------- tests/metadata.TestCase | 33 +++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 17 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 16901e78..17d5cfd2 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -1111,7 +1111,7 @@ def main(): # Read all app and srclib metadata pkgs = common.read_pkg_args(options.appid, True) - allapps = metadata.read_metadata(not options.onserver, pkgs) + allapps = metadata.read_metadata(not options.onserver, pkgs, sort_by_time=True) apps = common.read_app_args(options.appid, allapps, True) for appid, app in list(apps.items()): diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index d65eb81a..c29c1e4c 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -26,6 +26,7 @@ import logging import textwrap import io import yaml +from collections import OrderedDict # use libyaml if it is available try: from yaml import CLoader @@ -703,35 +704,50 @@ def read_srclibs(): srclibs[srclibname] = parse_srclib(metadatapath) -def read_metadata(xref=True, check_vcs=[]): - """ - Read all metadata. Returns a list of 'app' objects (which are dictionaries as - returned by the parse_txt_metadata function. +def read_metadata(xref=True, check_vcs=[], sort_by_time=False): + """Return a list of App instances sorted newest first + + This reads all of the metadata files in a 'data' repository, then + builds a list of App instances from those files. The list is + sorted based on creation time, newest first. Most of the time, + the newer files are the most interesting. + + If there are multiple metadata files for a single appid, then the first + file that is parsed wins over all the others, and the rest throw an + exception. So the original .txt format is parsed first, at least until + newer formats stabilize. check_vcs is the list of packageNames to check for .fdroid.yml in source + """ # Always read the srclibs before the apps, since they can use a srlib as # their source repository. read_srclibs() - apps = {} + apps = OrderedDict() for basedir in ('metadata', 'tmp'): if not os.path.exists(basedir): os.makedirs(basedir) - # If there are multiple metadata files for a single appid, then the first - # file that is parsed wins over all the others, and the rest throw an - # exception. So the original .txt format is parsed first, at least until - # newer formats stabilize. - - for metadatapath in sorted(glob.glob(os.path.join('metadata', '*.txt')) - + glob.glob(os.path.join('metadata', '*.json')) - + glob.glob(os.path.join('metadata', '*.yml')) - + glob.glob('.fdroid.txt') - + glob.glob('.fdroid.json') - + glob.glob('.fdroid.yml')): + metadatafiles = (glob.glob(os.path.join('metadata', '*.txt')) + + glob.glob(os.path.join('metadata', '*.json')) + + glob.glob(os.path.join('metadata', '*.yml')) + + glob.glob('.fdroid.txt') + + glob.glob('.fdroid.json') + + glob.glob('.fdroid.yml')) + + if sort_by_time: + entries = ((os.stat(path).st_mtime, path) for path in metadatafiles) + metadatafiles = [] + for _ignored, path in sorted(entries, reverse=True): + metadatafiles.append(path) + else: + # most things want the index alpha sorted for stability + metadatafiles = sorted(metadatafiles) + + for metadatapath in metadatafiles: packageName, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath)) if packageName in apps: warn_or_exception(_("Found multiple metadata files for {appid}") diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index 306f1c46..a9fdd129 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -2,9 +2,12 @@ # http://www.drdobbs.com/testing/unit-testing-with-python/240165163 +import glob import inspect import optparse import os +import random +import shutil import sys import unittest import yaml @@ -120,6 +123,36 @@ class MetadataTest(unittest.TestCase): self.maxDiff = None self.assertEqual(result.read(), orig.read()) + def test_read_metadata_sort_by_time(self): + # setup/reset test dir if necessary and setup params + testbasedir = os.path.dirname(__file__) + tmpdir = os.path.join(testbasedir, '..', '.testfiles') + if not os.path.exists(tmpdir): + os.makedirs(tmpdir) + testdir = tempfile.mkdtemp(prefix='test_read_metadata_sort_by_time_', dir=tmpdir) + metadatadir = os.path.join(testdir, 'metadata') + os.makedirs(metadatadir) + fdroidserver.common.config = {'accepted_formats': ['txt']} + + randomlist = [] + randomapps = glob.glob(os.path.join(testbasedir, 'metadata', '*.txt')) + random.shuffle(randomapps) + i = 1 + for f in randomapps: + shutil.copy(f, metadatadir) + new = os.path.join(metadatadir, os.path.basename(f)) + stat = os.stat(new) + os.utime(new, (stat.st_ctime, stat.st_mtime + i)) + # prepend new item so newest is always first + randomlist = [os.path.basename(f)[:-4]] + randomlist + i += 1 + os.chdir(testdir) + allapps = fdroidserver.metadata.read_metadata(xref=True, sort_by_time=True) + allappids = [] + for appid, app in allapps.items(): + allappids.append(appid) + self.assertEqual(randomlist, allappids) + if __name__ == "__main__": parser = optparse.OptionParser() -- 2.30.2