From: Daniel Martí Date: Sun, 28 Feb 2016 13:37:17 +0000 (+0000) Subject: Merge branch 'p1' into 'master' X-Git-Tag: 0.7.0~98 X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=commitdiff_plain;h=bf3247743344edf676a51c568cc35f6f0bdc95e5;hp=-c;p=fdroidserver.git Merge branch 'p1' into 'master' Fix pubkey extraction on update Replacement of !86. Fix pubkey extraction in case of non-empty _JAVA_OPTIONS. Fixes #133. I didn't actually run the test suite (it looks like there are some preparations to be done for that), but I checked the commands from `test_fdroid_popen_stderr_redirect` in ipython. See merge request !103 --- bf3247743344edf676a51c568cc35f6f0bdc95e5 diff --combined fdroidserver/common.py index 36c588d0,5ea9e549..0b4c2dd0 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@@ -139,25 -139,22 +139,25 @@@ def fill_config_defaults(thisconfig) continue j = os.path.basename(d) # the last one found will be the canonical one, so order appropriately - for regex in (r'1\.([6-9])\.0\.jdk', # OSX - r'jdk1\.([6-9])\.0_[0-9]+.jdk', # OSX and Oracle tarball - r'jdk([6-9])-openjdk', # Arch - r'java-([6-9])-openjdk', # Arch - r'java-([6-9])-jdk', # Arch (oracle) - r'java-1\.([6-9])\.0-.*', # RedHat - r'java-([6-9])-oracle', # Debian WebUpd8 - r'jdk-([6-9])-oracle-.*', # Debian make-jpkg - r'java-([6-9])-openjdk-[^c][^o][^m].*'): # Debian + for regex in [ + r'^1\.([6-9])\.0\.jdk$', # OSX + r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball + r'^jdk([6-9])-openjdk$', # Arch + r'^java-([6-9])-openjdk$', # Arch + r'^java-([6-9])-jdk$', # Arch (oracle) + r'^java-1\.([6-9])\.0-.*$', # RedHat + r'^java-([6-9])-oracle$', # Debian WebUpd8 + r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg + r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian + ]: m = re.match(regex, j) - if m: - osxhome = os.path.join(d, 'Contents', 'Home') - if os.path.exists(osxhome): - thisconfig['java_paths'][m.group(1)] = osxhome - else: - thisconfig['java_paths'][m.group(1)] = d + if not m: + continue + osxhome = os.path.join(d, 'Contents', 'Home') + if os.path.exists(osxhome): + thisconfig['java_paths'][m.group(1)] = osxhome + else: + thisconfig['java_paths'][m.group(1)] = d for java_version in ('7', '8', '9'): if java_version not in thisconfig['java_paths']: @@@ -1626,7 -1623,7 +1626,7 @@@ def SdkToolsPopen(commands, cwd=None, o cwd=cwd, output=output) - def FDroidPopen(commands, cwd=None, output=True): + def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True): """ Run a command and capture the possibly huge output. @@@ -1642,15 -1639,28 +1642,28 @@@ logging.debug("Directory: %s" % cwd) logging.debug("> %s" % ' '.join(commands)) + stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE result = PopenResult() p = None try: p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + stdout=subprocess.PIPE, stderr=stderr_param) except OSError as e: raise BuildException("OSError while trying to execute " + ' '.join(commands) + ': ' + str(e)) + if not stderr_to_stdout and options.verbose: + stderr_queue = Queue() + stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue) + + while not stderr_reader.eof(): + while not stderr_queue.empty(): + line = stderr_queue.get() + sys.stderr.write(line) + sys.stderr.flush() + + time.sleep(0.1) + stdout_queue = Queue() stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue) diff --combined fdroidserver/update.py index 67de608d,9de64717..38a61c8b --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@@ -405,7 -405,7 +405,7 @@@ def getsig(apkpath) return md5(cert_encoded.encode('hex')).hexdigest() -def scan_apks(apps, apkcache, repodir, knownapks): +def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False): """Scan the apks in the given repo directory. This also extracts the icons. @@@ -414,8 -414,6 +414,8 @@@ :param apkcache: current apk cache information :param repodir: repo directory to scan :param knownapks: known apks info + :param use_date_from_apk: use date from APK (instead of current date) + for newly added APKs :returns: (apks, cachechanged) where apks is a list of apk information, and cachechanged is True if the apkcache got changed. """ @@@ -570,16 -568,12 +570,16 @@@ # has to be more than 24 hours newer because ZIP/APK files do not # store timezone info manifest = apkzip.getinfo('AndroidManifest.xml') - dt_obj = datetime(*manifest.date_time) - checkdt = dt_obj - timedelta(1) - if datetime.today() < checkdt: - logging.warn('System clock is older than manifest in: ' - + apkfilename + '\nSet clock to that time using:\n' - + 'sudo date -s "' + str(dt_obj) + '"') + if manifest.date_time[1] == 0: # month can't be zero + logging.debug('AndroidManifest.xml has no date') + else: + dt_obj = datetime(*manifest.date_time) + checkdt = dt_obj - timedelta(1) + if datetime.today() < checkdt: + logging.warn('System clock is older than manifest in: ' + + apkfilename + + '\nSet clock to that time using:\n' + + 'sudo date -s "' + str(dt_obj) + '"') iconfilename = "%s.%s.png" % ( apk['id'], @@@ -691,10 -685,6 +691,10 @@@ # Record in known apks, getting the added date at the same time.. added = knownapks.recordapk(apk['apkname'], apk['id']) if added: + if use_date_from_apk and manifest.date_time[1] != 0: + added = datetime(*manifest.date_time).timetuple() + logging.debug("Using date from APK") + apk['added'] = added apkcache[apkfilename] = apk @@@ -726,7 -716,8 +726,8 @@@ def extract_pubkey() '-alias', config['repo_keyalias'], '-keystore', config['keystore'], '-storepass:file', config['keystorepassfile']] - + config['smartcardoptions'], output=False) + + config['smartcardoptions'], + output=False, stderr_to_stdout=False) if p.returncode != 0 or len(p.output) < 20: msg = "Failed to get repo pubkey!" if config['keystore'] == 'NONE': @@@ -1140,8 -1131,6 +1141,8 @@@ def main() help="Clean update - don't uses caches, reprocess all apks") parser.add_argument("--nosign", action="store_true", default=False, help="When configured for signed indexes, create only unsigned indexes at this stage") + parser.add_argument("--use-date-from-apk", action="store_true", default=False, + help="Use date from apk instead of current time for newly added apks") options = parser.parse_args() config = common.read_config(options) @@@ -1216,7 -1205,7 +1217,7 @@@ delete_disabled_builds(apps, apkcache, repodirs) # Scan all apks in the main repo - apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks) + apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk) # Generate warnings for apk's with no metadata (or create skeleton # metadata files, if requested on the command line) @@@ -1258,7 -1247,7 +1259,7 @@@ # Scan the archive repo for apks as well if len(repodirs) > 1: - archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks) + archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk) if cc: cachechanged = True else: