From: Hans-Christoph Steiner Date: Thu, 23 Jul 2015 22:09:57 +0000 (-0700) Subject: consolidate all post-parse metadata checks X-Git-Tag: 0.5.0~140^2~9 X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=commitdiff_plain;h=0425788b7d760d29fb3e39097a9ca5a4d2b7987f;p=fdroidserver.git consolidate all post-parse metadata checks This puts all of the needed post parsing checks on the metadata into a single method that is used by all parsing methods (.txt, JSON, XML, YAML). This provides the single place to normalize the internal representation of the metadata. It would be good to also change the internal representation to use more Python bool/int types so that less post parsing is needed for JSON, XML, and YAML. The SMSSecure test .pickle was changed to account for the use of lstrip() and rstrip() on all 'script' types. This also changes the example JSON to use ints for versionCodes --- diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index fa862876..10976871 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -585,14 +585,58 @@ def get_default_app_info_list(): def post_metadata_parse(thisinfo): + supported_metadata = app_defaults.keys() + ['comments', 'builds', 'id'] + for k, v in thisinfo.iteritems(): + if k not in supported_metadata: + raise MetaDataException("Unrecognised metadata: {0}: {1}" + .format(k, v)) + if type(v) in (float, int): + thisinfo[k] = str(v) + + # convert to the odd internal format + for k in ('Description', 'Maintainer Notes'): + if isinstance(thisinfo[k], basestring): + text = thisinfo[k].rstrip().lstrip() + thisinfo[k] = text.split('\n') + + supported_flags = (flag_defaults.keys() + + ['vercode', 'version', 'versionCode', 'versionName']) + esc_newlines = re.compile('\\\\( |\\n)') + for build in thisinfo['builds']: - for k, v in build.iteritems(): + for k, v in build.items(): + if k not in supported_flags: + raise MetaDataException("Unrecognised build flag: {0}={1}" + .format(k, v)) + if k == 'versionCode': build['vercode'] = str(v) del build['versionCode'] elif k == 'versionName': build['version'] = str(v) del build['versionName'] + elif type(v) in (float, int): + build[k] = str(v) + else: + keyflagtype = flagtype(k) + if keyflagtype == 'list': + # these can be bools, strings or lists, but ultimately are lists + if isinstance(v, basestring): + build[k] = [v] + elif isinstance(v, bool): + if v: + build[k] = ['yes'] + else: + build[k] = ['no'] + elif keyflagtype == 'script': + build[k] = re.sub(esc_newlines, '', v).lstrip().rstrip() + elif keyflagtype == 'bool': + # TODO handle this using > project.properties && echo "android.library.reference.4=android-libs/HtmlSpanner/htmlspanner" >> project.properties && find . -type f -print0 | xargs -0 sed -i \'s/org.rootcommands/org.sufficientlysecure.rootcommands/g\' && cp android-libs/Donations/ant-templates/other/DonationsConfig.java android-libs/Donations/src/org/donations/' -p339 -sg121 +p336 +sg119 I00 -sg122 -Nsa(dp340 -g74 +sS'commit' +p337 +S'v2.1' +p338 +sa(dp339 +g73 I00 -sg75 +sg74 S'42' -p341 +p340 +sg76 +I00 sg77 I00 sg78 -I00 -sg79 -I00 +g79 sg80 -I00 -sg81 -g82 +g81 sS'srclibs' -p342 -(lp343 +p341 +(lp342 S'RootCommands@c940b0e503' -p344 -asg85 +p343 +asg84 +Nsg85 NsS'extlibs' -p345 -(lp346 +p344 +(lp345 S'htmlcleaner/htmlcleaner-2.2.jar' -p347 +p346 asS'init' -p348 +p347 S'rm android-libs/Donations/custom_rules.xml && git clone https://github.com/dschuermann/HtmlSpanner android-libs/HtmlSpanner' -p349 +p348 sg89 S'2.3' -p350 +p349 sS'subdir' -p351 +p350 S'AdAway' -p352 +p351 sg93 -g94 -sg95 -g96 -sg97 g27 -sg98 +sg94 +I00 +sg95 g27 -sg99 +sg96 I00 -sg100 -g101 +sg97 +g98 sS'update' -p353 -(lp354 -g330 +p352 +(lp353 +g329 aS'android-libs/Donations' -p355 +p354 aS'android-libs/ActionBarSherlock' -p356 +p355 aS'android-libs/HtmlSpanner/htmlspanner' -p357 -asg105 +p356 +asg102 I00 -sg106 +sg103 I00 -sS'commit' -p358 -S'v2.3' -p359 -sg109 -g110 +sg104 +g105 +sg106 +g107 sS'buildjni' -p360 -(lp361 -g113 -asg114 +p357 +(lp358 +g110 +asg111 +g112 +sg113 +Nsg114 g115 sg116 Nsg117 -Nsg118 -g119 +I00 sS'prebuild' -p362 +p359 S'rm -rf ../update_zip libs/root-commands-1.2.jar libs/htmlspanner-0.2-fork.jar && cp -f libs/htmlcleaner-2.2.jar android-libs/HtmlSpanner/htmlspanner/libs/ && echo "android.library.reference.3=$$RootCommands$$" >> project.properties && echo "android.library.reference.4=android-libs/HtmlSpanner/htmlspanner" >> project.properties && find . -type f -print0 | xargs -0 sed -i \'s/org.rootcommands/org.sufficientlysecure.rootcommands/g\' && cp android-libs/Donations/ant-templates/other/DonationsConfig.java android-libs/Donations/src/org/donations/' -p363 -sg121 +p360 +sg119 I00 -sg122 -Nsa(dp364 -g74 +sS'commit' +p361 +S'v2.3' +p362 +sa(dp363 +g73 I00 -sg75 +sg74 S'45' -p365 +p364 +sg76 +I00 sg77 I00 sg78 -I00 -sg79 -I00 -sg118 -g119 -sg81 -g82 -sg83 -g84 -sg85 +g79 +sS'gradle' +p365 +(lp366 +g110 +asg82 +g83 +sg84 +Nsg85 Nsg86 g87 sg88 g27 sg89 S'2.6' -p366 -sS'subdir' p367 -S'AdAway' +sS'subdir' p368 +S'AdAway' +p369 sg93 -g94 -sg95 -g80 -sg97 g27 -sg98 +sg94 +I00 +sg95 g27 -sg99 +sg96 I00 sS'preassemble' -p369 -(lp370 +p370 +(lp371 S'renameExecutables' -p371 -asg102 -g103 -sg105 +p372 +asg99 +g100 +sg102 I00 -sg106 +sg103 I00 -sS'commit' -p372 -S'v2.6' +sg104 +g105 +sg106 +g107 +sS'buildjni' p373 -sg109 +(lp374 g110 -sS'buildjni' -p374 -(lp375 -g113 -asg114 -g115 +asg111 +g112 +sg113 +Nsg114 +g117 sg116 -Nsg117 -NsS'gradle' -p376 -(lp377 -g113 -asg120 +Nsg80 +g81 +sg118 g27 -sg121 +sg119 I00 -sg122 -Nsa(dp378 -g74 +sS'commit' +p375 +S'v2.6' +p376 +sa(dp377 +g73 I00 -sg75 +sg74 S'46' -p379 +p378 +sg76 +I00 sg77 I00 sg78 -I00 -sg79 -I00 -sg118 -g119 -sg81 -g82 -sg83 -g84 -sg85 +g79 +sS'gradle' +p379 +(lp380 +g110 +asg82 +g83 +sg84 +Nsg85 Nsg86 g87 sg88 g27 sg89 S'2.7' -p380 -sS'subdir' p381 -S'AdAway' +sS'subdir' p382 +S'AdAway' +p383 sg93 -g94 -sg95 -g80 -sg97 g27 -sg98 +sg94 +I00 +sg95 g27 -sg99 +sg96 I00 sS'preassemble' -p383 -(lp384 +p384 +(lp385 S'renameExecutables' -p385 -asg102 -g103 -sg105 +p386 +asg99 +g100 +sg102 I00 -sg106 +sg103 I00 -sS'commit' -p386 -S'v2.7' +sg104 +g105 +sg106 +g107 +sS'buildjni' p387 -sg109 +(lp388 g110 -sS'buildjni' -p388 -(lp389 -g113 -asg114 -g115 +asg111 +g112 +sg113 +Nsg114 +g117 sg116 -Nsg117 -NsS'gradle' -p390 -(lp391 -g113 -asg120 +Nsg80 +g81 +sg118 g27 -sg121 +sg119 I00 -sg122 -Nsa(dp392 -g74 +sS'commit' +p389 +S'v2.7' +p390 +sa(dp391 +g73 I00 -sg75 +sg74 S'47' -p393 +p392 +sg76 +I00 sg77 I00 sg78 -I00 -sg79 -I00 -sg118 -g119 -sg81 -g82 -sg83 -g84 -sg85 +g79 +sS'gradle' +p393 +(lp394 +g110 +asg82 +g83 +sg84 +Nsg85 Nsg86 g87 sg88 g27 sg89 S'2.8' -p394 -sS'subdir' p395 -S'AdAway' +sS'subdir' p396 +S'AdAway' +p397 sg93 -g94 -sg95 -g80 -sg97 g27 -sg98 +sg94 +I00 +sg95 g27 -sg99 +sg96 I00 sS'preassemble' -p397 -(lp398 +p398 +(lp399 S'renameExecutables' -p399 -asg102 -g103 -sg105 +p400 +asg99 +g100 +sg102 I00 -sg106 +sg103 I00 -sS'commit' -p400 -S'v2.8' +sg104 +g105 +sg106 +g107 +sS'buildjni' p401 -sg109 +(lp402 g110 -sS'buildjni' -p402 -(lp403 -g113 -asg114 -g115 +asg111 +g112 +sg113 +Nsg114 +g117 sg116 -Nsg117 -NsS'gradle' -p404 -(lp405 -g113 -asg120 +Nsg80 +g81 +sg118 g27 -sg121 +sg119 I00 -sg122 -Nsa(dp406 -g74 +sS'commit' +p403 +S'v2.8' +p404 +sa(dp405 +g73 I00 -sg75 +sg74 S'48' -p407 +p406 +sg76 +I00 sg77 I00 sg78 -I00 -sg79 -I00 -sg118 -g119 -sg81 -g82 -sg83 -g84 -sg85 +g79 +sS'gradle' +p407 +(lp408 +g110 +asg82 +g83 +sg84 +Nsg85 Nsg86 g87 sg88 g27 sg89 S'2.8.1' -p408 -sS'subdir' p409 -S'AdAway' +sS'subdir' p410 +S'AdAway' +p411 sg93 -g94 -sg95 -g80 -sg97 g27 -sg98 +sg94 +I00 +sg95 g27 -sg99 +sg96 I00 sS'preassemble' -p411 -(lp412 +p412 +(lp413 S'renameExecutables' -p413 -asg102 -g103 -sg105 +p414 +asg99 +g100 +sg102 I00 -sg106 +sg103 I00 -sS'commit' -p414 -S'v2.8.1' +sg104 +g105 +sg106 +g107 +sS'buildjni' p415 -sg109 +(lp416 g110 -sS'buildjni' -p416 -(lp417 -g113 -asg114 -g115 +asg111 +g112 +sg113 +Nsg114 +g117 sg116 -Nsg117 -NsS'gradle' -p418 -(lp419 -g113 -asg120 +Nsg80 +g81 +sg118 g27 -sg121 +sg119 I00 -sg122 -Nsa(dp420 -g74 +sS'commit' +p417 +S'v2.8.1' +p418 +sa(dp419 +g73 I00 -sg75 +sg74 S'49' -p421 +p420 +sg76 +I00 sg77 I00 sg78 -I00 -sg79 -I00 -sg118 -g119 -sg81 -g82 -sg83 -g84 -sg85 +g79 +sS'gradle' +p421 +(lp422 +g110 +asg82 +g83 +sg84 +Nsg85 Nsg86 g87 sg88 g27 sg89 S'2.9' -p422 -sS'subdir' p423 -S'AdAway' +sS'subdir' p424 +S'AdAway' +p425 sg93 -g94 -sg95 -g80 -sg97 g27 -sg98 +sg94 +I00 +sg95 g27 -sg99 +sg96 I00 sS'preassemble' -p425 -(lp426 +p426 +(lp427 S'renameExecutables' -p427 -asg102 -g103 -sg105 +p428 +asg99 +g100 +sg102 I00 -sg106 +sg103 I00 -sS'commit' -p428 -S'v2.9' +sg104 +g105 +sg106 +g107 +sS'buildjni' p429 -sg109 +(lp430 g110 -sS'buildjni' -p430 -(lp431 -g113 -asg114 -g115 +asg111 +g112 +sg113 +Nsg114 +g117 sg116 -Nsg117 -NsS'gradle' -p432 -(lp433 -g113 -asg120 +Nsg80 +g81 +sg118 g27 -sg121 +sg119 I00 -sg122 -Nsa(dp434 -g74 +sS'commit' +p431 +S'v2.9' +p432 +sa(dp433 +g73 I00 -sg75 +sg74 S'50' -p435 +p434 +sg76 +I00 sg77 I00 sg78 -I00 -sg79 -I00 -sg118 -g119 -sg81 -g82 -sg83 -g84 -sg85 +g79 +sS'gradle' +p435 +(lp436 +g110 +asg82 +g83 +sg84 +Nsg85 Nsg86 g87 sg88 g27 sg89 S'2.9.1' -p436 -sS'subdir' p437 -S'AdAway' +sS'subdir' p438 +S'AdAway' +p439 sg93 -g94 -sg95 -g80 -sg97 g27 -sg98 +sg94 +I00 +sg95 g27 -sg99 +sg96 I00 sS'preassemble' -p439 -(lp440 +p440 +(lp441 S'renameExecutables' -p441 -asg102 -g103 -sg105 +p442 +asg99 +g100 +sg102 I00 -sg106 +sg103 I00 -sS'commit' -p442 -S'v2.9.1' +sg104 +g105 +sg106 +g107 +sS'buildjni' p443 -sg109 +(lp444 g110 -sS'buildjni' -p444 -(lp445 -g113 -asg114 -g115 +asg111 +g112 +sg113 +Nsg114 +g117 sg116 -Nsg117 -NsS'gradle' -p446 -(lp447 -g113 -asg120 +Nsg80 +g81 +sg118 g27 -sg121 +sg119 I00 -sg122 -Nsa(dp448 -g74 +sS'commit' +p445 +S'v2.9.1' +p446 +sa(dp447 +g73 I00 -sg75 +sg74 S'51' -p449 +p448 +sg76 +I00 sg77 I00 sg78 -I00 -sg79 -I00 -sg118 -g119 -sg81 -g82 -sg83 -g84 -sg85 +g79 +sS'gradle' +p449 +(lp450 +g110 +asg82 +g83 +sg84 +Nsg85 Nsg86 g87 sg88 g27 sg89 S'2.9.2' -p450 -sS'subdir' p451 -S'AdAway' +sS'subdir' p452 +S'AdAway' +p453 sg93 -g94 -sg95 -g80 -sg97 g27 -sg98 +sg94 +I00 +sg95 g27 -sg99 +sg96 I00 sS'preassemble' -p453 -(lp454 +p454 +(lp455 S'renameExecutables' -p455 -asg102 -g103 -sg105 +p456 +asg99 +g100 +sg102 I00 -sg106 +sg103 I00 -sS'commit' -p456 -S'v2.9.2' +sg104 +g105 +sg106 +g107 +sS'buildjni' p457 -sg109 +(lp458 g110 -sS'buildjni' -p458 -(lp459 -g113 -asg114 -g115 +asg111 +g112 +sg113 +Nsg114 +g117 sg116 -Nsg117 -NsS'gradle' -p460 -(lp461 -g113 -asg120 +Nsg80 +g81 +sg118 g27 -sg121 +sg119 I00 -sg122 -Nsa(dp462 -g74 +sS'commit' +p459 +S'v2.9.2' +p460 +sa(dp461 +g73 I00 -sg75 +sg74 S'52' -p463 +p462 +sg76 +I00 sg77 I00 sg78 -I00 -sg79 -I00 -sg118 -g119 -sg81 -g82 -sg83 -g84 -sg85 +g79 +sS'gradle' +p463 +(lp464 +g110 +asg82 +g83 +sg84 +Nsg85 Nsg86 g87 sg88 g27 sg89 S'3.0' -p464 -sS'subdir' p465 -S'AdAway' +sS'subdir' p466 +S'AdAway' +p467 sg93 -g94 -sg95 -g80 -sg97 g27 -sg98 +sg94 +I00 +sg95 g27 -sg99 +sg96 I00 sS'preassemble' -p467 -(lp468 +p468 +(lp469 S'renameExecutables' -p469 -asg102 -g103 -sg105 +p470 +asg99 +g100 +sg102 I00 -sg106 +sg103 I00 -sS'commit' -p470 -S'v3.0' +sg104 +g105 +sg106 +g107 +sS'buildjni' p471 -sg109 +(lp472 g110 -sS'buildjni' -p472 -(lp473 -g113 -asg114 -g115 +asg111 +g112 +sg113 +Nsg114 +g117 sg116 -Nsg117 -NsS'gradle' -p474 -(lp475 -g113 -asg120 +Nsg80 +g81 +sg118 g27 -sg121 +sg119 I00 -sg122 -NsasS'FlattrID' -p476 +sS'commit' +p473 +S'v3.0' +p474 +sasS'FlattrID' +p475 S'369138' -p477 +p476 sS'Disabled' -p478 +p477 NsS'Update Check Name' -p479 +p478 NsS'Vercode Operation' -p480 +p479 NsS'Current Version' -p481 +p480 S'3.0' -p482 -s. +p481 +s. \ No newline at end of file diff --git a/tests/metadata/org.smssecure.smssecure.pickle b/tests/metadata/org.smssecure.smssecure.pickle index 8885d55e..860acaf7 100644 --- a/tests/metadata/org.smssecure.smssecure.pickle +++ b/tests/metadata/org.smssecure.smssecure.pickle @@ -218,7 +218,7 @@ S'yes' p111 asS'prebuild' p112 -S"touch signing.properties && pushd $$GradleWitness$$ && gradle jar && popd && cp $$GradleWitness$$/build/libs/GradleWitness.jar libs/gradle-witness.jar && sed -i -e '20,22d' build.gradle && pushd $$PreferenceFragment$$ && gradle uploadArchives && popd && sed -i -e '/5470f5872514a6226fa1fc6f4e000991f38805691c534cf0bd2778911fc773ad/d' build.gradle && mkdir smil && pushd smil && wget -c http://www.w3.org/TR/smil-boston-dom/java-binding.zip && unzip java-binding.zip && popd && cp -fR smil/java/org src/ && rm -fR smil && sed -i -e '/org.w3c.smil/d' build.gradle && cp -fR $$AospMms$$/src/org src/ " +S"touch signing.properties && pushd $$GradleWitness$$ && gradle jar && popd && cp $$GradleWitness$$/build/libs/GradleWitness.jar libs/gradle-witness.jar && sed -i -e '20,22d' build.gradle && pushd $$PreferenceFragment$$ && gradle uploadArchives && popd && sed -i -e '/5470f5872514a6226fa1fc6f4e000991f38805691c534cf0bd2778911fc773ad/d' build.gradle && mkdir smil && pushd smil && wget -c http://www.w3.org/TR/smil-boston-dom/java-binding.zip && unzip java-binding.zip && popd && cp -fR smil/java/org src/ && rm -fR smil && sed -i -e '/org.w3c.smil/d' build.gradle && cp -fR $$AospMms$$/src/org src/" p113 sS'novcheck' p114