3 # lint.py - part of the FDroid server tool
4 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU Affero General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See th
14 # GNU Affero General Public License for more details.
16 # You should have received a copy of the GNU Affero General Public Licen
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19 from argparse import ArgumentParser
26 from . import metadata
27 from . import rewritemeta
33 def enforce_https(domain):
34 return (re.compile(r'.*[^sS]://[^/]*' + re.escape(domain) + r'(/.*)?'),
35 domain + " URLs should always use https://")
39 enforce_https('github.com'),
40 enforce_https('gitlab.com'),
41 enforce_https('bitbucket.org'),
42 enforce_https('apache.org'),
43 enforce_https('google.com'),
44 enforce_https('svn.code.sf.net'),
48 def forbid_shortener(domain):
49 return (re.compile(r'https?://[^/]*' + re.escape(domain) + r'/.*'),
50 "URL shorteners should not be used")
53 http_url_shorteners = [
54 forbid_shortener('goo.gl'),
55 forbid_shortener('t.co'),
56 forbid_shortener('ur1.ca'),
57 forbid_shortener('is.gd'),
58 forbid_shortener('bit.ly'),
59 forbid_shortener('tiny.cc'),
60 forbid_shortener('tinyurl.com'),
63 http_checks = https_enforcings + http_url_shorteners + [
64 (re.compile(r'.*github\.com/[^/]+/[^/]+\.git'),
65 "Appending .git is not necessary"),
66 (re.compile(r'.*://[^/]*(github|gitlab|bitbucket|rawgit)[^/]*/([^/]+/){1,3}master'),
67 "Use /HEAD instead of /master to point at a file in the default branch"),
71 'WebSite': http_checks,
72 'SourceCode': http_checks,
73 'Repo': https_enforcings,
74 'IssueTracker': http_checks + [
75 (re.compile(r'.*github\.com/[^/]+/[^/]+/*$'),
76 "/issues is missing"),
77 (re.compile(r'.*gitlab\.com/[^/]+/[^/]+/*$'),
78 "/issues is missing"),
80 'Donate': http_checks + [
81 (re.compile(r'.*flattr\.com'),
82 "Flattr donation methods belong in the FlattrID flag"),
84 'Changelog': http_checks,
87 "Unnecessary leading space"),
88 (re.compile(r'.*\s$'),
89 "Unnecessary trailing space"),
92 (re.compile(r'.*\b(free software|open source)\b.*', re.IGNORECASE),
93 "No need to specify that the app is Free Software"),
94 (re.compile(r'.*((your|for).*android|android.*(app|device|client|port|version))', re.IGNORECASE),
95 "No need to specify that the app is for Android"),
96 (re.compile(r'.*[a-z0-9][.!?]( |$)'),
97 "Punctuation should be avoided"),
99 "Unnecessary leading space"),
100 (re.compile(r'.*\s$'),
101 "Unnecessary trailing space"),
104 (re.compile(r'\s*[*#][^ .]'),
105 "Invalid bulleted list"),
107 "Unnecessary leading space"),
108 (re.compile(r'.*\s$'),
109 "Unnecessary trailing space"),
110 (re.compile(r'.*([^[]|^)\[[^:[\]]+( |\]|$)'),
111 "Invalid link - use [http://foo.bar Link title] or [http://foo.bar]"),
112 (re.compile(r'(^|.* )https?://[^ ]+'),
113 "Unlinkified link - use [http://foo.bar Link title] or [http://foo.bar]"),
117 locale_pattern = re.compile(r'^[a-z]{2,3}(-[A-Z][A-Z])?$')
120 def check_regexes(app):
121 for f, checks in regex_checks.items():
124 t = metadata.fieldtype(f)
125 if t == metadata.TYPE_MULTILINE:
126 for l in v.splitlines():
128 yield "%s at line '%s': %s" % (f, l, r)
133 yield "%s '%s': %s" % (f, v, r)
136 def get_lastbuild(builds):
140 if not build.disable:
141 vercode = int(build.versionCode)
142 if lowest_vercode == -1 or vercode < lowest_vercode:
143 lowest_vercode = vercode
144 if not lastbuild or int(build.versionCode) > int(lastbuild.versionCode):
149 def check_ucm_tags(app):
150 lastbuild = get_lastbuild(app.builds)
151 if (lastbuild is not None
153 and app.UpdateCheckMode == 'RepoManifest'
154 and not lastbuild.commit.startswith('unknown')
155 and lastbuild.versionCode == app.CurrentVersionCode
156 and not lastbuild.forcevercode
157 and any(s in lastbuild.commit for s in '.,_-/')):
158 yield "Last used commit '%s' looks like a tag, but Update Check Mode is '%s'" % (
159 lastbuild.commit, app.UpdateCheckMode)
162 def check_char_limits(app):
163 limits = config['char_limits']
165 if len(app.Summary) > limits['summary']:
166 yield "Summary of length %s is over the %i char limit" % (
167 len(app.Summary), limits['summary'])
169 if len(app.Description) > limits['description']:
170 yield "Description of length %s is over the %i char limit" % (
171 len(app.Description), limits['description'])
174 def check_old_links(app):
184 if any(s in app.Repo for s in usual_sites):
185 for f in ['WebSite', 'SourceCode', 'IssueTracker', 'Changelog']:
187 if any(s in v for s in old_sites):
188 yield "App is in '%s' but has a link to '%s'" % (app.Repo, v)
191 def check_useless_fields(app):
192 if app.UpdateCheckName == app.id:
193 yield "Update Check Name is set to the known app id - it can be removed"
196 filling_ucms = re.compile(r'^(Tags.*|RepoManifest.*)')
199 def check_checkupdates_ran(app):
200 if filling_ucms.match(app.UpdateCheckMode):
201 if not app.AutoName and not app.CurrentVersion and app.CurrentVersionCode == '0':
202 yield "UCM is set but it looks like checkupdates hasn't been run yet"
205 def check_empty_fields(app):
206 if not app.Categories:
207 yield "Categories are not set"
210 all_categories = set([
221 "Science & Education",
231 def check_categories(app):
232 for categ in app.Categories:
233 if categ not in all_categories:
234 yield "Category '%s' is not valid" % categ
237 def check_duplicates(app):
238 if app.Name and app.Name == app.AutoName:
239 yield "Name '%s' is just the auto name - remove it" % app.Name
242 for f in ['Source Code', 'Web Site', 'Issue Tracker', 'Changelog']:
248 yield "Duplicate link in '%s': %s" % (f, v)
252 name = app.Name or app.AutoName
253 if app.Summary and name:
254 if app.Summary.lower() == name.lower():
255 yield "Summary '%s' is just the app's name" % app.Summary
257 if app.Summary and app.Description and len(app.Description) == 1:
258 if app.Summary.lower() == app.Description[0].lower():
259 yield "Description '%s' is just the app's summary" % app.Summary
262 for l in app.Description.splitlines():
266 yield "Description has a duplicate line"
270 desc_url = re.compile(r'(^|[^[])\[([^ ]+)( |\]|$)')
273 def check_mediawiki_links(app):
274 wholedesc = ' '.join(app.Description)
275 for um in desc_url.finditer(wholedesc):
277 for m, r in http_checks:
279 yield "URL '%s' in Description: %s" % (url, r)
282 def check_bulleted_lists(app):
283 validchars = ['*', '#']
286 for l in app.Description.splitlines():
291 if l[0] == lchar and l[1] == ' ':
293 if lcount > 2 and lchar not in validchars:
294 yield "Description has a list (%s) but it isn't bulleted (*) nor numbered (#)" % lchar
301 def check_builds(app):
302 supported_flags = set(metadata.build_flags)
303 # needed for YAML and JSON
304 for build in app.builds:
306 if build.disable.startswith('Generated by import.py'):
307 yield "Build generated by `fdroid import` - remove disable line once ready"
309 for s in ['master', 'origin', 'HEAD', 'default', 'trunk']:
310 if build.commit and build.commit.startswith(s):
311 yield "Branch '%s' used as commit in build '%s'" % (s, build.versionName)
312 for srclib in build.srclibs:
313 ref = srclib.split('@')[1].split('/')[0]
314 if ref.startswith(s):
315 yield "Branch '%s' used as commit in srclib '%s'" % (s, srclib)
316 for key in build.keys():
317 if key not in supported_flags:
318 yield key + ' is not an accepted build field'
321 def check_files_dir(app):
322 dir_path = os.path.join('metadata', app.id)
323 if not os.path.isdir(dir_path):
326 for name in os.listdir(dir_path):
327 path = os.path.join(dir_path, name)
328 if not (os.path.isfile(path) or name == 'signatures' or locale_pattern.match(name)):
329 yield "Found non-file at %s" % path
333 used = {'signatures', }
334 for build in app.builds:
335 for fname in build.patch:
336 if fname not in files:
337 yield "Unknown file %s in build '%s'" % (fname, build.versionName)
341 for name in files.difference(used):
342 if locale_pattern.match(name):
344 yield "Unused file at %s" % os.path.join(dir_path, name)
347 def check_format(app):
348 if options.format and not rewritemeta.proper_format(app):
349 yield "Run rewritemeta to fix formatting"
352 def check_license_tag(app):
353 '''Ensure all license tags are in https://spdx.org/license-list'''
354 if app.License.rstrip('+') not in SPDX:
355 yield 'Invalid license tag "%s"! Use only tags from https://spdx.org/license-list' \
359 def check_extlib_dir(apps):
360 dir_path = os.path.join('build', 'extlib')
362 for root, dirs, names in os.walk(dir_path):
364 files.add(os.path.join(root, name)[len(dir_path) + 1:])
368 for build in app.builds:
369 for path in build.extlibs:
370 if path not in files:
371 yield "%s: Unknown extlib %s in build '%s'" % (app.id, path, build.versionName)
375 for path in files.difference(used):
376 if any(path.endswith(s) for s in [
378 'source.txt', 'origin.txt', 'md5.txt',
379 'LICENSE', 'LICENSE.txt',
380 'COPYING', 'COPYING.txt',
381 'NOTICE', 'NOTICE.txt',
384 yield "Unused extlib at %s" % os.path.join(dir_path, path)
387 def check_for_unsupported_metadata_files(basedir=""):
388 """Checks whether any non-metadata files are in metadata/"""
393 formats = config['accepted_formats']
394 for f in glob.glob(basedir + 'metadata/*') + glob.glob(basedir + 'metadata/.*'):
398 exists = exists or os.path.exists(f + '.' + t)
400 print('"' + f + '/" has no matching metadata file!')
402 elif not os.path.splitext(f)[1][1:] in formats:
403 print('"' + f.replace(basedir, '')
404 + '" is not a supported file format: (' + ','.join(formats) + ')')
412 global config, options
414 # Parse command line...
415 parser = ArgumentParser(usage="%(prog)s [options] [APPID [APPID ...]]")
416 common.setup_global_opts(parser)
417 parser.add_argument("-f", "--format", action="store_true", default=False,
418 help="Also warn about formatting issues, like rewritemeta -l")
419 parser.add_argument("appid", nargs='*', help="app-id in the form APPID")
420 metadata.add_metadata_arguments(parser)
421 options = parser.parse_args()
422 metadata.warnings_action = options.W
424 config = common.read_config(options)
427 allapps = metadata.read_metadata(xref=True)
428 apps = common.read_app_args(options.appid, allapps, False)
430 anywarns = check_for_unsupported_metadata_files()
432 apps_check_funcs = []
433 if len(options.appid) == 0:
434 # otherwise it finds tons of unused extlibs
435 apps_check_funcs.append(check_extlib_dir)
436 for check_func in apps_check_funcs:
437 for warn in check_func(apps.values()):
441 for appid, app in apps.items():
450 check_checkupdates_ran,
451 check_useless_fields,
455 check_mediawiki_links,
456 check_bulleted_lists,
463 for check_func in app_check_funcs:
464 for warn in check_func(app):
466 print("%s: %s" % (appid, warn))
472 # A compiled, public domain list of official SPDX license tags from:
473 # https://github.com/sindresorhus/spdx-license-list/blob/v3.0.1/spdx-simple.json
474 # The deprecated license tags have been removed from the list, they are at the
475 # bottom, starting after the last license tags that start with Z.
476 # This is at the bottom, since its a long list of data
478 "PublicDomain", # an F-Droid addition, until we can enforce a better option
518 "BSD-2-Clause-FreeBSD",
519 "BSD-2-Clause-NetBSD",
521 "BSD-3-Clause-Clear",
522 "BSD-3-Clause-No-Nuclear-License",
523 "BSD-3-Clause-No-Nuclear-License-2014",
524 "BSD-3-Clause-No-Nuclear-Warranty",
528 "BSD-3-Clause-Attribution",
544 "CNRI-Python-GPL-Compatible",
678 "MPL-2.0-no-copyleft-exception",
802 "zlib-acknowledgement",
808 if __name__ == "__main__":