3 # lint.py - part of the FDroid server tool
4 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU Affero General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See th
14 # GNU Affero General Public License for more details.
16 # You should have received a copy of the GNU Affero General Public Licen
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19 from argparse import ArgumentParser
27 from . import metadata
28 from . import rewritemeta
34 def enforce_https(domain):
35 return (re.compile(r'.*[^sS]://[^/]*' + re.escape(domain) + r'(/.*)?'),
36 domain + " URLs should always use https://")
40 enforce_https('github.com'),
41 enforce_https('gitlab.com'),
42 enforce_https('bitbucket.org'),
43 enforce_https('apache.org'),
44 enforce_https('google.com'),
45 enforce_https('svn.code.sf.net'),
49 def forbid_shortener(domain):
50 return (re.compile(r'https?://[^/]*' + re.escape(domain) + r'/.*'),
51 _("URL shorteners should not be used"))
54 http_url_shorteners = [
55 forbid_shortener('goo.gl'),
56 forbid_shortener('t.co'),
57 forbid_shortener('ur1.ca'),
58 forbid_shortener('is.gd'),
59 forbid_shortener('bit.ly'),
60 forbid_shortener('tiny.cc'),
61 forbid_shortener('tinyurl.com'),
64 http_checks = https_enforcings + http_url_shorteners + [
65 (re.compile(r'.*github\.com/[^/]+/[^/]+\.git'),
66 _("Appending .git is not necessary")),
67 (re.compile(r'.*://[^/]*(github|gitlab|bitbucket|rawgit)[^/]*/([^/]+/){1,3}master'),
68 _("Use /HEAD instead of /master to point at a file in the default branch")),
72 'WebSite': http_checks,
73 'SourceCode': http_checks,
74 'Repo': https_enforcings,
75 'IssueTracker': http_checks + [
76 (re.compile(r'.*github\.com/[^/]+/[^/]+/*$'),
77 _("/issues is missing")),
78 (re.compile(r'.*gitlab\.com/[^/]+/[^/]+/*$'),
79 _("/issues is missing")),
81 'Donate': http_checks + [
82 (re.compile(r'.*flattr\.com'),
83 _("Flattr donation methods belong in the FlattrID flag")),
85 'Changelog': http_checks,
88 _("Unnecessary leading space")),
89 (re.compile(r'.*\s$'),
90 _("Unnecessary trailing space")),
93 (re.compile(r'.*\b(free software|open source)\b.*', re.IGNORECASE),
94 _("No need to specify that the app is Free Software")),
95 (re.compile(r'.*((your|for).*android|android.*(app|device|client|port|version))', re.IGNORECASE),
96 _("No need to specify that the app is for Android")),
97 (re.compile(r'.*[a-z0-9][.!?]( |$)'),
98 _("Punctuation should be avoided")),
100 _("Unnecessary leading space")),
101 (re.compile(r'.*\s$'),
102 _("Unnecessary trailing space")),
105 (re.compile(r'\s*[*#][^ .]'),
106 _("Invalid bulleted list")),
108 _("Unnecessary leading space")),
109 (re.compile(r'.*\s$'),
110 _("Unnecessary trailing space")),
111 (re.compile(r'.*([^[]|^)\[[^:[\]]+( |\]|$)'),
112 _("Invalid link - use [http://foo.bar Link title] or [http://foo.bar]")),
113 (re.compile(r'(^|.* )https?://[^ ]+'),
114 _("Unlinkified link - use [http://foo.bar Link title] or [http://foo.bar]")),
118 locale_pattern = re.compile(r'^[a-z]{2,3}(-[A-Z][A-Z])?$')
121 def check_regexes(app):
122 for f, checks in regex_checks.items():
125 t = metadata.fieldtype(f)
126 if t == metadata.TYPE_MULTILINE:
127 for l in v.splitlines():
129 yield "%s at line '%s': %s" % (f, l, r)
134 yield "%s '%s': %s" % (f, v, r)
137 def get_lastbuild(builds):
141 if not build.disable:
142 vercode = int(build.versionCode)
143 if lowest_vercode == -1 or vercode < lowest_vercode:
144 lowest_vercode = vercode
145 if not lastbuild or int(build.versionCode) > int(lastbuild.versionCode):
150 def check_ucm_tags(app):
151 lastbuild = get_lastbuild(app.builds)
152 if (lastbuild is not None
154 and app.UpdateCheckMode == 'RepoManifest'
155 and not lastbuild.commit.startswith('unknown')
156 and lastbuild.versionCode == app.CurrentVersionCode
157 and not lastbuild.forcevercode
158 and any(s in lastbuild.commit for s in '.,_-/')):
159 yield _("Last used commit '{commit}' looks like a tag, but Update Check Mode is '{ucm}'")\
160 .format(commit=lastbuild.commit, ucm=app.UpdateCheckMode)
163 def check_char_limits(app):
164 limits = config['char_limits']
166 if len(app.Summary) > limits['summary']:
167 yield _("Summary of length {length} is over the {limit} char limit")\
168 .format(length=len(app.Summary), limit=limits['summary'])
170 if len(app.Description) > limits['description']:
171 yield _("Description of length {length} is over the {limit} char limit")\
172 .format(length=len(app.Description), limit=limits['description'])
175 def check_old_links(app):
185 if any(s in app.Repo for s in usual_sites):
186 for f in ['WebSite', 'SourceCode', 'IssueTracker', 'Changelog']:
188 if any(s in v for s in old_sites):
189 yield _("App is in '{repo}' but has a link to {url}")\
190 .format(repo=app.Repo, url=v)
193 def check_useless_fields(app):
194 if app.UpdateCheckName == app.id:
195 yield _("Update Check Name is set to the known app id - it can be removed")
198 filling_ucms = re.compile(r'^(Tags.*|RepoManifest.*)')
201 def check_checkupdates_ran(app):
202 if filling_ucms.match(app.UpdateCheckMode):
203 if not app.AutoName and not app.CurrentVersion and app.CurrentVersionCode == '0':
204 yield _("UCM is set but it looks like checkupdates hasn't been run yet")
207 def check_empty_fields(app):
208 if not app.Categories:
209 yield _("Categories are not set")
212 all_categories = set([
223 "Science & Education",
233 def check_categories(app):
234 for categ in app.Categories:
235 if categ not in all_categories:
236 yield _("Category '%s' is not valid" % categ)
239 def check_duplicates(app):
240 if app.Name and app.Name == app.AutoName:
241 yield _("Name '%s' is just the auto name - remove it") % app.Name
244 for f in ['Source Code', 'Web Site', 'Issue Tracker', 'Changelog']:
250 yield _("Duplicate link in '{field}': {url}").format(field=f, url=v)
254 name = app.Name or app.AutoName
255 if app.Summary and name:
256 if app.Summary.lower() == name.lower():
257 yield _("Summary '%s' is just the app's name") % app.Summary
259 if app.Summary and app.Description and len(app.Description) == 1:
260 if app.Summary.lower() == app.Description[0].lower():
261 yield _("Description '%s' is just the app's summary") % app.Summary
264 for l in app.Description.splitlines():
268 yield _("Description has a duplicate line")
272 desc_url = re.compile(r'(^|[^[])\[([^ ]+)( |\]|$)')
275 def check_mediawiki_links(app):
276 wholedesc = ' '.join(app.Description)
277 for um in desc_url.finditer(wholedesc):
279 for m, r in http_checks:
281 yield _("URL {url} in Description: {error}").format(url=url, error=r)
284 def check_bulleted_lists(app):
285 validchars = ['*', '#']
288 for l in app.Description.splitlines():
293 if l[0] == lchar and l[1] == ' ':
295 if lcount > 2 and lchar not in validchars:
296 yield _("Description has a list (%s) but it isn't bulleted (*) nor numbered (#)") % lchar
303 def check_builds(app):
304 supported_flags = set(metadata.build_flags)
305 # needed for YAML and JSON
306 for build in app.builds:
308 if build.disable.startswith('Generated by import.py'):
309 yield _("Build generated by `fdroid import` - remove disable line once ready")
311 for s in ['master', 'origin', 'HEAD', 'default', 'trunk']:
312 if build.commit and build.commit.startswith(s):
313 yield _("Branch '{branch}' used as commit in build '{versionName}'")\
314 .format(branch=s, versionName=build.versionName)
315 for srclib in build.srclibs:
316 ref = srclib.split('@')[1].split('/')[0]
317 if ref.startswith(s):
318 yield _("Branch '{branch}' used as commit in srclib '{srclib}'")\
319 .format(branch=s, srclib=srclib)
320 for key in build.keys():
321 if key not in supported_flags:
322 yield _('%s is not an accepted build field') % key
325 def check_files_dir(app):
326 dir_path = os.path.join('metadata', app.id)
327 if not os.path.isdir(dir_path):
330 for name in os.listdir(dir_path):
331 path = os.path.join(dir_path, name)
332 if not (os.path.isfile(path) or name == 'signatures' or locale_pattern.match(name)):
333 yield _("Found non-file at %s") % path
337 used = {'signatures', }
338 for build in app.builds:
339 for fname in build.patch:
340 if fname not in files:
341 yield _("Unknown file '{filename}' in build '{versionName}'")\
342 .format(filename=fname, versionName=build.versionName)
346 for name in files.difference(used):
347 if locale_pattern.match(name):
349 yield _("Unused file at %s") % os.path.join(dir_path, name)
352 def check_format(app):
353 if options.format and not rewritemeta.proper_format(app):
354 yield _("Run rewritemeta to fix formatting")
357 def check_license_tag(app):
358 '''Ensure all license tags are in https://spdx.org/license-list'''
359 if app.License.rstrip('+') not in SPDX:
360 yield _('Invalid license tag "%s"! Use only tags from https://spdx.org/license-list') \
364 def check_extlib_dir(apps):
365 dir_path = os.path.join('build', 'extlib')
366 unused_extlib_files = set()
367 for root, dirs, files in os.walk(dir_path):
369 unused_extlib_files.add(os.path.join(root, name)[len(dir_path) + 1:])
373 for build in app.builds:
374 for path in build.extlibs:
375 if path not in unused_extlib_files:
376 yield _("{appid}: Unknown extlib {path} in build '{versionName}'")\
377 .format(appid=app.id, path=path, versionName=build.versionName)
381 for path in unused_extlib_files.difference(used):
382 if any(path.endswith(s) for s in [
384 'source.txt', 'origin.txt', 'md5.txt',
385 'LICENSE', 'LICENSE.txt',
386 'COPYING', 'COPYING.txt',
387 'NOTICE', 'NOTICE.txt',
390 yield _("Unused extlib at %s") % os.path.join(dir_path, path)
393 def check_for_unsupported_metadata_files(basedir=""):
394 """Checks whether any non-metadata files are in metadata/"""
399 formats = config['accepted_formats']
400 for f in glob.glob(basedir + 'metadata/*') + glob.glob(basedir + 'metadata/.*'):
404 exists = exists or os.path.exists(f + '.' + t)
406 print(_('"%s/" has no matching metadata file!') % f)
408 elif not os.path.splitext(f)[1][1:] in formats:
409 print('"' + f.replace(basedir, '')
410 + '" is not a supported file format: (' + ','.join(formats) + ')')
418 global config, options
420 # Parse command line...
421 parser = ArgumentParser(usage="%(prog)s [options] [APPID [APPID ...]]")
422 common.setup_global_opts(parser)
423 parser.add_argument("-f", "--format", action="store_true", default=False,
424 help=_("Also warn about formatting issues, like rewritemeta -l"))
425 parser.add_argument("appid", nargs='*', help=_("applicationId in the form APPID"))
426 metadata.add_metadata_arguments(parser)
427 options = parser.parse_args()
428 metadata.warnings_action = options.W
430 config = common.read_config(options)
433 allapps = metadata.read_metadata(xref=True)
434 apps = common.read_app_args(options.appid, allapps, False)
436 anywarns = check_for_unsupported_metadata_files()
438 apps_check_funcs = []
439 if len(options.appid) == 0:
440 # otherwise it finds tons of unused extlibs
441 apps_check_funcs.append(check_extlib_dir)
442 for check_func in apps_check_funcs:
443 for warn in check_func(apps.values()):
447 for appid, app in apps.items():
456 check_checkupdates_ran,
457 check_useless_fields,
461 check_mediawiki_links,
462 check_bulleted_lists,
469 for check_func in app_check_funcs:
470 for warn in check_func(app):
472 print("%s: %s" % (appid, warn))
478 # A compiled, public domain list of official SPDX license tags from:
479 # https://github.com/sindresorhus/spdx-license-list/blob/v3.0.1/spdx-simple.json
480 # The deprecated license tags have been removed from the list, they are at the
481 # bottom, starting after the last license tags that start with Z.
482 # This is at the bottom, since its a long list of data
484 "PublicDomain", # an F-Droid addition, until we can enforce a better option
524 "BSD-2-Clause-FreeBSD",
525 "BSD-2-Clause-NetBSD",
527 "BSD-3-Clause-Clear",
528 "BSD-3-Clause-No-Nuclear-License",
529 "BSD-3-Clause-No-Nuclear-License-2014",
530 "BSD-3-Clause-No-Nuclear-Warranty",
534 "BSD-3-Clause-Attribution",
550 "CNRI-Python-GPL-Compatible",
684 "MPL-2.0-no-copyleft-exception",
808 "zlib-acknowledgement",
814 if __name__ == "__main__":