3 # lint.py - part of the FDroid server tool
4 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU Affero General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See th
14 # GNU Affero General Public License for more details.
16 # You should have received a copy of the GNU Affero General Public Licen
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19 from argparse import ArgumentParser
28 from . import metadata
29 from . import rewritemeta
35 def enforce_https(domain):
36 return (re.compile(r'^[^h][^t][^t][^p][^s]://[^/]*' + re.escape(domain) + r'(/.*)?', re.IGNORECASE),
37 domain + " URLs should always use https://")
41 enforce_https('github.com'),
42 enforce_https('gitlab.com'),
43 enforce_https('bitbucket.org'),
44 enforce_https('apache.org'),
45 enforce_https('google.com'),
46 enforce_https('git.code.sf.net'),
47 enforce_https('svn.code.sf.net'),
48 enforce_https('anongit.kde.org'),
49 enforce_https('savannah.nongnu.org'),
50 enforce_https('git.savannah.nongnu.org'),
51 enforce_https('download.savannah.nongnu.org'),
52 enforce_https('savannah.gnu.org'),
53 enforce_https('git.savannah.gnu.org'),
54 enforce_https('download.savannah.gnu.org'),
55 enforce_https('github.io'),
56 enforce_https('gitlab.io'),
57 enforce_https('githubusercontent.com'),
61 def forbid_shortener(domain):
62 return (re.compile(r'https?://[^/]*' + re.escape(domain) + r'/.*'),
63 _("URL shorteners should not be used"))
66 http_url_shorteners = [
67 forbid_shortener('1url.com'),
68 forbid_shortener('adf.ly'),
69 forbid_shortener('bc.vc'),
70 forbid_shortener('bit.do'),
71 forbid_shortener('bit.ly'),
72 forbid_shortener('bitly.com'),
73 forbid_shortener('budurl.com'),
74 forbid_shortener('buzurl.com'),
75 forbid_shortener('cli.gs'),
76 forbid_shortener('cur.lv'),
77 forbid_shortener('cutt.us'),
78 forbid_shortener('db.tt'),
79 forbid_shortener('filoops.info'),
80 forbid_shortener('goo.gl'),
81 forbid_shortener('is.gd'),
82 forbid_shortener('ity.im'),
83 forbid_shortener('j.mp'),
84 forbid_shortener('l.gg'),
85 forbid_shortener('lnkd.in'),
86 forbid_shortener('moourl.com'),
87 forbid_shortener('ow.ly'),
88 forbid_shortener('para.pt'),
89 forbid_shortener('po.st'),
90 forbid_shortener('q.gs'),
91 forbid_shortener('qr.ae'),
92 forbid_shortener('qr.net'),
93 forbid_shortener('rdlnk.com'),
94 forbid_shortener('scrnch.me'),
95 forbid_shortener('short.nr'),
96 forbid_shortener('sn.im'),
97 forbid_shortener('snipurl.com'),
98 forbid_shortener('su.pr'),
99 forbid_shortener('t.co'),
100 forbid_shortener('tiny.cc'),
101 forbid_shortener('tinyarrows.com'),
102 forbid_shortener('tinyurl.com'),
103 forbid_shortener('tr.im'),
104 forbid_shortener('tweez.me'),
105 forbid_shortener('twitthis.com'),
106 forbid_shortener('twurl.nl'),
107 forbid_shortener('tyn.ee'),
108 forbid_shortener('u.bb'),
109 forbid_shortener('u.to'),
110 forbid_shortener('ur1.ca'),
111 forbid_shortener('urlof.site'),
112 forbid_shortener('v.gd'),
113 forbid_shortener('vzturl.com'),
114 forbid_shortener('x.co'),
115 forbid_shortener('xrl.us'),
116 forbid_shortener('yourls.org'),
117 forbid_shortener('zip.net'),
118 forbid_shortener('✩.ws'),
119 forbid_shortener('➡.ws'),
122 http_checks = https_enforcings + http_url_shorteners + [
123 (re.compile(r'.*github\.com/[^/]+/[^/]+\.git'),
124 _("Appending .git is not necessary")),
125 (re.compile(r'.*://[^/]*(github|gitlab|bitbucket|rawgit)[^/]*/([^/]+/){1,3}master'),
126 _("Use /HEAD instead of /master to point at a file in the default branch")),
130 'WebSite': http_checks,
131 'SourceCode': http_checks,
132 'Repo': https_enforcings,
133 'UpdateCheckMode': https_enforcings,
134 'IssueTracker': http_checks + [
135 (re.compile(r'.*github\.com/[^/]+/[^/]+/*$'),
136 _("/issues is missing")),
137 (re.compile(r'.*gitlab\.com/[^/]+/[^/]+/*$'),
138 _("/issues is missing")),
140 'Donate': http_checks + [
141 (re.compile(r'.*flattr\.com'),
142 _("Flattr donation methods belong in the FlattrID flag")),
143 (re.compile(r'.*liberapay\.com'),
144 _("Liberapay donation methods belong in the LiberapayID flag")),
146 'Changelog': http_checks,
149 _("Unnecessary leading space")),
150 (re.compile(r'.*\s$'),
151 _("Unnecessary trailing space")),
154 (re.compile(r'.*\b(free software|open source)\b.*', re.IGNORECASE),
155 _("No need to specify that the app is Free Software")),
156 (re.compile(r'.*((your|for).*android|android.*(app|device|client|port|version))', re.IGNORECASE),
157 _("No need to specify that the app is for Android")),
158 (re.compile(r'.*[a-z0-9][.!?]( |$)'),
159 _("Punctuation should be avoided")),
161 _("Unnecessary leading space")),
162 (re.compile(r'.*\s$'),
163 _("Unnecessary trailing space")),
165 'Description': https_enforcings + http_url_shorteners + [
166 (re.compile(r'\s*[*#][^ .]'),
167 _("Invalid bulleted list")),
169 _("Unnecessary leading space")),
170 (re.compile(r'.*\s$'),
171 _("Unnecessary trailing space")),
172 (re.compile(r'.*<(applet|base|body|button|embed|form|head|html|iframe|img|input|link|object|picture|script|source|style|svg|video).*', re.IGNORECASE),
173 _("Forbidden HTML tags")),
174 (re.compile(r'''.*\s+src=["']javascript:.*'''),
175 _("Javascript in HTML src attributes")),
179 locale_pattern = re.compile(r'^[a-z]{2,3}(-[A-Z][A-Z])?$')
182 def check_regexes(app):
183 for f, checks in regex_checks.items():
186 t = metadata.fieldtype(f)
187 if t == metadata.TYPE_MULTILINE:
188 for l in v.splitlines():
190 yield "%s at line '%s': %s" % (f, l, r)
195 yield "%s '%s': %s" % (f, v, r)
198 def get_lastbuild(builds):
202 if not build.disable:
203 vercode = int(build.versionCode)
204 if lowest_vercode == -1 or vercode < lowest_vercode:
205 lowest_vercode = vercode
206 if not lastbuild or int(build.versionCode) > int(lastbuild.versionCode):
211 def check_update_check_data_url(app):
212 """UpdateCheckData must have a valid HTTPS URL to protect checkupdates runs
214 if app.UpdateCheckData:
215 urlcode, codeex, urlver, verex = app.UpdateCheckData.split('|')
216 for url in (urlcode, urlver):
218 parsed = urllib.parse.urlparse(url)
219 if not parsed.scheme or not parsed.netloc:
220 yield _('UpdateCheckData not a valid URL: {url}').format(url=url)
221 if parsed.scheme != 'https':
222 yield _('UpdateCheckData must use HTTPS URL: {url}').format(url=url)
225 def check_vercode_operation(app):
226 if app.VercodeOperation and not common.VERCODE_OPERATION_RE.match(app.VercodeOperation):
227 yield _('Invalid VercodeOperation: {field}').format(field=app.VercodeOperation)
230 def check_ucm_tags(app):
231 lastbuild = get_lastbuild(app.builds)
232 if (lastbuild is not None
234 and app.UpdateCheckMode == 'RepoManifest'
235 and not lastbuild.commit.startswith('unknown')
236 and lastbuild.versionCode == app.CurrentVersionCode
237 and not lastbuild.forcevercode
238 and any(s in lastbuild.commit for s in '.,_-/')):
239 yield _("Last used commit '{commit}' looks like a tag, but Update Check Mode is '{ucm}'")\
240 .format(commit=lastbuild.commit, ucm=app.UpdateCheckMode)
243 def check_char_limits(app):
244 limits = config['char_limits']
246 if len(app.Summary) > limits['summary']:
247 yield _("Summary of length {length} is over the {limit} char limit")\
248 .format(length=len(app.Summary), limit=limits['summary'])
250 if len(app.Description) > limits['description']:
251 yield _("Description of length {length} is over the {limit} char limit")\
252 .format(length=len(app.Description), limit=limits['description'])
255 def check_old_links(app):
265 if any(s in app.Repo for s in usual_sites):
266 for f in ['WebSite', 'SourceCode', 'IssueTracker', 'Changelog']:
268 if any(s in v for s in old_sites):
269 yield _("App is in '{repo}' but has a link to {url}")\
270 .format(repo=app.Repo, url=v)
273 def check_useless_fields(app):
274 if app.UpdateCheckName == app.id:
275 yield _("Update Check Name is set to the known app id - it can be removed")
278 filling_ucms = re.compile(r'^(Tags.*|RepoManifest.*)')
281 def check_checkupdates_ran(app):
282 if filling_ucms.match(app.UpdateCheckMode):
283 if not app.AutoName and not app.CurrentVersion and app.CurrentVersionCode == '0':
284 yield _("UCM is set but it looks like checkupdates hasn't been run yet")
287 def check_empty_fields(app):
288 if not app.Categories:
289 yield _("Categories are not set")
292 all_categories = set([
303 "Science & Education",
313 def check_categories(app):
314 for categ in app.Categories:
315 if categ not in all_categories:
316 yield _("Category '%s' is not valid" % categ)
319 def check_duplicates(app):
320 if app.Name and app.Name == app.AutoName:
321 yield _("Name '%s' is just the auto name - remove it") % app.Name
324 for f in ['Source Code', 'Web Site', 'Issue Tracker', 'Changelog']:
330 yield _("Duplicate link in '{field}': {url}").format(field=f, url=v)
334 name = app.Name or app.AutoName
335 if app.Summary and name:
336 if app.Summary.lower() == name.lower():
337 yield _("Summary '%s' is just the app's name") % app.Summary
339 if app.Summary and app.Description and len(app.Description) == 1:
340 if app.Summary.lower() == app.Description[0].lower():
341 yield _("Description '%s' is just the app's summary") % app.Summary
344 for l in app.Description.splitlines():
348 yield _("Description has a duplicate line")
352 desc_url = re.compile(r'(^|[^[])\[([^ ]+)( |\]|$)')
355 def check_mediawiki_links(app):
356 wholedesc = ' '.join(app.Description)
357 for um in desc_url.finditer(wholedesc):
359 for m, r in http_checks:
361 yield _("URL {url} in Description: {error}").format(url=url, error=r)
364 def check_bulleted_lists(app):
365 validchars = ['*', '#']
368 for l in app.Description.splitlines():
373 if l[0] == lchar and l[1] == ' ':
375 if lcount > 2 and lchar not in validchars:
376 yield _("Description has a list (%s) but it isn't bulleted (*) nor numbered (#)") % lchar
383 def check_builds(app):
384 supported_flags = set(metadata.build_flags)
385 # needed for YAML and JSON
386 for build in app.builds:
388 if build.disable.startswith('Generated by import.py'):
389 yield _("Build generated by `fdroid import` - remove disable line once ready")
391 for s in ['master', 'origin', 'HEAD', 'default', 'trunk']:
392 if build.commit and build.commit.startswith(s):
393 yield _("Branch '{branch}' used as commit in build '{versionName}'")\
394 .format(branch=s, versionName=build.versionName)
395 for srclib in build.srclibs:
397 ref = srclib.split('@')[1].split('/')[0]
398 if ref.startswith(s):
399 yield _("Branch '{branch}' used as commit in srclib '{srclib}'")\
400 .format(branch=s, srclib=srclib)
402 yield _('srclibs missing name and/or @') + ' (srclibs: ' + srclib + ')'
403 for key in build.keys():
404 if key not in supported_flags:
405 yield _('%s is not an accepted build field') % key
408 def check_files_dir(app):
409 dir_path = os.path.join('metadata', app.id)
410 if not os.path.isdir(dir_path):
413 for name in os.listdir(dir_path):
414 path = os.path.join(dir_path, name)
415 if not (os.path.isfile(path) or name == 'signatures' or locale_pattern.match(name)):
416 yield _("Found non-file at %s") % path
420 used = {'signatures', }
421 for build in app.builds:
422 for fname in build.patch:
423 if fname not in files:
424 yield _("Unknown file '{filename}' in build '{versionName}'")\
425 .format(filename=fname, versionName=build.versionName)
429 for name in files.difference(used):
430 if locale_pattern.match(name):
432 yield _("Unused file at %s") % os.path.join(dir_path, name)
435 def check_format(app):
436 if options.format and not rewritemeta.proper_format(app):
437 yield _("Run rewritemeta to fix formatting")
440 def check_license_tag(app):
441 '''Ensure all license tags are in https://spdx.org/license-list'''
442 if app.License.rstrip('+') not in SPDX:
443 yield _('Invalid license tag "%s"! Use only tags from https://spdx.org/license-list') \
447 def check_extlib_dir(apps):
448 dir_path = os.path.join('build', 'extlib')
449 unused_extlib_files = set()
450 for root, dirs, files in os.walk(dir_path):
452 unused_extlib_files.add(os.path.join(root, name)[len(dir_path) + 1:])
456 for build in app.builds:
457 for path in build.extlibs:
458 if path not in unused_extlib_files:
459 yield _("{appid}: Unknown extlib {path} in build '{versionName}'")\
460 .format(appid=app.id, path=path, versionName=build.versionName)
464 for path in unused_extlib_files.difference(used):
465 if any(path.endswith(s) for s in [
467 'source.txt', 'origin.txt', 'md5.txt',
468 'LICENSE', 'LICENSE.txt',
469 'COPYING', 'COPYING.txt',
470 'NOTICE', 'NOTICE.txt',
473 yield _("Unused extlib at %s") % os.path.join(dir_path, path)
476 def check_for_unsupported_metadata_files(basedir=""):
477 """Checks whether any non-metadata files are in metadata/"""
482 formats = config['accepted_formats']
483 for f in glob.glob(basedir + 'metadata/*') + glob.glob(basedir + 'metadata/.*'):
487 exists = exists or os.path.exists(f + '.' + t)
489 print(_('"%s/" has no matching metadata file!') % f)
491 elif not os.path.splitext(f)[1][1:] in formats:
492 print('"' + f.replace(basedir, '')
493 + '" is not a supported file format: (' + ','.join(formats) + ')')
501 global config, options
503 # Parse command line...
504 parser = ArgumentParser(usage="%(prog)s [options] [APPID [APPID ...]]")
505 common.setup_global_opts(parser)
506 parser.add_argument("-f", "--format", action="store_true", default=False,
507 help=_("Also warn about formatting issues, like rewritemeta -l"))
508 parser.add_argument("appid", nargs='*', help=_("applicationId in the form APPID"))
509 metadata.add_metadata_arguments(parser)
510 options = parser.parse_args()
511 metadata.warnings_action = options.W
513 config = common.read_config(options)
516 allapps = metadata.read_metadata(xref=True)
517 apps = common.read_app_args(options.appid, allapps, False)
519 anywarns = check_for_unsupported_metadata_files()
521 apps_check_funcs = []
522 if len(options.appid) == 0:
523 # otherwise it finds tons of unused extlibs
524 apps_check_funcs.append(check_extlib_dir)
525 for check_func in apps_check_funcs:
526 for warn in check_func(apps.values()):
530 for appid, app in apps.items():
536 check_update_check_data_url,
537 check_vercode_operation,
541 check_checkupdates_ran,
542 check_useless_fields,
546 check_mediawiki_links,
547 check_bulleted_lists,
554 for check_func in app_check_funcs:
555 for warn in check_func(app):
557 print("%s: %s" % (appid, warn))
563 # A compiled, public domain list of official SPDX license tags from:
564 # https://github.com/sindresorhus/spdx-license-list/blob/v3.0.1/spdx-simple.json
565 # The deprecated license tags have been removed from the list, they are at the
566 # bottom, starting after the last license tags that start with Z.
567 # This is at the bottom, since its a long list of data
569 "PublicDomain", # an F-Droid addition, until we can enforce a better option
609 "BSD-2-Clause-FreeBSD",
610 "BSD-2-Clause-NetBSD",
612 "BSD-3-Clause-Clear",
613 "BSD-3-Clause-No-Nuclear-License",
614 "BSD-3-Clause-No-Nuclear-License-2014",
615 "BSD-3-Clause-No-Nuclear-Warranty",
619 "BSD-3-Clause-Attribution",
635 "CNRI-Python-GPL-Compatible",
769 "MPL-2.0-no-copyleft-exception",
893 "zlib-acknowledgement",
899 if __name__ == "__main__":