2 # -*- coding: utf-8 -*-
4 # stats.py - part of the FDroid server tools
5 # Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
26 from optparse import OptionParser
32 def carbon_send(key, value):
34 s.connect((carbon_host, carbon_port))
35 msg = '%s %d %d\n' % (key, value, int(time.time()))
41 # Read configuration...
42 global update_stats, stats_to_carbon
44 stats_to_carbon = False
45 execfile('config.py', globals())
48 print "Stats are disabled - check your configuration"
51 # Parse command line...
52 parser = OptionParser()
53 parser.add_option("-v", "--verbose", action="store_true", default=False,
54 help="Spew out even more information than normal")
55 parser.add_option("-d", "--download", action="store_true", default=False,
56 help="Download logs we don't have")
57 (options, args) = parser.parse_args()
59 # Get all metadata-defined apps...
60 metaapps = common.read_metadata(options.verbose)
63 logsdir = os.path.join(statsdir, 'logs')
64 logsarchivedir = os.path.join(logsdir, 'archive')
65 datadir = os.path.join(statsdir, 'data')
66 if not os.path.exists(statsdir):
68 if not os.path.exists(logsdir):
70 if not os.path.exists(datadir):
74 # Get any access logs we don't have...
78 print 'Retrieving logs'
79 ssh = paramiko.SSHClient()
80 ssh.load_system_host_keys()
81 ssh.connect('f-droid.org', username='fdroid', timeout=10,
82 key_filename=webserver_keyfile)
84 ftp.get_channel().settimeout(60)
90 if f.startswith('access-') and f.endswith('.log.gz'):
92 destpath = os.path.join(logsdir, f)
93 destsize = ftp.stat(f).st_size
94 if (not os.path.exists(destpath) or
95 os.path.getsize(destpath) != destsize):
96 print "...retrieving " + f
98 except Exception as e:
110 print 'Processing logs...'
111 logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] "GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) \d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
112 logsearch = re.compile(logexpr).search
115 knownapks = common.KnownApks()
116 for logfile in glob.glob(os.path.join(logsdir,'access-*.log.gz')):
118 print '...' + logfile
119 logdate = logfile[len(logsdir) + 1 + len('access-'):-7]
120 p = subprocess.Popen(["zcat", logfile], stdout = subprocess.PIPE)
121 matches = (logsearch(line) for line in p.stdout)
122 for match in matches:
123 if match and match.group('statuscode') == '200':
124 uri = match.group('uri')
125 if uri.endswith('.apk'):
126 _, apkname = os.path.split(uri)
127 app = knownapks.getapp(apkname)
135 if not apkname in unknownapks:
136 unknownapks.append(apkname)
138 # Calculate and write stats for total downloads...
141 for app, count in apps.iteritems():
142 lst.append(app + " " + str(count))
144 carbon_send('fdroid.download.' + app.replace('.', '_'), count)
145 alldownloads += count
146 lst.append("ALL " + str(alldownloads))
147 f = open('stats/total_downloads_app.txt', 'w')
148 f.write('# Total downloads by application, since October 2011\n')
149 for line in sorted(lst):
153 # Calculate and write stats for repo types...
156 if len(app['Repo Type']) == 0:
159 if app['Repo Type'] == 'srclib':
160 rtype = common.getsrclibvcs(app['Repo'])
162 rtype = app['Repo Type']
163 if rtype in repotypes:
164 repotypes[rtype] += 1;
167 f = open('stats/repotypes.txt', 'w')
168 for rtype, count in repotypes.iteritems():
169 f.write(rtype + ' ' + str(count) + '\n')
172 # Calculate and write stats for update check modes...
175 checkmode = app['Update Check Mode'].split('/')[0]
176 if checkmode in ucms:
177 ucms[checkmode] += 1;
180 f = open('stats/update_check_modes.txt', 'w')
181 for checkmode, count in ucms.iteritems():
182 f.write(checkmode + ' ' + str(count) + '\n')
185 # Calculate and write stats for licenses...
188 license = app['License']
189 if license in licenses:
190 licenses[license] += 1;
192 licenses[license] = 1
193 f = open('stats/licenses.txt', 'w')
194 for license, count in licenses.iteritems():
195 f.write(license + ' ' + str(count) + '\n')
198 # Write list of latest apps added to the repo...
199 latest = knownapks.getlatest(10)
200 f = open('stats/latestapps.txt', 'w')
205 if len(unknownapks) > 0:
206 print '\nUnknown apks:'
207 for apk in unknownapks:
212 if __name__ == "__main__":