2 # -*- coding: utf-8 -*-
4 # stats.py - part of the FDroid server tools
5 # Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
28 from optparse import OptionParser
35 # Read configuration...
36 execfile('config.py', globals())
38 # Parse command line...
39 parser = OptionParser()
40 parser.add_option("-v", "--verbose", action="store_true", default=False,
41 help="Spew out even more information than normal")
42 parser.add_option("-d", "--download", action="store_true", default=False,
43 help="Download logs we don't have")
44 (options, args) = parser.parse_args()
48 logsdir = os.path.join(statsdir, 'logs')
49 logsarchivedir = os.path.join(logsdir, 'archive')
50 datadir = os.path.join(statsdir, 'data')
51 if not os.path.exists(statsdir):
53 if not os.path.exists(logsdir):
55 if not os.path.exists(datadir):
59 # Get any access logs we don't have...
63 print 'Retrieving logs'
64 ssh = paramiko.SSHClient()
65 ssh.load_system_host_keys()
66 ssh.connect('f-droid.org', username='fdroid', timeout=10,
67 key_filename=webserver_keyfile)
69 ftp.get_channel().settimeout(15)
75 if f.startswith('access-') and f.endswith('.log'):
77 destpath = os.path.join(logsdir, f)
78 archivepath = os.path.join(logsarchivedir, f + '.gz')
79 if os.path.exists(archivepath):
80 if os.path.exists(destpath):
81 # Just in case we have it archived but failed to remove
85 destsize = ftp.stat(f).st_size
86 if (not os.path.exists(destpath) or
87 os.path.getsize(destpath) != destsize):
88 print "...retrieving " + f
90 except Exception as e:
101 logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] "GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) \d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
102 logsearch = re.compile(logexpr).search
105 knownapks = common.KnownApks()
106 for logfile in glob.glob(os.path.join(logsdir,'access-*.log')):
107 logdate = logfile[len(logsdir) + 1 + len('access-'):-4]
108 matches = (logsearch(line) for line in file(logfile))
109 for match in matches:
110 if match and match.group('statuscode') == '200':
111 uri = match.group('uri')
112 if uri.endswith('.apk'):
113 _, apkname = os.path.split(uri)
114 app = knownapks.getapp(apkname)
122 if not apkname in unknownapks:
123 unknownapks.append(apkname)
125 # Calculate and write stats for total downloads...
126 f = open('stats/total_downloads_app.txt', 'w')
129 for app, count in apps.iteritems():
130 lst.append(app + " " + str(count))
131 alldownloads += count
132 lst.append("ALL " + str(alldownloads))
133 f.write('# Total downloads by application, since October 2011\n')
134 for line in sorted(lst):
138 # Write list of latest apps added to the repo...
139 latest = knownapks.getlatest(10)
140 f = open('stats/latestapps.txt', 'w')
145 if len(unknownapks) > 0:
146 print '\nUnknown apks:'
147 for apk in unknownapks:
152 if __name__ == "__main__":