Commit | Line | Data |
---|---|---|
57ab47fa MW |
1 | #! /bin/bash |
2 | # No, we can not deal with sh alone. | |
3 | ||
4 | set -e | |
5 | set -u | |
6 | # ERR traps should be inherited from functions too. (And command | |
7 | # substitutions and subshells and whatnot, but for us the function is | |
8 | # the important part here) | |
9 | set -E | |
10 | ||
11 | # websync script for Debian | |
12 | # Based losely on the old websync written by an | |
13 | # unknown number of different people over the years and ftpsync. | |
14 | # | |
15 | # Copyright (C) 2008,2009 Joerg Jaspert <joerg@debian.org> | |
16 | # | |
17 | # This program is free software; you can redistribute it and/or | |
18 | # modify it under the terms of the GNU General Public License as | |
19 | # published by the Free Software Foundation; version 2. | |
20 | # | |
21 | # This program is distributed in the hope that it will be useful, but | |
22 | # WITHOUT ANY WARRANTY; without even the implied warranty of | |
23 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
24 | # General Public License for more details. | |
25 | # | |
26 | # You should have received a copy of the GNU General Public License | |
27 | # along with this program; if not, write to the Free Software | |
28 | # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. | |
29 | ||
30 | # In case the admin somehow wants to have this script located someplace else, | |
31 | # he can set BASEDIR, and we will take that. If it is unset we take ${HOME} | |
32 | # How the admin sets this isn't our place to deal with. One could use a wrapper | |
33 | # for that. Or pam_env. Or whatever fits in the local setup. :) | |
34 | BASEDIR=${BASEDIR:-"${HOME}"} | |
35 | ||
36 | # Script version. DO NOT CHANGE, *unless* you change the master copy maintained | |
37 | # by Joerg Jaspert and the Debian mirroradm group. | |
38 | # This is used to track which mirror is using which script version. | |
39 | VERSION="0815" | |
40 | ||
41 | # Source our common functions | |
42 | . "${BASEDIR}/etc/common" | |
43 | ||
44 | ######################################################################## | |
45 | ######################################################################## | |
46 | ## functions ## | |
47 | ######################################################################## | |
48 | ######################################################################## | |
49 | # All the stuff we want to do when we exit, no matter where | |
50 | cleanup() { | |
51 | trap - ERR TERM HUP INT QUIT EXIT | |
52 | # all done. Mail the log, exit. | |
53 | log "Mirrorsync done"; | |
54 | if [ -n "${MAILTO}" ]; then | |
55 | # In case rsync had something on stderr | |
56 | if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then | |
57 | mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error" | |
58 | fi | |
59 | if [ "x${ERRORSONLY}x" = "xfalsex" ]; then | |
60 | # And the normal log | |
61 | MAILFILES="${LOG}" | |
62 | if [ "x${FULLLOGS}x" = "xtruex" ]; then | |
63 | # Someone wants full logs including rsync | |
64 | MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log" | |
65 | fi | |
66 | cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] web sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} | |
67 | fi | |
68 | fi | |
69 | ||
70 | savelog "${LOGDIR}/rsync-${NAME}.log" | |
71 | savelog "${LOGDIR}/rsync-${NAME}.error" | |
72 | savelog "$LOG" > /dev/null | |
73 | ||
74 | rm -f "${LOCK}" | |
75 | } | |
76 | ||
77 | ||
78 | # Check rsyncs return value | |
79 | check_rsync() { | |
80 | ||
81 | ret=$1 | |
82 | msg=$2 | |
83 | ||
84 | # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED | |
85 | # and us re-running. If it's not, uplink is broken anyways. | |
86 | case "${ret}" in | |
87 | 0) return 0;; | |
88 | 24) return 0;; | |
89 | 23) return 2;; | |
90 | 30) return 2;; | |
91 | *) | |
92 | error "ERROR: ${msg}" | |
93 | return 1 | |
94 | ;; | |
95 | esac | |
96 | } | |
97 | ||
98 | ######################################################################## | |
99 | ######################################################################## | |
100 | ||
101 | # As what are we called? | |
102 | NAME="`basename $0`" | |
103 | ||
104 | # Now source the config. | |
105 | . "${BASEDIR}/etc/${NAME}.conf" | |
106 | ||
107 | ######################################################################## | |
108 | # Config options go here. Feel free to overwrite them in the config # | |
109 | # file if you need to. # | |
110 | # On debian.org machines the defaults should be ok. # | |
111 | ######################################################################## | |
112 | ||
113 | ######################################################################## | |
114 | # There should be nothing to edit here, use the config file # | |
115 | ######################################################################## | |
116 | MIRRORNAME=${MIRRORNAME:-`hostname -f`} | |
117 | # Where to put logfiles in | |
118 | LOGDIR=${LOGDIR:-"${BASEDIR}/log"} | |
119 | # Our own logfile | |
120 | LOG=${LOG:-"${LOGDIR}/${NAME}.log"} | |
121 | ||
122 | # Where should we put all the mirrored files? | |
123 | TO=${TO:-"/org/www.debian.org/www"} | |
124 | ||
125 | # used by log() and error() | |
126 | PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"} | |
127 | ||
128 | # Where to send mails about mirroring to? | |
129 | if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then | |
130 | # We are not on a debian.org host | |
131 | MAILTO=${MAILTO:-"root"} | |
132 | else | |
133 | # Yay, on a .debian.org host | |
134 | MAILTO=${MAILTO:-"mirrorlogs@debian.org"} | |
135 | fi | |
136 | # Want errors only or every log? | |
137 | ERRORSONLY=${ERRORSONLY:-"true"} | |
138 | # Want full logs, ie. including the rsync one? | |
139 | FULLLOGS=${FULLLOGS:-"false"} | |
140 | ||
141 | # How many logfiles to keep | |
142 | LOGROTATE=${LOGROTATE:-14} | |
143 | ||
144 | # Our lockfile | |
145 | LOCK=${LOCK:-"${TO}/Website-Update-in-Progress-${MIRRORNAME}"} | |
146 | # Do we need another rsync run? | |
147 | UPDATEREQUIRED="${TO}/Website-Update-Required-${MIRRORNAME}" | |
148 | # Trace file for mirror stats and checks (make sure we get full hostname) | |
149 | TRACE=${TRACE:-".project/trace/${MIRRORNAME}"} | |
150 | ||
151 | # rsync program | |
152 | RSYNC=${RSYNC:-rsync} | |
153 | # Rsync filter rules. Used to protect various files we always want to keep, even if we otherwise delete | |
154 | # excluded files | |
155 | RSYNC_FILTER=${RSYNC_FILTER:-"--filter=protect_Website-Update-in-Progress-${MIRRORNAME} --filter=protect_${TRACE} --filter=protect_Website-Update-Required-${MIRRORNAME}"} | |
156 | # Default rsync options for *every* rsync call | |
157 | RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --timeout 3600 --stats ${RSYNC_FILTER}"} | |
158 | RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded"} | |
159 | # Which rsync share to use on our upstream mirror? | |
160 | RSYNC_PATH=${RSYNC_PATH:-"web.debian.org"} | |
161 | ||
162 | # our username for the rsync share | |
163 | RSYNC_USER=${RSYNC_USER:-""} | |
164 | # the password | |
165 | RSYNC_PASSWORD=${RSYNC_PASSWORD:-""} | |
166 | ||
167 | # a possible proxy | |
168 | RSYNC_PROXY=${RSYNC_PROXY:-""} | |
169 | ||
170 | # General excludes. | |
171 | EXCLUDE=${EXCLUDE:-"--exclude ${HOSTNAME}"} | |
172 | ||
173 | # The temp directory used by rsync --delay-updates is not | |
174 | # world-readable remotely. Always exclude it to avoid errors. | |
175 | EXCLUDE="${EXCLUDE} --exclude .~tmp~/" | |
176 | ||
177 | # And site specific excludes, by default its the sponsor stuff that should be local to all (except templates) | |
178 | SITE_FILTER=${SITE_FILTER:-"--include sponsor.deb.* --exclude sponsor_img.* --exclude sponsor.html --exclude sponsor.*.html --filter=protect_sponsor_img.* --filter=protect_sponsor.html --filter=protect_sponsor.*.html"} | |
179 | ||
180 | # Hooks | |
181 | HOOK1=${HOOK1:-""} | |
182 | HOOK2=${HOOK2:-""} | |
183 | HOOK3=${HOOK3:-""} | |
184 | HOOK4=${HOOK4:-""} | |
185 | ||
186 | # Are we a hub? | |
187 | HUB=${HUB:-"false"} | |
188 | ||
189 | # Some sane defaults | |
190 | cd "${BASEDIR}" | |
191 | umask 022 | |
192 | ||
193 | # If we are here for the first time, create the | |
194 | # destination and the trace directory | |
195 | mkdir -p "${TO}/.project/trace" | |
196 | ||
197 | # Used to make sure we will have the archive fully and completly synced before | |
198 | # we stop, even if we get multiple pushes while this script is running. | |
199 | # Otherwise we can end up with a half-synced archive: | |
200 | # - get a push | |
201 | # - sync, while locked | |
202 | # - get another push. Of course no extra sync run then happens, we are locked. | |
203 | # - done. Archive not correctly synced, we don't have all the changes from the second push. | |
204 | touch "${UPDATEREQUIRED}" | |
205 | ||
206 | # Check to see if another sync is in progress | |
207 | if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then | |
208 | if ! $(kill -0 $(cat ${LOCK}) 2>/dev/null); then | |
209 | # Process does either not exist or is not owned by us. | |
210 | echo "$$" > "${LOCK}" | |
211 | else | |
212 | echo "Unable to start rsync, lock file still exists, PID $(cat ${LOCK})" | |
213 | exit 1 | |
214 | fi | |
215 | fi | |
216 | ||
217 | trap cleanup EXIT ERR TERM HUP INT QUIT | |
218 | ||
219 | # Start log by redirecting everything there. | |
220 | exec >"$LOG" 2>&1 </dev/null | |
221 | ||
222 | # Look who pushed us and note that in the log. | |
223 | log "Mirrorsync start" | |
224 | PUSHFROM="${SSH_CONNECTION%%\ *}" | |
225 | if [ -n "${PUSHFROM}" ]; then | |
226 | log "We got pushed from ${PUSHFROM}" | |
227 | fi | |
228 | log "Acquired main lock" | |
229 | ||
230 | HOOK=( | |
231 | HOOKNR=1 | |
232 | HOOKSCR=${HOOK1} | |
233 | ) | |
234 | hook $HOOK | |
235 | ||
236 | # Now, we might want to sync from anonymous too. | |
237 | # This is that deep in this script so hook1 could, if wanted, change things! | |
238 | if [ -z ${RSYNC_USER} ]; then | |
239 | RSYNCPTH="${RSYNC_HOST}" | |
240 | else | |
241 | RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}" | |
242 | fi | |
243 | ||
244 | # Now do the actual mirroring, and run as long as we have an updaterequired file. | |
245 | export RSYNC_PASSWORD | |
246 | export RSYNC_PROXY | |
247 | ||
248 | while [ -e "${UPDATEREQUIRED}" ]; do | |
249 | log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists" | |
250 | ||
251 | rm -f "${UPDATEREQUIRED}" | |
252 | log "Syncing: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SITE_FILTER} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" | |
253 | ||
254 | set +e | |
255 | ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SITE_FILTER} \ | |
256 | ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error" | |
257 | result=$? | |
258 | set -e | |
259 | ||
260 | log "Back from rsync with returncode ${result}" | |
261 | ||
262 | set +e | |
263 | check_rsync $result "Sync went wrong, got errorcode ${result}. Logfile: ${LOG}" | |
264 | GO=$? | |
265 | set -e | |
266 | ||
267 | if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then | |
268 | log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" | |
269 | elif [ ${GO} -ne 0 ]; then | |
270 | exit 3 | |
271 | fi | |
272 | ||
273 | HOOK=( | |
274 | HOOKNR=2 | |
275 | HOOKSCR=${HOOK2} | |
276 | ) | |
277 | hook $HOOK | |
278 | ||
279 | done | |
280 | ||
281 | mkdir -p "${TO}/.project/trace" | |
282 | LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}" | |
283 | echo "Used websync version: ${VERSION}" >> "${TO}/${TRACE}" | |
284 | echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}" | |
285 | ||
286 | HOOK=( | |
287 | HOOKNR=3 | |
288 | HOOKSCR=${HOOK3} | |
289 | ) | |
290 | hook $HOOK | |
291 | ||
292 | if [ x${HUB} = "xtrue" ]; then | |
293 | log "Trigger slave mirrors" | |
294 | ${BASEDIR}/bin/runmirrors "websync" | |
295 | log "Trigger slave done" | |
296 | ||
297 | HOOK=( | |
298 | HOOKNR=4 | |
299 | HOOKSCR=${HOOK4} | |
300 | ) | |
301 | hook $HOOK | |
302 | fi | |
303 | ||
304 | # All done, rest is done by cleanup hook. |