Commit | Line | Data |
---|---|---|
db2c5b8b MW |
1 | #! /bin/bash |
2 | # No, we can not deal with sh alone. | |
3 | ||
4 | set -e | |
5 | set -u | |
6 | # ERR traps should be inherited from functions too. (And command | |
7 | # substitutions and subshells and whatnot, but for us the function is | |
8 | # the important part here) | |
9 | set -E | |
10 | ||
11 | # ftpsync script for Debian | |
12 | # Based losely on a number of existing scripts, written by an | |
13 | # unknown number of different people over the years. | |
14 | # | |
15 | # Copyright (C) 2008,2009,2010,2011 Joerg Jaspert <joerg@debian.org> | |
16 | # | |
17 | # This program is free software; you can redistribute it and/or | |
18 | # modify it under the terms of the GNU General Public License as | |
19 | # published by the Free Software Foundation; version 2. | |
20 | # | |
21 | # This program is distributed in the hope that it will be useful, but | |
22 | # WITHOUT ANY WARRANTY; without even the implied warranty of | |
23 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
24 | # General Public License for more details. | |
25 | # | |
26 | # You should have received a copy of the GNU General Public License | |
27 | # along with this program; if not, write to the Free Software | |
28 | # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. | |
29 | ||
30 | # In case the admin somehow wants to have this script located someplace else, | |
31 | # he can set BASEDIR, and we will take that. If it is unset we take ${HOME} | |
32 | # How the admin sets this isn't our place to deal with. One could use a wrapper | |
33 | # for that. Or pam_env. Or whatever fits in the local setup. :) | |
34 | BASEDIR=${BASEDIR:-"${HOME}"} | |
35 | ||
36 | # Script version. DO NOT CHANGE, *unless* you change the master copy maintained | |
37 | # by Joerg Jaspert and the Debian mirroradm group. | |
38 | # This is used to track which mirror is using which script version. | |
39 | VERSION="80387" | |
40 | ||
41 | # Source our common functions | |
42 | . "${BASEDIR}/etc/common" | |
43 | ||
44 | ######################################################################## | |
45 | ######################################################################## | |
46 | ## functions ## | |
47 | ######################################################################## | |
48 | ######################################################################## | |
49 | # We want to be able to get told what kind of sync we should do. This | |
50 | # might be anything, from the archive to sync, the stage to do, etc. A | |
51 | # list of currently understood and valid options is below. Multiple | |
52 | # options are seperated by space. All the words have to have the word | |
53 | # sync: in front or nothing will get used! | |
54 | # | |
55 | # Option Behaviour | |
56 | # stage1 Only do stage1 sync | |
57 | # stage2 Only do stage2 sync | |
58 | # all Do a complete sync | |
59 | # mhop Do a mhop sync, usually additionally to stage1 | |
60 | # archive:foo Sync archive foo (if config for foo is available) | |
61 | # callback Call back when done (needs proper ssh setup for this to | |
62 | # work). It will always use the "command" callback:$HOSTNAME | |
63 | # where $HOSTNAME is the one defined below/in config and | |
64 | # will happen before slave mirrors are triggered. | |
65 | # | |
66 | # So to get us to sync all of the archive behind bpo and call back when | |
67 | # we are done, a trigger command of | |
68 | # "ssh $USER@$HOST sync:all sync:archive:bpo sync:callback" will do the | |
69 | # trick. | |
70 | check_commandline() { | |
71 | while [ $# -gt 0 ]; do | |
72 | case "$1" in | |
73 | sync:stage1) | |
74 | SYNCSTAGE1="true" | |
75 | SYNCALL="false" | |
76 | ;; | |
77 | sync:stage2) | |
78 | SYNCSTAGE2="true" | |
79 | SYNCALL="false" | |
80 | ;; | |
81 | sync:callback) | |
82 | SYNCCALLBACK="true" | |
83 | ;; | |
84 | sync:archive:*) | |
85 | ARCHIVE=${1##sync:archive:} | |
86 | # We do not like / or . in the remotely supplied archive name. | |
87 | ARCHIVE=${ARCHIVE//\/} | |
88 | ARCHIVE=${ARCHIVE//.} | |
89 | ;; | |
90 | sync:all) | |
91 | SYNCALL="true" | |
92 | ;; | |
93 | sync:mhop) | |
94 | SYNCMHOP="true" | |
95 | ;; | |
96 | *) | |
97 | echo "Unknown option ${1} ignored" | |
98 | ;; | |
99 | esac | |
100 | shift # Check next set of parameters. | |
101 | done | |
102 | } | |
103 | ||
104 | # All the stuff we want to do when we exit, no matter where | |
105 | cleanup() { | |
106 | trap - ERR TERM HUP INT QUIT EXIT | |
107 | # all done. Mail the log, exit. | |
108 | log "Mirrorsync done"; | |
109 | ||
110 | # Lets get a statistical value | |
111 | SPEED="unknown" | |
112 | if [ -f "${LOGDIR}/rsync-${NAME}.log" ]; then | |
113 | SPEED=$( | |
114 | SPEEDLINE=$(egrep '[0-9.]+ bytes/sec' "${LOGDIR}/rsync-${NAME}.log") | |
115 | set "nothing" ${SPEEDLINE} | |
70068cad | 116 | echo ${8:-""} |
db2c5b8b MW |
117 | ) |
118 | if [ -n "${SPEED}" ]; then | |
119 | SPEED=${SPEED%%.*} | |
120 | SPEED=$(( $SPEED / 1024 )) | |
121 | fi | |
122 | fi | |
123 | log "Rsync transfer speed: ${SPEED} KB/s" | |
124 | ||
125 | if [ -n "${MAILTO}" ]; then | |
126 | # In case rsync had something on stderr | |
127 | if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then | |
128 | mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error" | |
129 | fi | |
130 | if [ "x${ERRORSONLY}x" = "xfalsex" ]; then | |
131 | # And the normal log | |
132 | MAILFILES="${LOG}" | |
133 | if [ "x${FULLLOGS}x" = "xtruex" ]; then | |
134 | # Someone wants full logs including rsync | |
135 | MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log" | |
136 | fi | |
137 | cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] archive sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} | |
138 | fi | |
139 | fi | |
140 | ||
141 | savelog "${LOGDIR}/rsync-${NAME}.log" | |
142 | savelog "${LOGDIR}/rsync-${NAME}.error" | |
143 | savelog "$LOG" > /dev/null | |
144 | ||
145 | rm -f "${LOCK}" | |
146 | } | |
147 | ||
148 | # Check rsyncs return value | |
149 | check_rsync() { | |
150 | ret=$1 | |
151 | msg=$2 | |
152 | ||
153 | # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED | |
154 | # and us re-running. If it's not, uplink is broken anyways. | |
155 | case "${ret}" in | |
156 | 0) return 0;; | |
157 | 24) return 0;; | |
158 | 23) return 2;; | |
159 | 30) return 2;; | |
160 | *) | |
161 | error "ERROR: ${msg}" | |
162 | return 1 | |
163 | ;; | |
164 | esac | |
165 | } | |
166 | ||
167 | ######################################################################## | |
168 | ######################################################################## | |
169 | ||
170 | ||
171 | # As what are we called? | |
172 | NAME="$(basename $0)" | |
173 | # The original command line arguments need to be saved! | |
174 | if [ $# -gt 0 ]; then | |
175 | ORIGINAL_COMMAND=$* | |
176 | else | |
177 | ORIGINAL_COMMAND="" | |
178 | fi | |
179 | ||
180 | SSH_ORIGINAL_COMMAND=${SSH_ORIGINAL_COMMAND:-""} | |
181 | # Now, check if we got told about stuff via ssh | |
182 | if [ -n "${SSH_ORIGINAL_COMMAND}" ]; then | |
183 | # We deliberately add "nothing" and ignore it right again, to avoid | |
184 | # people from outside putting some set options in the first place, | |
185 | # making us parse them... | |
186 | set "nothing" "${SSH_ORIGINAL_COMMAND}" | |
187 | shift | |
188 | # Yes, unqouted $* here. Or the function will only see it as one | |
189 | # parameter, which doesnt help the case in it. | |
190 | check_commandline $* | |
191 | fi | |
192 | ||
193 | # Now, we can locally override all the above variables by just putting | |
194 | # them into the .ssh/authorized_keys file forced command. | |
195 | if [ -n "${ORIGINAL_COMMAND}" ]; then | |
196 | set ${ORIGINAL_COMMAND} | |
197 | check_commandline $* | |
198 | fi | |
199 | ||
200 | # If we have been told to do stuff for a different archive than default, | |
201 | # set the name accordingly. | |
202 | ARCHIVE=${ARCHIVE:-""} | |
203 | if [ -n "${ARCHIVE}" ]; then | |
204 | NAME="${NAME}-${ARCHIVE}" | |
205 | fi | |
206 | ||
207 | # Now source the config for the archive we run on. | |
208 | # (Yes, people can also overwrite the options above in the config file | |
209 | # if they want to) | |
210 | if [ -f "${BASEDIR}/etc/${NAME}.conf" ]; then | |
211 | . "${BASEDIR}/etc/${NAME}.conf" | |
212 | else | |
213 | echo "Nono, you can't tell us about random archives. Bad boy!" | |
214 | exit 1 | |
215 | fi | |
216 | ||
217 | ######################################################################## | |
218 | # Config options go here. Feel free to overwrite them in the config # | |
219 | # file if you need to. # | |
220 | # On debian.org machines the defaults should be ok. # | |
221 | # # | |
222 | # The following extra variables can be defined in the config file: # | |
223 | # # | |
224 | # ARCH_EXCLUDE # | |
225 | # can be used to exclude a complete architecture from # | |
226 | # mirrorring. Use as space seperated list. # | |
227 | # Possible values are: # | |
70068cad MW |
228 | # alpha, amd64, arm, armel, armhf, hppa, hurd-i386, i386, ia64, mips # |
229 | # mipsel, powerpc, s390, s390x, sparc, kfreebsd-i386, kfreebsd-amd64 # | |
db2c5b8b | 230 | # and source. # |
70068cad | 231 | # eg. ARCH_EXCLUDE="alpha arm armel mipsel mips s390 s390x sparc" # |
db2c5b8b MW |
232 | # # |
233 | # An unset value will mirror all architectures # | |
234 | ######################################################################## | |
235 | ||
236 | ######################################################################## | |
237 | # There should be nothing to edit here, use the config file # | |
238 | ######################################################################## | |
239 | MIRRORNAME=${MIRRORNAME:-$(hostname -f)} | |
240 | # Where to put logfiles in | |
241 | LOGDIR=${LOGDIR:-"${BASEDIR}/log"} | |
242 | # Our own logfile | |
243 | LOG=${LOG:-"${LOGDIR}/${NAME}.log"} | |
244 | ||
245 | # Where should we put all the mirrored files? | |
70068cad | 246 | TO=${TO:-"/srv/mirrors/debian/"} |
db2c5b8b MW |
247 | |
248 | # used by log() and error() | |
249 | PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"} | |
250 | ||
251 | # Where to send mails about mirroring to? | |
252 | if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then | |
253 | # We are not on a debian.org host | |
254 | MAILTO=${MAILTO:-"root"} | |
255 | else | |
256 | # Yay, on a .debian.org host | |
257 | MAILTO=${MAILTO:-"mirrorlogs@debian.org"} | |
258 | fi | |
259 | # Want errors only or every log? | |
260 | ERRORSONLY=${ERRORSONLY:-"true"} | |
261 | # Want full logs, ie. including the rsync one? | |
262 | FULLLOGS=${FULLLOGS:-"false"} | |
263 | ||
264 | # How many logfiles to keep | |
265 | LOGROTATE=${LOGROTATE:-14} | |
266 | ||
267 | # Our lockfile | |
268 | LOCK=${LOCK:-"${TO}/Archive-Update-in-Progress-${MIRRORNAME}"} | |
269 | # timeout for the lockfile, in case we have bash older than v4 (and no /proc) | |
270 | LOCKTIMEOUT=${LOCKTIMEOUT:-3600} | |
271 | # Do we need another rsync run? | |
272 | UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}" | |
273 | # Trace file for mirror stats and checks (make sure we get full hostname) | |
274 | TRACE=${TRACE:-"project/trace/${MIRRORNAME}"} | |
275 | ||
276 | # rsync program | |
277 | RSYNC=${RSYNC:-rsync} | |
278 | # Rsync filter rules. Used to protect various files we always want to keep, even if we otherwise delete | |
279 | # excluded files | |
280 | RSYNC_FILTER=${RSYNC_FILTER:-"--filter=protect_Archive-Update-in-Progress-${MIRRORNAME} --filter=protect_${TRACE} --filter=protect_Archive-Update-Required-${MIRRORNAME}"} | |
281 | # limit I/O bandwidth. Value is KBytes per second, unset or 0 is unlimited | |
282 | RSYNC_BW=${RSYNC_BW:-0} | |
283 | # Default rsync options for *every* rsync call | |
284 | RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --timeout 3600 --stats ${RSYNC_FILTER}"} | |
285 | # Options we only use in the first pass, where we do not want packages/sources to fly in yet and don't want to delete files | |
286 | RSYNC_OPTIONS1=${RSYNC_OPTIONS1:-"--exclude Packages* --exclude Sources* --exclude Release* --exclude InRelease --exclude ls-lR*"} | |
287 | # Options for the second pass, where we do want everything, including deletion of old and now unused files | |
288 | RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded"} | |
289 | # Which rsync share to use on our upstream mirror? | |
70068cad | 290 | RSYNC_PATH=${RSYNC_PATH:-"debian"} |
db2c5b8b MW |
291 | |
292 | # Now add the bwlimit option. As default is 0 we always add it, rsync interprets | |
293 | # 0 as unlimited, so this is safe. | |
294 | RSYNC_OPTIONS="--bwlimit=${RSYNC_BW} ${RSYNC_OPTIONS}" | |
295 | ||
296 | # We have no default host to sync from, but will error out if its unset | |
297 | RSYNC_HOST=${RSYNC_HOST:-""} | |
298 | # Error out if we have no host to sync from | |
299 | if [ -z "${RSYNC_HOST}" ]; then | |
300 | error "Missing a host to mirror from, please set RSYNC_HOST variable in ${BASEDIR}/etc/${NAME}.conf" | |
301 | fi | |
302 | ||
303 | # our username for the rsync share | |
304 | RSYNC_USER=${RSYNC_USER:-""} | |
305 | # the password | |
306 | RSYNC_PASSWORD=${RSYNC_PASSWORD:-""} | |
307 | ||
308 | # a possible proxy | |
309 | RSYNC_PROXY=${RSYNC_PROXY:-""} | |
310 | ||
311 | # Do we sync stage1? | |
312 | SYNCSTAGE1=${SYNCSTAGE1:-"false"} | |
313 | # Do we sync stage2? | |
314 | SYNCSTAGE2=${SYNCSTAGE2:-"false"} | |
315 | # Do we sync all? | |
316 | SYNCALL=${SYNCALL:-"true"} | |
317 | # Do we have a mhop sync? | |
318 | SYNCMHOP=${SYNCMHOP:-"false"} | |
319 | # Do we callback? | |
320 | SYNCCALLBACK=${SYNCCALLBACK:-"false"} | |
321 | # If we call back we need some more options defined in the config file. | |
322 | CALLBACKUSER=${CALLBACKUSER:-"archvsync"} | |
323 | CALLBACKHOST=${CALLBACKHOST:-"none"} | |
324 | CALLBACKKEY=${CALLBACKKEY:-"none"} | |
325 | ||
326 | # General excludes. Don't list architecture specific stuff here, use ARCH_EXCLUDE for that! | |
327 | EXCLUDE=${EXCLUDE:-""} | |
328 | ||
329 | # The temp directory used by rsync --delay-updates is not | |
330 | # world-readable remotely. Always exclude it to avoid errors. | |
331 | EXCLUDE="${EXCLUDE} --exclude .~tmp~/" | |
332 | ||
333 | SOURCE_EXCLUDE=${SOURCE_EXCLUDE:-""} | |
334 | ARCH_EXCLUDE=${ARCH_EXCLUDE:-""} | |
335 | # Exclude architectures defined in $ARCH_EXCLUDE | |
336 | for ARCH in ${ARCH_EXCLUDE}; do | |
337 | EXCLUDE="${EXCLUDE} --exclude binary-${ARCH}/ --exclude installer-${ARCH}/ --exclude Contents-${ARCH}.gz --exclude Contents-${ARCH}.bz2 --exclude Contents-${ARCH}.diff/ --exclude arch-${ARCH}.files --exclude arch-${ARCH}.list.gz --exclude *_${ARCH}.deb --exclude *_${ARCH}.udeb --exclude *_${ARCH}.changes" | |
338 | if [ "${ARCH}" = "source" ]; then | |
339 | if [ -z ${SOURCE_EXCLUDE} ]; then | |
70068cad | 340 | SOURCE_EXCLUDE=" --exclude source/ --exclude *.tar.gz --exclude *.diff.gz --exclude *.tar.bz2 --exclude *.tar.xz -exclude *.diff.bz2 --exclude *.dsc " |
db2c5b8b MW |
341 | fi |
342 | fi | |
343 | done | |
344 | ||
345 | # Hooks | |
346 | HOOK1=${HOOK1:-""} | |
347 | HOOK2=${HOOK2:-""} | |
348 | HOOK3=${HOOK3:-""} | |
349 | HOOK4=${HOOK4:-""} | |
350 | HOOK5=${HOOK5:-""} | |
351 | ||
352 | # Are we a hub? | |
353 | HUB=${HUB:-"false"} | |
354 | ||
355 | ######################################################################## | |
356 | # Really nothing to see below here. Only code follows. # | |
357 | ######################################################################## | |
358 | ######################################################################## | |
359 | ||
360 | # Some sane defaults | |
361 | cd "${BASEDIR}" | |
362 | umask 022 | |
363 | ||
364 | # If we are here for the first time, create the | |
365 | # destination and the trace directory | |
366 | mkdir -p "${TO}/project/trace" | |
367 | ||
368 | # Used to make sure we will have the archive fully and completly synced before | |
369 | # we stop, even if we get multiple pushes while this script is running. | |
370 | # Otherwise we can end up with a half-synced archive: | |
371 | # - get a push | |
372 | # - sync, while locked | |
373 | # - get another push. Of course no extra sync run then happens, we are locked. | |
374 | # - done. Archive not correctly synced, we don't have all the changes from the second push. | |
375 | touch "${UPDATEREQUIRED}" | |
376 | ||
377 | # Check to see if another sync is in progress | |
378 | if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then | |
379 | if [ ${BASH_VERSINFO[0]} -gt 3 ] || [ -L /proc/self ]; then | |
380 | # We have a recent enough bash version, lets do it the easy way, | |
381 | # the lock will contain the right pid, thanks to $BASHPID | |
382 | if ! $(kill -0 $(cat ${LOCK}) 2>/dev/null); then | |
383 | # Process does either not exist or is not owned by us. | |
384 | echo "$$" > "${LOCK}" | |
385 | else | |
386 | echo "Unable to start rsync, lock file still exists, PID $(cat ${LOCK})" | |
387 | exit 1 | |
388 | fi | |
389 | else | |
390 | # Old bash, means we dont have the right pid in our lockfile | |
391 | # So take a different way - guess if it is still there by comparing its age. | |
392 | # Not optimal, but hey. | |
393 | stamptime=$(date --reference="${LOCK}" +%s) | |
394 | unixtime=$(date +%s) | |
395 | difference=$(( $unixtime - $stamptime )) | |
396 | if [ ${difference} -ge ${LOCKTIMEOUT} ]; then | |
397 | # Took longer than LOCKTIMEOUT minutes? Assume it broke and take the lock | |
398 | echo "$$" > "${LOCK}" | |
399 | else | |
400 | echo "Unable to start rsync, lock file younger than one hour" | |
401 | exit 1 | |
402 | fi | |
403 | fi | |
404 | fi | |
405 | ||
406 | # When we exit normally we call cleanup on our own. Otherwise we want it called by | |
407 | # this trap. (We can not trap on EXIT, because that is called when the main script | |
408 | # exits. Which also happens when we background the mainroutine, ie. while we still | |
409 | # run!) | |
410 | trap cleanup ERR TERM HUP INT QUIT | |
411 | ||
412 | # Start log by redirecting stdout and stderr there and closing stdin | |
413 | exec >"$LOG" 2>&1 <&- | |
414 | log "Mirrorsync start" | |
415 | ||
416 | # Look who pushed us and note that in the log. | |
70068cad | 417 | SSH_CONNECTION=${SSH_CONNECTION:-""} |
db2c5b8b MW |
418 | PUSHFROM="${SSH_CONNECTION%%\ *}" |
419 | if [ -n "${PUSHFROM}" ]; then | |
420 | log "We got pushed from ${PUSHFROM}" | |
421 | fi | |
422 | ||
423 | if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then | |
424 | if [ "xnonex" = "x${CALLBACKHOST}x" ] || [ "xnonex" = "x${CALLBACKKEY}x" ]; then | |
425 | SYNCCALLBACK="false" | |
426 | error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback" | |
427 | fi | |
428 | fi | |
429 | ||
430 | HOOK=( | |
431 | HOOKNR=1 | |
432 | HOOKSCR=${HOOK1} | |
433 | ) | |
434 | hook $HOOK | |
435 | ||
436 | # Now, we might want to sync from anonymous too. | |
437 | # This is that deep in this script so hook1 could, if wanted, change things! | |
438 | if [ -z ${RSYNC_USER} ]; then | |
439 | RSYNCPTH="${RSYNC_HOST}" | |
440 | else | |
441 | RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}" | |
442 | fi | |
443 | ||
444 | # Now do the actual mirroring, and run as long as we have an updaterequired file. | |
445 | export RSYNC_PASSWORD | |
446 | export RSYNC_PROXY | |
447 | ||
448 | while [ -e "${UPDATEREQUIRED}" ]; do | |
449 | log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists" | |
450 | ||
451 | # if we want stage1 *or* all | |
452 | if [ "xtruex" = "x${SYNCSTAGE1}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then | |
453 | while [ -e "${UPDATEREQUIRED}" ]; do | |
454 | rm -f "${UPDATEREQUIRED}" | |
455 | log "Running stage1: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" | |
456 | ||
457 | set +e | |
458 | # Step one, sync everything except Packages/Releases | |
459 | ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \ | |
460 | ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error" | |
461 | result=$? | |
462 | set -e | |
463 | ||
464 | log "Back from rsync with returncode ${result}" | |
465 | done | |
466 | else | |
467 | # Fake a good resultcode | |
468 | result=0 | |
469 | fi # Sync stage 1? | |
470 | rm -f "${UPDATEREQUIRED}" | |
471 | ||
472 | set +e | |
473 | check_rsync $result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}" | |
474 | GO=$? | |
475 | set -e | |
476 | if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then | |
477 | log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" | |
478 | elif [ ${GO} -ne 0 ]; then | |
479 | exit 3 | |
480 | fi | |
481 | ||
482 | HOOK=( | |
483 | HOOKNR=2 | |
484 | HOOKSCR=${HOOK2} | |
485 | ) | |
486 | hook $HOOK | |
487 | ||
488 | # if we want stage2 *or* all | |
489 | if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then | |
490 | log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" | |
491 | ||
492 | set +e | |
493 | # We are lucky, it worked. Now do step 2 and sync again, this time including | |
494 | # the packages/releases files | |
495 | ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \ | |
496 | ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>"${LOGDIR}/rsync-${NAME}.log" 2>>"${LOGDIR}/rsync-${NAME}.error" | |
497 | result=$? | |
498 | set -e | |
499 | ||
500 | log "Back from rsync with returncode ${result}" | |
501 | else | |
502 | # Fake a good resultcode | |
503 | result=0 | |
504 | fi # Sync stage 2? | |
505 | ||
506 | set +e | |
507 | check_rsync $result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}" | |
508 | GO=$? | |
509 | set -e | |
510 | if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then | |
511 | log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" | |
512 | elif [ ${GO} -ne 0 ]; then | |
513 | exit 4 | |
514 | fi | |
515 | ||
516 | HOOK=( | |
517 | HOOKNR=3 | |
518 | HOOKSCR=${HOOK3} | |
519 | ) | |
520 | hook $HOOK | |
521 | done | |
522 | ||
523 | # We only update our tracefile when we had a stage2 or an all sync. | |
524 | # Otherwise we would update it after stage1 already, which is wrong. | |
525 | if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then | |
526 | if [ -d "$(dirname "${TO}/${TRACE}")" ]; then | |
527 | LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}" | |
528 | echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}" | |
529 | echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}" | |
530 | fi | |
531 | fi | |
532 | ||
533 | HOOK=( | |
534 | HOOKNR=4 | |
535 | HOOKSCR=${HOOK4} | |
536 | ) | |
537 | hook $HOOK | |
538 | ||
539 | if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then | |
540 | set +e | |
541 | callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}" | |
542 | set -e | |
543 | fi | |
544 | ||
545 | # Remove the Archive-Update-in-Progress file before we push our downstreams. | |
546 | rm -f "${LOCK}" | |
547 | ||
548 | if [ x${HUB} = "xtrue" ]; then | |
549 | # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop | |
550 | if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then | |
551 | RUNMIRRORARGS="" | |
552 | if [ -n "${ARCHIVE}" ]; then | |
553 | # We tell runmirrors about the archive we are running on. | |
554 | RUNMIRRORARGS="-a ${ARCHIVE}" | |
555 | fi | |
556 | # We also tell runmirrors that we are running it from within ftpsync, so it can change | |
557 | # the way it works with mhop based on that. | |
558 | RUNMIRRORARGS="${RUNMIRRORARGS} -f" | |
559 | ||
560 | if [ "xtruex" = "x${SYNCSTAGE1}x" ]; then | |
561 | # This is true when we have a mhop sync. A normal multi-stage push sending stage1 will | |
562 | # not get to this point. | |
563 | # So if that happens, tell runmirrors we are doing mhop | |
564 | RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop" | |
565 | elif [ "xtruex" = "x${SYNCSTAGE2}x" ]; then | |
566 | RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2" | |
567 | elif [ "xtruex" = "x${SYNCALL}x" ]; then | |
568 | RUNMIRRORARGS="${RUNMIRRORARGS} -k all" | |
569 | fi | |
570 | log "Trigger slave mirrors using ${RUNMIRRORARGS}" | |
571 | ${BASEDIR}/bin/runmirrors ${RUNMIRRORARGS} | |
572 | log "Trigger slave done" | |
573 | ||
574 | HOOK=( | |
575 | HOOKNR=5 | |
576 | HOOKSCR=${HOOK5} | |
577 | ) | |
578 | hook $HOOK | |
579 | fi | |
580 | fi | |
581 | ||
582 | # All done, lets call cleanup | |
583 | cleanup |