Commit | Line | Data |
---|---|---|
57ab47fa MW |
1 | #! /bin/bash |
2 | # No, we can not deal with sh alone. | |
3 | ||
4 | set -e | |
5 | set -u | |
6 | # ERR traps should be inherited from functions too. (And command | |
7 | # substitutions and subshells and whatnot, but for us the function is | |
8 | # the important part here) | |
9 | set -E | |
10 | ||
11 | # ftpsync script for Debian | |
12 | # Based losely on a number of existing scripts, written by an | |
13 | # unknown number of different people over the years. | |
14 | # | |
2e267ae8 | 15 | # Copyright (C) 2008-2012 Joerg Jaspert <joerg@debian.org> |
57ab47fa MW |
16 | # |
17 | # This program is free software; you can redistribute it and/or | |
18 | # modify it under the terms of the GNU General Public License as | |
19 | # published by the Free Software Foundation; version 2. | |
20 | # | |
21 | # This program is distributed in the hope that it will be useful, but | |
22 | # WITHOUT ANY WARRANTY; without even the implied warranty of | |
23 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
24 | # General Public License for more details. | |
25 | # | |
26 | # You should have received a copy of the GNU General Public License | |
27 | # along with this program; if not, write to the Free Software | |
28 | # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. | |
29 | ||
30 | # In case the admin somehow wants to have this script located someplace else, | |
31 | # he can set BASEDIR, and we will take that. If it is unset we take ${HOME} | |
32 | # How the admin sets this isn't our place to deal with. One could use a wrapper | |
33 | # for that. Or pam_env. Or whatever fits in the local setup. :) | |
34 | BASEDIR=${BASEDIR:-"${HOME}"} | |
35 | ||
36 | # Script version. DO NOT CHANGE, *unless* you change the master copy maintained | |
37 | # by Joerg Jaspert and the Debian mirroradm group. | |
38 | # This is used to track which mirror is using which script version. | |
2e267ae8 | 39 | VERSION="20130605" |
57ab47fa MW |
40 | |
41 | # Source our common functions | |
42 | . "${BASEDIR}/etc/common" | |
43 | ||
44 | ######################################################################## | |
45 | ######################################################################## | |
46 | ## functions ## | |
47 | ######################################################################## | |
48 | ######################################################################## | |
49 | # We want to be able to get told what kind of sync we should do. This | |
50 | # might be anything, from the archive to sync, the stage to do, etc. A | |
51 | # list of currently understood and valid options is below. Multiple | |
52 | # options are seperated by space. All the words have to have the word | |
53 | # sync: in front or nothing will get used! | |
54 | # | |
55 | # Option Behaviour | |
56 | # stage1 Only do stage1 sync | |
57 | # stage2 Only do stage2 sync | |
58 | # all Do a complete sync | |
59 | # mhop Do a mhop sync, usually additionally to stage1 | |
60 | # archive:foo Sync archive foo (if config for foo is available) | |
61 | # callback Call back when done (needs proper ssh setup for this to | |
62 | # work). It will always use the "command" callback:$HOSTNAME | |
63 | # where $HOSTNAME is the one defined below/in config and | |
64 | # will happen before slave mirrors are triggered. | |
65 | # | |
66 | # So to get us to sync all of the archive behind bpo and call back when | |
67 | # we are done, a trigger command of | |
68 | # "ssh $USER@$HOST sync:all sync:archive:bpo sync:callback" will do the | |
69 | # trick. | |
70 | check_commandline() { | |
71 | while [ $# -gt 0 ]; do | |
72 | case "$1" in | |
73 | sync:stage1) | |
74 | SYNCSTAGE1="true" | |
75 | SYNCALL="false" | |
76 | ;; | |
77 | sync:stage2) | |
78 | SYNCSTAGE2="true" | |
79 | SYNCALL="false" | |
80 | ;; | |
81 | sync:callback) | |
82 | SYNCCALLBACK="true" | |
83 | ;; | |
84 | sync:archive:*) | |
85 | ARCHIVE=${1##sync:archive:} | |
86 | # We do not like / or . in the remotely supplied archive name. | |
87 | ARCHIVE=${ARCHIVE//\/} | |
88 | ARCHIVE=${ARCHIVE//.} | |
89 | ;; | |
90 | sync:all) | |
91 | SYNCALL="true" | |
92 | ;; | |
93 | sync:mhop) | |
94 | SYNCMHOP="true" | |
95 | ;; | |
96 | *) | |
97 | echo "Unknown option ${1} ignored" | |
98 | ;; | |
99 | esac | |
100 | shift # Check next set of parameters. | |
101 | done | |
102 | } | |
103 | ||
104 | # All the stuff we want to do when we exit, no matter where | |
105 | cleanup() { | |
106 | trap - ERR TERM HUP INT QUIT EXIT | |
107 | # all done. Mail the log, exit. | |
108 | log "Mirrorsync done"; | |
109 | ||
110 | # Lets get a statistical value | |
111 | SPEED="unknown" | |
112 | if [ -f "${LOGDIR}/rsync-${NAME}.log" ]; then | |
113 | SPEED=$( | |
114 | SPEEDLINE=$(egrep '[0-9.]+ bytes/sec' "${LOGDIR}/rsync-${NAME}.log") | |
115 | set "nothing" ${SPEEDLINE} | |
116 | echo ${8:-""} | |
117 | ) | |
118 | if [ -n "${SPEED}" ]; then | |
119 | SPEED=${SPEED%%.*} | |
120 | SPEED=$(( $SPEED / 1024 )) | |
121 | fi | |
122 | fi | |
123 | log "Rsync transfer speed: ${SPEED} KB/s" | |
124 | ||
125 | if [ -n "${MAILTO}" ]; then | |
126 | # In case rsync had something on stderr | |
127 | if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then | |
128 | mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error" | |
129 | fi | |
130 | if [ "x${ERRORSONLY}x" = "xfalsex" ]; then | |
131 | # And the normal log | |
132 | MAILFILES="${LOG}" | |
133 | if [ "x${FULLLOGS}x" = "xtruex" ]; then | |
134 | # Someone wants full logs including rsync | |
135 | MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log" | |
136 | fi | |
137 | cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] archive sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} | |
138 | fi | |
139 | fi | |
140 | ||
141 | savelog "${LOGDIR}/rsync-${NAME}.log" | |
142 | savelog "${LOGDIR}/rsync-${NAME}.error" | |
143 | savelog "$LOG" > /dev/null | |
144 | ||
145 | rm -f "${LOCK}" | |
146 | } | |
147 | ||
148 | # Check rsyncs return value | |
149 | check_rsync() { | |
150 | ret=$1 | |
151 | msg=$2 | |
152 | ||
153 | # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED | |
154 | # and us re-running. If it's not, uplink is broken anyways. | |
155 | case "${ret}" in | |
156 | 0) return 0;; | |
157 | 24) return 0;; | |
158 | 23) return 2;; | |
159 | 30) return 2;; | |
160 | *) | |
161 | error "ERROR: ${msg}" | |
162 | return 1 | |
163 | ;; | |
164 | esac | |
165 | } | |
166 | ||
167 | ######################################################################## | |
168 | ######################################################################## | |
169 | ||
170 | ||
171 | # As what are we called? | |
172 | NAME="$(basename $0)" | |
173 | # The original command line arguments need to be saved! | |
174 | if [ $# -gt 0 ]; then | |
175 | ORIGINAL_COMMAND=$* | |
176 | else | |
177 | ORIGINAL_COMMAND="" | |
178 | fi | |
179 | ||
180 | SSH_ORIGINAL_COMMAND=${SSH_ORIGINAL_COMMAND:-""} | |
181 | # Now, check if we got told about stuff via ssh | |
182 | if [ -n "${SSH_ORIGINAL_COMMAND}" ]; then | |
183 | # We deliberately add "nothing" and ignore it right again, to avoid | |
184 | # people from outside putting some set options in the first place, | |
185 | # making us parse them... | |
186 | set "nothing" "${SSH_ORIGINAL_COMMAND}" | |
187 | shift | |
188 | # Yes, unqouted $* here. Or the function will only see it as one | |
189 | # parameter, which doesnt help the case in it. | |
190 | check_commandline $* | |
191 | fi | |
192 | ||
193 | # Now, we can locally override all the above variables by just putting | |
194 | # them into the .ssh/authorized_keys file forced command. | |
195 | if [ -n "${ORIGINAL_COMMAND}" ]; then | |
196 | set ${ORIGINAL_COMMAND} | |
197 | check_commandline $* | |
198 | fi | |
199 | ||
200 | # If we have been told to do stuff for a different archive than default, | |
201 | # set the name accordingly. | |
202 | ARCHIVE=${ARCHIVE:-""} | |
203 | if [ -n "${ARCHIVE}" ]; then | |
204 | NAME="${NAME}-${ARCHIVE}" | |
205 | fi | |
206 | ||
207 | # Now source the config for the archive we run on. | |
208 | # (Yes, people can also overwrite the options above in the config file | |
209 | # if they want to) | |
210 | if [ -f "${BASEDIR}/etc/${NAME}.conf" ]; then | |
211 | . "${BASEDIR}/etc/${NAME}.conf" | |
212 | else | |
213 | echo "Nono, you can't tell us about random archives. Bad boy!" | |
214 | exit 1 | |
215 | fi | |
216 | ||
217 | ######################################################################## | |
218 | # Config options go here. Feel free to overwrite them in the config # | |
219 | # file if you need to. # | |
220 | # On debian.org machines the defaults should be ok. # | |
221 | # # | |
222 | # The following extra variables can be defined in the config file: # | |
223 | # # | |
224 | # ARCH_EXCLUDE # | |
225 | # can be used to exclude a complete architecture from # | |
226 | # mirrorring. Use as space seperated list. # | |
227 | # Possible values are: # | |
228 | # alpha, amd64, arm, armel, armhf, hppa, hurd-i386, i386, ia64, mips # | |
229 | # mipsel, powerpc, s390, s390x, sparc, kfreebsd-i386, kfreebsd-amd64 # | |
230 | # and source. # | |
231 | # eg. ARCH_EXCLUDE="alpha arm armel mipsel mips s390 s390x sparc" # | |
232 | # # | |
233 | # An unset value will mirror all architectures # | |
234 | ######################################################################## | |
235 | ||
236 | ######################################################################## | |
237 | # There should be nothing to edit here, use the config file # | |
238 | ######################################################################## | |
239 | MIRRORNAME=${MIRRORNAME:-$(hostname -f)} | |
240 | # Where to put logfiles in | |
241 | LOGDIR=${LOGDIR:-"${BASEDIR}/log"} | |
242 | # Our own logfile | |
243 | LOG=${LOG:-"${LOGDIR}/${NAME}.log"} | |
244 | ||
245 | # Where should we put all the mirrored files? | |
246 | TO=${TO:-"/srv/mirrors/debian/"} | |
247 | ||
248 | # used by log() and error() | |
249 | PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"} | |
250 | ||
251 | # Where to send mails about mirroring to? | |
252 | if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then | |
253 | # We are not on a debian.org host | |
254 | MAILTO=${MAILTO:-"root"} | |
255 | else | |
256 | # Yay, on a .debian.org host | |
257 | MAILTO=${MAILTO:-"mirrorlogs@debian.org"} | |
258 | fi | |
259 | # Want errors only or every log? | |
260 | ERRORSONLY=${ERRORSONLY:-"true"} | |
261 | # Want full logs, ie. including the rsync one? | |
262 | FULLLOGS=${FULLLOGS:-"false"} | |
263 | ||
264 | # How many logfiles to keep | |
265 | LOGROTATE=${LOGROTATE:-14} | |
266 | ||
267 | # Our lockfile | |
268 | LOCK=${LOCK:-"${TO}/Archive-Update-in-Progress-${MIRRORNAME}"} | |
269 | # timeout for the lockfile, in case we have bash older than v4 (and no /proc) | |
270 | LOCKTIMEOUT=${LOCKTIMEOUT:-3600} | |
2e267ae8 MW |
271 | # sleeping time when an AUIP file is found but is not ours |
272 | UIPSLEEP=${UIPSLEEP:-1200} | |
273 | # retries whenever an upstream (or possibly stale) AUIP file is found | |
274 | UIPRETRIES=${UIPRETRIES:-3} | |
57ab47fa MW |
275 | # Do we need another rsync run? |
276 | UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}" | |
277 | # Trace file for mirror stats and checks (make sure we get full hostname) | |
278 | TRACE=${TRACE:-"project/trace/${MIRRORNAME}"} | |
2e267ae8 MW |
279 | # The trace file can have different format/contents. Here you can select |
280 | # what it will be. | |
281 | # Possible values are | |
282 | # "full" - all information | |
283 | # "terse" - basic, timestamp only (date -u) | |
284 | # "touch" - just touch the file in existance | |
285 | # "none" - no tracefile at all | |
286 | # | |
287 | # Default and required value for Debian mirrors is full. | |
288 | EXTENDEDTRACE=${EXTENDEDTRACE:-"full"} | |
57ab47fa MW |
289 | |
290 | # rsync program | |
291 | RSYNC=${RSYNC:-rsync} | |
292 | # Rsync filter rules. Used to protect various files we always want to keep, even if we otherwise delete | |
293 | # excluded files | |
294 | RSYNC_FILTER=${RSYNC_FILTER:-"--filter=protect_Archive-Update-in-Progress-${MIRRORNAME} --filter=protect_${TRACE} --filter=protect_Archive-Update-Required-${MIRRORNAME}"} | |
295 | # limit I/O bandwidth. Value is KBytes per second, unset or 0 is unlimited | |
296 | RSYNC_BW=${RSYNC_BW:-0} | |
2e267ae8 MW |
297 | RSYNC_PROTOCOL=$(rsync_protocol) |
298 | ||
299 | # Set the delete method to --delete-delay if protocol version is 30 or | |
300 | # greater (meaning rsync 3.0.0 or greater is used). Use --delete-after | |
301 | # otherwise. | |
302 | if [ 30 -le $RSYNC_PROTOCOL ]; then | |
303 | RSYNC_DELETE_METHOD=delay | |
304 | else | |
305 | RSYNC_DELETE_METHOD=after | |
306 | fi | |
307 | ||
57ab47fa MW |
308 | # Default rsync options for *every* rsync call |
309 | RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --timeout 3600 --stats ${RSYNC_FILTER}"} | |
310 | # Options we only use in the first pass, where we do not want packages/sources to fly in yet and don't want to delete files | |
2e267ae8 | 311 | RSYNC_OPTIONS1=${RSYNC_OPTIONS1:-"--exclude=Packages* --exclude=Sources* --exclude=Release* --exclude=InRelease --exclude=i18n/* --exclude=ls-lR*"} |
57ab47fa | 312 | # Options for the second pass, where we do want everything, including deletion of old and now unused files |
2e267ae8 | 313 | RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-excluded"} |
57ab47fa MW |
314 | # Which rsync share to use on our upstream mirror? |
315 | RSYNC_PATH=${RSYNC_PATH:-"debian"} | |
316 | ||
2e267ae8 MW |
317 | # Extra rsync options as defined by the admin locally. Won't be set |
318 | # to any default by ftpsync. Those will be added to EACH AND EVERY rsync call. | |
319 | RSYNC_EXTRA=${RSYNC_EXTRA:-""} | |
320 | ||
57ab47fa MW |
321 | # Now add the bwlimit option. As default is 0 we always add it, rsync interprets |
322 | # 0 as unlimited, so this is safe. | |
2e267ae8 MW |
323 | RSYNC_OPTIONS="${RSYNC_EXTRA} --bwlimit=${RSYNC_BW} ${RSYNC_OPTIONS}" |
324 | ||
325 | # Finally, make sure RSYNC_OPTIONS2 has either --delete-after or --deleter-delay | |
326 | RSYNC_OPTION_REGEX="--delete-(after|delay)" | |
327 | if ! [[ ${RSYNC_OPTIONS2} =~ ${RSYNC_OPTION_REGEX} ]]; then | |
328 | RSYNC_OPTIONS2+=" --delete-${RSYNC_DELETE_METHOD}" | |
329 | fi | |
330 | unset RSYNC_OPTION_REGEX | |
57ab47fa MW |
331 | |
332 | # We have no default host to sync from, but will error out if its unset | |
333 | RSYNC_HOST=${RSYNC_HOST:-""} | |
334 | # Error out if we have no host to sync from | |
335 | if [ -z "${RSYNC_HOST}" ]; then | |
336 | error "Missing a host to mirror from, please set RSYNC_HOST variable in ${BASEDIR}/etc/${NAME}.conf" | |
337 | fi | |
338 | ||
339 | # our username for the rsync share | |
340 | RSYNC_USER=${RSYNC_USER:-""} | |
341 | # the password | |
342 | RSYNC_PASSWORD=${RSYNC_PASSWORD:-""} | |
343 | ||
344 | # a possible proxy | |
345 | RSYNC_PROXY=${RSYNC_PROXY:-""} | |
346 | ||
347 | # Do we sync stage1? | |
348 | SYNCSTAGE1=${SYNCSTAGE1:-"false"} | |
349 | # Do we sync stage2? | |
350 | SYNCSTAGE2=${SYNCSTAGE2:-"false"} | |
351 | # Do we sync all? | |
352 | SYNCALL=${SYNCALL:-"true"} | |
353 | # Do we have a mhop sync? | |
354 | SYNCMHOP=${SYNCMHOP:-"false"} | |
2e267ae8 | 355 | # Do we callback? (May get changed later) |
57ab47fa MW |
356 | SYNCCALLBACK=${SYNCCALLBACK:-"false"} |
357 | # If we call back we need some more options defined in the config file. | |
358 | CALLBACKUSER=${CALLBACKUSER:-"archvsync"} | |
359 | CALLBACKHOST=${CALLBACKHOST:-"none"} | |
360 | CALLBACKKEY=${CALLBACKKEY:-"none"} | |
361 | ||
362 | # General excludes. Don't list architecture specific stuff here, use ARCH_EXCLUDE for that! | |
363 | EXCLUDE=${EXCLUDE:-""} | |
364 | ||
365 | # The temp directory used by rsync --delay-updates is not | |
2e267ae8 MW |
366 | # world-readable remotely. Always exclude it to avoid errors. |
367 | EXCLUDE="${EXCLUDE} --exclude=.~tmp~/" | |
57ab47fa MW |
368 | |
369 | SOURCE_EXCLUDE=${SOURCE_EXCLUDE:-""} | |
370 | ARCH_EXCLUDE=${ARCH_EXCLUDE:-""} | |
371 | # Exclude architectures defined in $ARCH_EXCLUDE | |
372 | for ARCH in ${ARCH_EXCLUDE}; do | |
2e267ae8 | 373 | EXCLUDE="${EXCLUDE} --exclude=binary-${ARCH}/ --exclude=installer-${ARCH}/ --exclude=Contents-${ARCH}.gz --exclude=Contents-udeb-${ARCH}.gz --exclude=Contents-${ARCH}.diff/ --exclude=arch-${ARCH}.files --exclude=arch-${ARCH}.list.gz --exclude=*_${ARCH}.deb --exclude=*_${ARCH}.udeb --exclude=*_${ARCH}.changes" |
57ab47fa MW |
374 | if [ "${ARCH}" = "source" ]; then |
375 | if [ -z ${SOURCE_EXCLUDE} ]; then | |
2e267ae8 | 376 | SOURCE_EXCLUDE=" --exclude=source/ --exclude=*.tar.gz --exclude=*.diff.gz --exclude=*.tar.bz2 --exclude=*.tar.xz --exclude=*.diff.bz2 --exclude=*.dsc " |
57ab47fa MW |
377 | fi |
378 | fi | |
379 | done | |
380 | ||
381 | # Hooks | |
382 | HOOK1=${HOOK1:-""} | |
383 | HOOK2=${HOOK2:-""} | |
384 | HOOK3=${HOOK3:-""} | |
385 | HOOK4=${HOOK4:-""} | |
386 | HOOK5=${HOOK5:-""} | |
387 | ||
388 | # Are we a hub? | |
389 | HUB=${HUB:-"false"} | |
390 | ||
391 | ######################################################################## | |
392 | # Really nothing to see below here. Only code follows. # | |
393 | ######################################################################## | |
394 | ######################################################################## | |
395 | ||
396 | # Some sane defaults | |
397 | cd "${BASEDIR}" | |
398 | umask 022 | |
399 | ||
400 | # If we are here for the first time, create the | |
401 | # destination and the trace directory | |
402 | mkdir -p "${TO}/project/trace" | |
403 | ||
404 | # Used to make sure we will have the archive fully and completly synced before | |
405 | # we stop, even if we get multiple pushes while this script is running. | |
406 | # Otherwise we can end up with a half-synced archive: | |
407 | # - get a push | |
408 | # - sync, while locked | |
409 | # - get another push. Of course no extra sync run then happens, we are locked. | |
410 | # - done. Archive not correctly synced, we don't have all the changes from the second push. | |
411 | touch "${UPDATEREQUIRED}" | |
412 | ||
413 | # Check to see if another sync is in progress | |
414 | if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then | |
415 | if [ ${BASH_VERSINFO[0]} -gt 3 ] || [ -L /proc/self ]; then | |
416 | # We have a recent enough bash version, lets do it the easy way, | |
417 | # the lock will contain the right pid, thanks to $BASHPID | |
2e267ae8 | 418 | if ! $(kill -0 $(< ${LOCK}) 2>/dev/null); then |
57ab47fa MW |
419 | # Process does either not exist or is not owned by us. |
420 | echo "$$" > "${LOCK}" | |
421 | else | |
2e267ae8 | 422 | echo "Unable to start rsync, lock file still exists, PID $(< ${LOCK})" |
57ab47fa MW |
423 | exit 1 |
424 | fi | |
425 | else | |
426 | # Old bash, means we dont have the right pid in our lockfile | |
427 | # So take a different way - guess if it is still there by comparing its age. | |
428 | # Not optimal, but hey. | |
429 | stamptime=$(date --reference="${LOCK}" +%s) | |
430 | unixtime=$(date +%s) | |
431 | difference=$(( $unixtime - $stamptime )) | |
432 | if [ ${difference} -ge ${LOCKTIMEOUT} ]; then | |
433 | # Took longer than LOCKTIMEOUT minutes? Assume it broke and take the lock | |
434 | echo "$$" > "${LOCK}" | |
435 | else | |
436 | echo "Unable to start rsync, lock file younger than one hour" | |
437 | exit 1 | |
438 | fi | |
439 | fi | |
440 | fi | |
441 | ||
442 | # When we exit normally we call cleanup on our own. Otherwise we want it called by | |
443 | # this trap. (We can not trap on EXIT, because that is called when the main script | |
444 | # exits. Which also happens when we background the mainroutine, ie. while we still | |
445 | # run!) | |
446 | trap cleanup ERR TERM HUP INT QUIT | |
447 | ||
448 | # Start log by redirecting stdout and stderr there and closing stdin | |
449 | exec >"$LOG" 2>&1 <&- | |
450 | log "Mirrorsync start" | |
451 | ||
452 | # Look who pushed us and note that in the log. | |
453 | SSH_CONNECTION=${SSH_CONNECTION:-""} | |
454 | PUSHFROM="${SSH_CONNECTION%%\ *}" | |
455 | if [ -n "${PUSHFROM}" ]; then | |
456 | log "We got pushed from ${PUSHFROM}" | |
457 | fi | |
458 | ||
459 | if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then | |
460 | if [ "xnonex" = "x${CALLBACKHOST}x" ] || [ "xnonex" = "x${CALLBACKKEY}x" ]; then | |
461 | SYNCCALLBACK="false" | |
462 | error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback" | |
463 | fi | |
464 | fi | |
465 | ||
466 | HOOK=( | |
467 | HOOKNR=1 | |
468 | HOOKSCR=${HOOK1} | |
469 | ) | |
470 | hook $HOOK | |
471 | ||
472 | # Now, we might want to sync from anonymous too. | |
473 | # This is that deep in this script so hook1 could, if wanted, change things! | |
474 | if [ -z ${RSYNC_USER} ]; then | |
475 | RSYNCPTH="${RSYNC_HOST}" | |
476 | else | |
477 | RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}" | |
478 | fi | |
479 | ||
480 | # Now do the actual mirroring, and run as long as we have an updaterequired file. | |
481 | export RSYNC_PASSWORD | |
482 | export RSYNC_PROXY | |
483 | ||
2e267ae8 MW |
484 | UPDATE_RETRIES=0 |
485 | ||
57ab47fa MW |
486 | while [ -e "${UPDATEREQUIRED}" ]; do |
487 | log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists" | |
488 | ||
489 | # if we want stage1 *or* all | |
490 | if [ "xtruex" = "x${SYNCSTAGE1}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then | |
491 | while [ -e "${UPDATEREQUIRED}" ]; do | |
492 | rm -f "${UPDATEREQUIRED}" | |
493 | log "Running stage1: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" | |
494 | ||
495 | set +e | |
496 | # Step one, sync everything except Packages/Releases | |
497 | ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \ | |
498 | ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error" | |
499 | result=$? | |
500 | set -e | |
501 | ||
502 | log "Back from rsync with returncode ${result}" | |
503 | done | |
504 | else | |
505 | # Fake a good resultcode | |
506 | result=0 | |
507 | fi # Sync stage 1? | |
508 | rm -f "${UPDATEREQUIRED}" | |
509 | ||
510 | set +e | |
511 | check_rsync $result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}" | |
512 | GO=$? | |
513 | set -e | |
514 | if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then | |
515 | log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" | |
516 | elif [ ${GO} -ne 0 ]; then | |
517 | exit 3 | |
518 | fi | |
519 | ||
520 | HOOK=( | |
521 | HOOKNR=2 | |
522 | HOOKSCR=${HOOK2} | |
523 | ) | |
524 | hook $HOOK | |
525 | ||
526 | # if we want stage2 *or* all | |
527 | if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then | |
2e267ae8 MW |
528 | upstream_uip=false |
529 | for aupfile in "${TO}/Archive-Update-in-Progress-"*; do | |
530 | case "$aupfile" in | |
531 | "${TO}/Archive-Update-in-Progress-*") | |
532 | error "Lock file is missing, this should not happen" | |
533 | ;; | |
534 | "${LOCK}") | |
535 | : | |
536 | ;; | |
537 | *) | |
538 | if [ -f "$aupfile" ]; then | |
539 | # Remove the file, it will be synced again if | |
540 | # upstream is still not done | |
541 | rm -f "$aupfile" | |
542 | else | |
543 | log "AUIP file '$aupfile' is not really a file, weird" | |
544 | fi | |
545 | upstream_uip=true | |
546 | ;; | |
547 | esac | |
548 | done | |
57ab47fa | 549 | |
2e267ae8 MW |
550 | if [ "xtruex" = "x${upstream_uip}x" ]; then |
551 | log "Upstream archive update in progress, skipping stage2" | |
552 | if [ ${UPDATE_RETRIES} -lt ${UIPRETRIES} ]; then | |
553 | log "Retrying update in ${UIPSLEEP}" | |
554 | touch "${UPDATEREQUIRED}" | |
555 | UPDATE_RETRIES=$(($UPDATE_RETRIES+1)) | |
556 | sleep "${UIPSLEEP}" | |
557 | result=0 | |
558 | else | |
559 | error "Update has been retried ${UPDATEREQUIRED} times, aborting" | |
560 | log "Perhaps upstream is still updating or there's a stale AUIP file" | |
561 | result=1 | |
562 | fi | |
563 | else | |
564 | log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" | |
565 | ||
566 | set +e | |
567 | # We are lucky, it worked. Now do step 2 and sync again, this time including | |
568 | # the packages/releases files | |
569 | ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \ | |
570 | ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>"${LOGDIR}/rsync-${NAME}.log" 2>>"${LOGDIR}/rsync-${NAME}.error" | |
571 | result=$? | |
572 | set -e | |
57ab47fa | 573 | |
2e267ae8 MW |
574 | log "Back from rsync with returncode ${result}" |
575 | fi | |
57ab47fa MW |
576 | else |
577 | # Fake a good resultcode | |
578 | result=0 | |
579 | fi # Sync stage 2? | |
580 | ||
581 | set +e | |
582 | check_rsync $result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}" | |
583 | GO=$? | |
584 | set -e | |
585 | if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then | |
586 | log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" | |
587 | elif [ ${GO} -ne 0 ]; then | |
588 | exit 4 | |
589 | fi | |
590 | ||
591 | HOOK=( | |
592 | HOOKNR=3 | |
593 | HOOKSCR=${HOOK3} | |
594 | ) | |
595 | hook $HOOK | |
596 | done | |
597 | ||
598 | # We only update our tracefile when we had a stage2 or an all sync. | |
599 | # Otherwise we would update it after stage1 already, which is wrong. | |
2e267ae8 | 600 | |
57ab47fa | 601 | if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then |
2e267ae8 MW |
602 | case ${EXTENDEDTRACE} in |
603 | none) | |
604 | log "No trace file wanted. Not creating one" | |
605 | ;; | |
606 | touch) | |
607 | log "Just touching the trace file" | |
608 | touch "${TO}/${TRACE}" | |
609 | ;; | |
610 | terse|full) | |
611 | log "Creating a ${EXTENDEDTRACE} trace file" | |
612 | if [ -d "$(dirname "${TO}/${TRACE}")" ]; then | |
613 | LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}.new" | |
614 | echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}.new" | |
615 | echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}.new" | |
616 | if [ "xfullx" = "x${EXTENDEDTRACE}x" ]; then | |
617 | GLOBALARCHLIST="source amd64 armel armhf hurd-i386 i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc" | |
618 | ||
619 | AEXCLUDE="^${ARCH_EXCLUDE// /\$|^}$" | |
620 | ARCHLIST="" | |
621 | for ARCH in ${GLOBALARCHLIST}; do | |
622 | if ! [[ ${ARCH} =~ ${AEXCLUDE} ]]; then | |
623 | ARCHLIST="${ARCHLIST} ${ARCH}" | |
624 | fi | |
625 | done | |
626 | out="GUESSED:{${ARCHLIST}}" | |
627 | echo "Architectures: ${out}" >> "${TO}/${TRACE}.new" | |
628 | echo "Upstream-mirror: ${RSYNC_HOST}" >> "${TO}/${TRACE}.new" | |
629 | fi # full trace | |
630 | mv "${TO}/${TRACE}.new" "${TO}/${TRACE}" | |
631 | fi | |
632 | ;; | |
633 | *) | |
634 | error "Unsupported EXTENDEDTRACE value configured in ${BASEDIR}/etc/${NAME}.conf, please fix" | |
635 | ;; | |
636 | esac | |
57ab47fa MW |
637 | fi |
638 | ||
2e267ae8 | 639 | |
57ab47fa MW |
640 | HOOK=( |
641 | HOOKNR=4 | |
642 | HOOKSCR=${HOOK4} | |
643 | ) | |
644 | hook $HOOK | |
645 | ||
646 | if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then | |
647 | set +e | |
648 | callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}" | |
649 | set -e | |
650 | fi | |
651 | ||
652 | # Remove the Archive-Update-in-Progress file before we push our downstreams. | |
653 | rm -f "${LOCK}" | |
654 | ||
2e267ae8 MW |
655 | # Check if there is a newer version of ftpsync. If so inform the admin, but not |
656 | # more than once every third day. | |
657 | if [ -r "${TO}/project/ftpsync/LATEST.VERSION" ]; then | |
658 | LATEST=$(< "${TO}/project/ftpsync/LATEST.VERSION") | |
659 | if ! [[ ${LATEST} =~ [0-9]+ ]]; then | |
660 | LATEST=0 | |
661 | fi | |
662 | if [ ${LATEST} -gt ${VERSION} ]; then | |
663 | if [ -n "${MAILTO}" ]; then | |
664 | difference=0 | |
665 | if [ -f "${LOGDIR}/ftpsync.newversion" ]; then | |
666 | stamptime=$(< "${LOGDIR}/ftpsync.newversion") | |
667 | unixtime=$(date +%s) | |
668 | difference=$(( $unixtime - $stamptime )) | |
669 | fi | |
670 | if [ ${difference} -ge 259200 ]; then | |
671 | # Only warn every third day | |
672 | mail -e -s "[$(hostname -s)] Update for ftpsync available" ${MAILTO} <<EOF | |
673 | Hello admin, | |
674 | ||
675 | i found that there is a new version of me available. | |
676 | Me lonely ftpsync is currently version: ${VERSION} | |
677 | New release of myself is available as: ${LATEST} | |
678 | ||
679 | Me, myself and I - and the Debian mirroradmins - would be very grateful | |
680 | if you could update me. You can find the latest version on your mirror, | |
681 | check $(hostname -s):${TO}/project/ftpsync/ftpsync-${LATEST}.tar.gz | |
682 | ||
683 | You can ensure the validity of that file by using sha512sum or md5sum | |
684 | against the available checksum files secured with a signature from the | |
685 | Debian FTPMaster signing key. | |
686 | ||
687 | EOF | |
688 | ||
689 | date +%s > "${LOGDIR}/ftpsync.newversion" | |
690 | fi | |
691 | fi | |
692 | else | |
693 | # Remove a possible stampfile | |
694 | rm -f "${LOGDIR}/ftpsync.newversion" | |
695 | fi | |
696 | fi | |
697 | ||
57ab47fa MW |
698 | if [ x${HUB} = "xtrue" ]; then |
699 | # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop | |
700 | if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then | |
701 | RUNMIRRORARGS="" | |
702 | if [ -n "${ARCHIVE}" ]; then | |
703 | # We tell runmirrors about the archive we are running on. | |
704 | RUNMIRRORARGS="-a ${ARCHIVE}" | |
705 | fi | |
706 | # We also tell runmirrors that we are running it from within ftpsync, so it can change | |
707 | # the way it works with mhop based on that. | |
708 | RUNMIRRORARGS="${RUNMIRRORARGS} -f" | |
709 | ||
710 | if [ "xtruex" = "x${SYNCSTAGE1}x" ]; then | |
711 | # This is true when we have a mhop sync. A normal multi-stage push sending stage1 will | |
712 | # not get to this point. | |
713 | # So if that happens, tell runmirrors we are doing mhop | |
714 | RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop" | |
715 | elif [ "xtruex" = "x${SYNCSTAGE2}x" ]; then | |
716 | RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2" | |
717 | elif [ "xtruex" = "x${SYNCALL}x" ]; then | |
718 | RUNMIRRORARGS="${RUNMIRRORARGS} -k all" | |
719 | fi | |
720 | log "Trigger slave mirrors using ${RUNMIRRORARGS}" | |
721 | ${BASEDIR}/bin/runmirrors ${RUNMIRRORARGS} | |
722 | log "Trigger slave done" | |
723 | ||
724 | HOOK=( | |
725 | HOOKNR=5 | |
726 | HOOKSCR=${HOOK5} | |
727 | ) | |
728 | hook $HOOK | |
729 | fi | |
730 | fi | |
731 | ||
732 | # All done, lets call cleanup | |
733 | cleanup |