Commit | Line | Data |
---|---|---|
db2c5b8b MW |
1 | # -*- mode:sh -*- |
2 | # Little common functions | |
3 | ||
4 | # push a mirror attached to us. | |
5 | # Arguments (using an array named SIGNAL_OPTS): | |
6 | # | |
7 | # $MIRROR - Name for the mirror, also basename for the logfile | |
8 | # $HOSTNAME - Hostname to push to | |
9 | # $USERNAME - Username there | |
10 | # $SSHPROTO - Protocol version, either 1 or 2. | |
11 | # $SSHKEY - the ssh private key file to use for this push | |
12 | # $SSHOPTS - any other option ssh accepts, passed blindly, be careful | |
13 | # $PUSHLOCKOWN - own lockfile name to touch after stage1 in pushtype=staged | |
14 | # $PUSHTYPE - what kind of push should be done? | |
15 | # all - normal, just push once with ssh backgrounded and finish | |
16 | # staged - staged. first push stage1, then wait for $PUSHLOCKs to appear, | |
17 | # then push stage2 | |
18 | # $PUSHARCHIVE - what archive to sync? (Multiple mirrors behind one ssh key!) | |
19 | # $PUSHCB - do we want a callback? | |
20 | # $PUSHKIND - whats going on? are we doing mhop push or already stage2? | |
21 | # $FROMFTPSYNC - set to true if we run from within ftpsync. | |
22 | # | |
23 | # This function assumes that the variable LOG is set to a directory where | |
24 | # logfiles can be written to. | |
25 | # Additionally $PUSHLOCKS has to be defined as a set of space delimited strings | |
26 | # (list of "lock"files) to wait for if you want pushtype=staged | |
27 | # | |
28 | # Pushes might be done in background (for type all). | |
29 | signal () { | |
30 | ARGS="SIGNAL_OPTS[*]" | |
31 | local ${!ARGS} | |
32 | ||
33 | MIRROR=${MIRROR:-""} | |
34 | HOSTNAME=${HOSTNAME:-""} | |
35 | USERNAME=${USERNAME:-""} | |
36 | SSHPROTO=${SSHPROTO:-""} | |
37 | SSHKEY=${SSHKEY:-""} | |
38 | SSHOPTS=${SSHOPTS:-""} | |
39 | PUSHLOCKOWN=${PUSHLOCKOWN:-""} | |
40 | PUSHTYPE=${PUSHTYPE:-"all"} | |
41 | PUSHARCHIVE=${PUSHARCHIVE:-""} | |
42 | PUSHCB=${PUSHCB:-""} | |
43 | PUSHKIND=${PUSHKIND:-"all"} | |
44 | FROMFTPSYNC=${FROMFTPSYNC:-"false"} | |
45 | ||
46 | # And now get # back to space... | |
47 | SSHOPTS=${SSHOPTS/\#/ } | |
48 | ||
49 | # Defaults we always want, no matter what | |
50 | SSH_OPTIONS="-o user=${USERNAME} -o BatchMode=yes -o ServerAliveInterval=45 -o ConnectTimeout=45 -o PasswordAuthentication=no" | |
51 | ||
52 | # If there are userdefined ssh options, add them. | |
53 | if [ -n "${SSH_OPTS}" ]; then | |
54 | SSH_OPTIONS="${SSH_OPTIONS} ${SSH_OPTS}" | |
55 | fi | |
56 | ||
57 | # Does this machine need a special key? | |
58 | if [ -n "${SSHKEY}" ]; then | |
59 | SSH_OPTIONS="${SSH_OPTIONS} -i ${SSHKEY}" | |
60 | fi | |
61 | ||
62 | # Does this machine have an extra own set of ssh options? | |
63 | if [ -n "${SSHOPTS}" ]; then | |
64 | SSH_OPTIONS="${SSH_OPTIONS} ${SSHOPTS}" | |
65 | fi | |
66 | ||
67 | # Set the protocol version | |
68 | if [ ${SSHPROTO} -ne 1 ] && [ ${SSHPROTO} -ne 2 ] && [ ${SSHPROTO} -ne 99 ]; then | |
69 | # Idiots, we only want 1 or 2. Cant decide? Lets force 2. | |
70 | SSHPROTO=2 | |
71 | fi | |
72 | if [ -n "${SSHPROTO}" ] && [ ${SSHPROTO} -ne 99 ]; then | |
73 | SSH_OPTIONS="${SSH_OPTIONS} -${SSHPROTO}" | |
74 | fi | |
75 | ||
76 | date -u >> "${LOGDIR}/${MIRROR}.log" | |
77 | ||
78 | PUSHARGS="" | |
79 | # PUSHARCHIVE empty or not, we always add the sync:archive: command to transfer. | |
80 | # Otherwise, if nothing else is added, ssh -f would not work ("no command to execute") | |
81 | # But ftpsync does treat "sync:archive:" as the main archive, so this works nicely. | |
82 | PUSHARGS="${PUSHARGS} sync:archive:${PUSHARCHIVE}" | |
83 | ||
84 | # We have a callback wish, tell downstreams | |
85 | if [ -n "${PUSHCB}" ]; then | |
86 | PUSHARGS="${PUSHARGS} sync:callback" | |
87 | fi | |
88 | # If we are running an mhop push AND our downstream is one to receive it, tell it. | |
89 | if [ "xmhopx" = "x${PUSHKIND}x" ] && [ "xmhopx" = "x${PUSHTYPE}x" ]; then | |
90 | PUSHARGS="${PUSHARGS} sync:mhop" | |
91 | fi | |
92 | ||
93 | if [ "xallx" = "x${PUSHTYPE}x" ]; then | |
94 | # Default normal "fire and forget" push. We background that, we do not care about the mirrors doings | |
95 | echo "Sending normal push" >> "${LOGDIR}/${MIRROR}.log" | |
96 | PUSHARGS1="sync:all" | |
97 | ssh -f $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" | |
98 | elif [ "xstagedx" = "x${PUSHTYPE}x" ] || [ "xmhopx" = "x${PUSHTYPE}x" ]; then | |
99 | # Want a staged push. Fine, lets do that. Not backgrounded. We care about the mirrors doings. | |
100 | echo "Sending staged push" >> "${LOGDIR}/${MIRROR}.log" | |
101 | ||
102 | # Only send stage1 if we havent already send it. When called with stage2, we already did. | |
103 | if [ "xstage2x" != "x${PUSHKIND}x" ]; then | |
104 | # Step1: Do a push to only sync stage1, do not background | |
105 | PUSHARGS1="sync:stage1" | |
106 | ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 | |
107 | touch "${PUSHLOCKOWN}" | |
108 | ||
109 | # Step2: Wait for all the other "lock"files to appear. | |
110 | tries=0 | |
111 | # We do not wait forever | |
112 | while [ ${tries} -lt ${PUSHDELAY} ]; do | |
113 | total=0 | |
114 | found=0 | |
115 | for file in ${PUSHLOCKS}; do | |
116 | total=$((total + 1)) | |
117 | if [ -f ${file} ]; then | |
118 | found=$((found + 1)) | |
119 | fi | |
120 | done | |
121 | if [ ${total} -eq ${found} ] || [ -f "${LOCKDIR}/all_stage1" ]; then | |
122 | touch "${LOCKDIR}/all_stage1" | |
123 | break | |
124 | fi | |
125 | tries=$((tries + 5)) | |
126 | sleep 5 | |
127 | done | |
128 | # In case we did not have all PUSHLOCKS and still continued, note it | |
129 | # This is a little racy, especially if the other parts decide to do this | |
130 | # at the same time, but it wont hurt more than a mail too much, so I don't care much | |
131 | if [ ${tries} -ge ${PUSHDELAY} ]; then | |
132 | echo "Failed to wait for all other mirrors. Failed ones are:" >> "${LOGDIR}/${MIRROR}.log" | |
133 | for file in ${PUSHLOCKS}; do | |
134 | if [ ! -f ${file} ]; then | |
135 | echo "${file}" >> "${LOGDIR}/${MIRROR}.log" | |
136 | error "Missing Pushlockfile ${file} after waiting ${tries} second, continuing" | |
137 | fi | |
138 | done | |
139 | fi | |
140 | rm -f "${PUSHLOCKOWN}" | |
141 | fi | |
142 | ||
143 | # Step3: It either timed out or we have all the "lock"files, do the rest | |
144 | # If we are doing mhop AND are called from ftpsync - we now exit. | |
145 | # That way we notify our uplink that we and all our clients are done with their | |
146 | # stage1. It can then finish its own, and if all our upstreams downlinks are done, | |
147 | # it will send us stage2. | |
148 | # If we are not doing mhop or are not called from ftpsync, we start stage2 | |
149 | if [ "xtruex" = "x${FROMFTPSYNC}x" ] && [ "xmhopx" = "x${PUSHKIND}x" ]; then | |
150 | return | |
151 | else | |
152 | PUSHARGS2="sync:stage2" | |
153 | echo "Now doing the second stage push" >> "${LOGDIR}/${MIRROR}.log" | |
154 | ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS2}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 | |
155 | fi | |
156 | else | |
157 | # Can't decide? Then you get nothing. | |
158 | return | |
159 | fi | |
160 | } | |
161 | ||
162 | # callback, used by ftpsync | |
163 | callback () { | |
164 | # Defaults we always want, no matter what | |
165 | SSH_OPTIONS="-o BatchMode=yes -o ServerAliveInterval=45 -o ConnectTimeout=45 -o PasswordAuthentication=no" | |
166 | ssh $SSH_OPTIONS -i "$3" -o"user $1" "$2" callback:${HOSTNAME} | |
167 | } | |
168 | ||
169 | # log something (basically echo it together with a timestamp) | |
170 | # | |
171 | # Set $PROGRAM to a string to have it added to the output. | |
172 | log () { | |
173 | if [ -z "${PROGRAM}" ]; then | |
174 | echo "$(date +"%b %d %H:%M:%S") $(hostname -s) [$$] $@" | |
175 | else | |
176 | echo "$(date +"%b %d %H:%M:%S") $(hostname -s) ${PROGRAM}[$$]: $@" | |
177 | fi | |
178 | } | |
179 | ||
180 | # log the message using log() but then also send a mail | |
181 | # to the address configured in MAILTO (if non-empty) | |
182 | error () { | |
183 | log "$@" | |
184 | if [ -z "${MAILTO}" ]; then | |
185 | echo "$@" | mail -e -s "[$PROGRAM@$(hostname -s)] ERROR [$$]" ${MAILTO} | |
186 | fi | |
187 | } | |
188 | ||
189 | # run a hook | |
190 | # needs array variable HOOK setup with HOOKNR being a number an HOOKSCR | |
191 | # the script to run. | |
192 | hook () { | |
193 | ARGS='HOOK[@]' | |
194 | local "${!ARGS}" | |
195 | if [ -n "${HOOKSCR}" ]; then | |
196 | log "Running hook $HOOKNR: ${HOOKSCR}" | |
197 | set +e | |
198 | "${HOOKSCR}" | |
199 | result=$? | |
200 | set -e | |
201 | log "Back from hook $HOOKNR, got returncode ${result}" | |
202 | return $result | |
203 | else | |
204 | return 0 | |
205 | fi | |
206 | } | |
207 | ||
208 | # Return the list of 2-stage mirrors. | |
209 | get2stage() { | |
210 | egrep '^(staged|mhop)' "${MIRRORS}" | { | |
211 | while read MTYPE MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do | |
212 | PUSHLOCKS="${LOCKDIR}/${MLNAME}.stage1 ${PUSHLOCKS}" | |
213 | done | |
214 | echo "$PUSHLOCKS" | |
215 | } | |
216 | } | |
217 | ||
218 | # Rotate logfiles | |
219 | savelog() { | |
220 | torotate="$1" | |
221 | count=${2:-${LOGROTATE}} | |
222 | while [ ${count} -gt 0 ]; do | |
223 | prev=$(( count - 1 )) | |
224 | if [ -e "${torotate}.${prev}" ]; then | |
225 | mv "${torotate}.${prev}" "${torotate}.${count}" | |
226 | fi | |
227 | count=$prev | |
228 | done | |
229 | mv "${torotate}" "${torotate}.0" | |
230 | } |