2 # Anon rsync partial mirror of Debian with package pool support.
3 # Copyright 1999, 2000 by Joey Hess <joeyh@debian.org>, GPL'd.
4 # Beaten on by Colin Watson <cjwatson@debian.org>.
6 # Flags to pass to rsync. More can be specified on the command line.
7 # These flags are always passed to rsync:
8 FLAGS="$@ -rLpt --partial"
9 # These flags are not passed in when we are getting files from pools.
10 # In particular, --delete is a horrid idea at that point, but good here.
11 FLAGS_NOPOOL="$FLAGS --exclude Packages --delete"
12 # And these flags are passed in only when we are getting files from pools.
13 # Remember, do _not_ include --delete.
15 # The host to connect to. Currently must carry both non-us and main
16 # and support anon rsync, which limits the options somewhat.
17 HOST=ftp.uk.debian.org
18 # Where to put the mirror (absolute path, please):
20 # The distribution to mirror:
22 # Architecture to mirror:
24 # Should source be mirrored too?
26 # The sections to mirror (main, non-free, etc):
27 SECTIONS="main contrib non-free"
28 # Should a contents file kept up to date?
30 # Should symlinks be generated to every deb, in an "all" directory?
31 # I find this is very handy to ease looking up deb filenames.
34 ###############################################################################
36 mkdir -p $DEST/dists $DEST/pool $DEST/non-US/pool
38 # Snarf the contents file.
39 if [ "$CONTENTS" = yes ]; then
42 $HOST::debian/dists/$DIST/Contents-${ARCH}.gz \
46 if [ "$SOURCE" = yes ]; then
52 # Download packages files (and .debs and sources too, until we move fully
54 for type in binary-${ARCH} $SOURCE; do
55 for section in $SECTIONS; do
56 mkdir -p $DEST/non-US/dists/$DIST/non-US/$section/$type
58 $HOST::debian/non-US/dists/$DIST/non-US/$section/$type \
59 $DEST/non-US/dists/$DIST/non-US/$section/
60 mkdir -p $DEST/dists/$DIST/$section/$type
62 $HOST::debian/dists/$DIST/$section/$type \
63 $DEST/dists/$DIST/$section/
67 # Update the non-US package pool.
68 # TODO: probably needs to be optimized, we'll see as time goes by..
69 cd $DEST/non-US/pool || exit 1
72 # Get a list of all the files that are in the pool based on the Packages
73 # files that were already updated. Thanks to aj for the awk-fu.
74 for file in `find $DEST/non-US -name Packages.gz | \
75 xargs -r zgrep -i ^Filename: | cut -d ' ' -f 2 | grep ^pool/` \
76 `find $DEST/non-US -name Sources.gz | xargs -r zcat | \
77 awk '/^Directory:/ {D=$2} /Files:/,/^$/ { \
78 if ($1 != "Files:" && $0 != "") print D "/" $3; \
81 DIRS="`dirname $file` $DIRS"
82 echo $file >> .filelist
85 # Remove leading "pool" from all files in the file list.
86 # The "./" we change it to is there so the file names
87 # exactly match in the delete step and the files that get downloaded
89 sed 's!^pool/!./!' .filelist > .filelist.new
90 mv -f .filelist.new .filelist
92 (cd .. && mkdir -p $DIRS)
93 # Tell rsync to download only the files in the list. The exclude is here
94 # to make the recursion not get anything else.
95 # TODO: main pool needs to be donwloaded from too, once there is one.
97 $HOST::debian/non-US/pool/ --include-from .filelist --exclude '*' .
98 echo non-US pool rsynced
100 # Delete all files that are not in the list, then any empty directories.
101 # This also kills the filelist.
102 find -type f | fgrep -vxf .filelist | xargs -r rm -f
103 find -type d -empty | xargs -r rmdir -p --ignore-fail-on-non-empty
104 # End of non-US package pool update.
106 # Update the package pool.
107 # TODO: probably needs to be optimized, we'll see as time goes by..
108 cd $DEST/pool || exit 1
111 # Get a list of all the files that are in the pool based on the Packages
112 # files that were already updated. Thanks to aj for the awk-fu.
113 for file in `find $DEST -name non-US -prune -o -name Packages.gz | \
114 xargs -r zgrep -i ^Filename: | cut -d ' ' -f 2 | grep ^pool/` \
115 `find $DEST -name non-US -prune -o -name Sources.gz | \
117 awk '/^Directory:/ {D=$2} /Files:/,/^$/ { \
118 if ($1 != "Files:" && $0 != "") print D "/" $3; \
121 DIRS="`dirname $file` $DIRS"
122 echo $file >> .filelist
125 # Remove leading "pool" from all files in the file list.
126 # The "./" we change it to is there so the file names
127 # exactly match in the delete step and the files that get downloaded
129 sed 's!^pool/!./!' .filelist > .filelist.new
130 mv -f .filelist.new .filelist
132 (cd .. && mkdir -p $DIRS)
133 # Tell rsync to download only the files in the list. The exclude is here
134 # to make the recursion not get anything else.
135 # TODO: main pool needs to be donwloaded from too, once there is one.
137 $HOST::debian/pool/ --include-from .filelist --exclude '*' .
138 # Delete all files that are not in the list, then any empty directories.
139 # This also kills the filelist.
140 find -type f | fgrep -vxf .filelist | xargs -r rm -f
141 find -type d -empty | xargs -r rmdir -p --ignore-fail-on-non-empty
142 # End of package pool update.
144 # Update symlinks (I like to have a link to every .deb in one directory).
145 if [ "$SYMLINK_FARM" = yes ]; then
147 cd $DEST/all || exit 1
148 find -name \*.deb | xargs -r rm -f
149 find .. -name "*.deb" -type f | grep -v ^../all | \
150 xargs -r -i ln -sf {} .
153 # Waste bandwidth. Put a partial mirror on your laptop today!