Wharfie

--- a/trunk/wharfie/lib/makefile.py
+++ b/trunk/wharfie/lib/makefile.py
@@ -25,10 +25,36 @@
 # We use templates, which are checking their execution context,
 # to be sure, that target commands are really only executed
 # in the change root of the image.
+#
+# We use snar files to create incremental archives. Those files are targeted
+# to incremental backups, where the filestructure stays roughly the same. So
+# beside the name and timestamps of the files, it also relies on inodes.
+#
+# As we are packing and extracting the images for every step, the inodes are
+# constantly changing. As tar doesn't support to ignore inodes on incremental
+# backups, we are refreshing the backups in every step.
+#
+#          input snar    output snar    output tar
+# level 0: <none>        l0.snar        l0.tar
+# level 1: l0.snar       l1.snar        l1.tar
+# level 2: l1.snar       l2.snar        l2.tar
+#
+# When we prepare the rootfs for level 2, we have to do the following:
+#
+# - tar -xf l0.tar -g l0.snar
+#
+# - tar -xf l1.tar -g l1.snar
+# - cp l0.snar l1.snar
+# - tar -cf /dev/zero -g l1.snar .
+#
+# - tar -xf l2.tar -g l2.snar
+# - cp l1.snar l2.snar
+# - tar -cf /dev/zero -g l2.snar .
+#
+#################################################################################
 templateTrgCmd = "#!/bin/bash\n[ -f ../Wharfile ] && exit 2;\n%s\nexit \$$?\n"
 templateHostCmd = "#!/bin/bash\n[ ! -f ../Wharfile ] && exit 2;\n%s\nexit \$$?\n"
 
-archiveName = 'rootfs.tar';
 makeTargets = list();
 environment = list();
 finalTarget = list();
@@ -36,18 +62,15 @@
 #
 # Write a Makefile
 #
-def write_makefile(filename, dry_run, installpath='.'):
+def write_makefile(filename, dry_run, installpath='.', incremental=False):
+    backup_levels=list()
     f = open(filename, 'w');
     # write header
     f.write("ifneq (${VERBOSE},y)\n")
     f.write("Q=@\n")
     f.write("endif\n")
     f.write("\n")
-    f.write("OUTPUT_FILE=%s\n" % archiveName)
-    f.write("%s: %s\n" % (archiveName, "".join(finalTarget)));
-    f.write("\tcp $< $@\n");
-    f.write("\n");
-
+    f.write("all: %s\n\n" % (finalTarget[0]));
     
     # write all environment variables
     for env in environment:
@@ -61,7 +84,7 @@
         f.write("%s: %s\n" % (target['name'], " ".join(target['dep'])));
         # target and host commands have to extract and repack always.
         # simple commands don't do that.
-        if 'trgcmd' in target or 'hostcmd' in target:
+        if 'trgcmd' in target or 'hostcmd' in target or 'temporary' in target:
             f.write("\t${Q}-mkdir $$(basename $@ .tar)\n");
 
             if 'trgcmd' in target:
@@ -69,32 +92,53 @@
                 f.write("\t${Q}echo '******************************'\n");
                 f.write("\t${Q}echo '%s'\n" % target['comment']);
                 f.write("\t${Q}echo '******************************'\n");
-                f.write("\t${Q}(echo -e \"%s\") | tee ./$$(basename $@ .tar)/.trg.sh\n" % cmd.replace('\\n', '\\\\\\n').replace('\n', '\\n').replace('!', '"\'!\'"'));
-                f.write("\t${Q}chmod a+x ./$$(basename $@ .tar)/.trg.sh\n");
+                f.write("\t${Q}(echo -e \"%s\") | ${SUDO} tee .trg.sh\n" % cmd.replace('\\n', '\\\\\\n').replace('\n', '\\n').replace('!', '"\'!\'"'));
+                f.write("\t${Q}${SUDO} chmod a+x .trg.sh\n");
 
             if 'hostcmd' in target:
                 cmd = templateHostCmd % (target['hostcmd'].replace('$', '\\$$').replace('"', '\\"'));
                 f.write("\t${Q}echo '******************************'\n");
                 f.write("\t${Q}echo '%s'\n" % target['comment']);
                 f.write("\t${Q}echo '******************************'\n");
-                f.write("\t${Q}(echo -e \"%s\") | tee ./$$(basename $@ .tar)/.hst.sh\n" % cmd.replace('\\n', '\\\\\\\\n').replace('\n', '\\n').replace('!', '"\'!\'"'));
-                f.write("\t${Q}chmod a+x ./$$(basename $@ .tar)/.hst.sh\n");
+                f.write("\t${Q}(echo -e \"%s\") | ${SUDO} tee .hst.sh\n" % cmd.replace('\\n', '\\\\\\\\n').replace('\n', '\\n').replace('!', '"\'!\'"'));
+                f.write("\t${Q}${SUDO} chmod a+x .hst.sh\n");
 
             # start command here ...
             f.write("\t${Q}${SUDO} bash -c \"");
-        
-            f.write("cd $$(basename $@ .tar); tar -xf ../$<; "); 
+
+            # extracting from multiple backup levels
+            backup_levels.append(target["dep"][0])
+            f.write("cd $$(basename $@ .tar);")
+            lastincrement = None
+            for increment in backup_levels[-1:]:
+                if lastincrement == None:
+                    # level-0 backup
+                    f.write("[ -f ../%s.snar ]" % increment);
+                    f.write(" && tar -g ../%s.snar -xf ../%s" % (increment, increment)); 
+                    f.write(" || tar -xf ../%s;" % increment);
+                else:
+                    # level-x backup, update snar after extract
+                    f.write("tar -g ../%s.snar -xf ../%s;" % (increment, increment)); 
+
+                lastincrement = increment
+
             if not dry_run:
-                f.write("if [ -f .trg.sh ]; then chroot . ./.trg.sh || exit 1; fi; rm -f ./.trg.sh;");
-                f.write("if [ -f .hst.sh ]; then ./.hst.sh || exit 1; fi; rm -f ./.hst.sh;");
+                f.write("if [ -f ../.trg.sh ]; then mv ../.trg.sh .; chroot . ./.trg.sh || exit 1; fi; rm -f ./.trg.sh;");
+                f.write("if [ -f ../.hst.sh ]; then ../.hst.sh || exit 1; fi; rm -f ../.hst.sh;");
             if not 'temporary' in target or not target['temporary']:
+                # copy over snar file from previous level and add changes to the new one
+                f.write("[ ! -f ../$<.snar ] || cp ../$<.snar ../$@.snar;");
+            if incremental:
+                f.write("tar -g ../$@.snar -cf '../$@' .;");
+            else:
                 f.write("tar -cf '../$@' .;");
-                f.write("\"\n");
+            f.write("\"\n");
             # ... end command
             
 
             if 'temporary' in target and target['temporary']:
-                f.write("\t${Q}cp $< $@\n");
+                f.write("\t[ ! -f $< ] || cp $< $@;");
+                f.write("\t[ ! -f $<.snar ] || cp $<.snar $@.snar;\n");
 
             f.write("\t${Q}-${SUDO} rm -Rf ./$$(basename $@ .tar)\n");
         #