Wharfie

#!/usr/bin/python
#
# Copyright 2017 Ingo Hornberger <ingo_@gmx.net>
#
# This software is licensed under the MIT License
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of this
# software and associated documentation files (the "Software"), to deal in the Software
# without restriction, including without limitation the rights to use, copy, modify,
# merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be included in all copies
# or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
# OR OTHER DEALINGS IN THE SOFTWARE.
#
################################################################################
# We use templates, which are checking their execution context,
# to be sure, that target commands are really only executed
# in the change root of the image.
#
# We use snar files to create incremental archives. Those files are targeted
# to incremental backups, where the filestructure stays roughly the same. So
# beside the name and timestamps of the files, it also relies on inodes.
#
# As we are packing and extracting the images for every step, the inodes are
# constantly changing. As tar doesn't support to ignore inodes on incremental
# backups, we are refreshing the backups in every step.
#
#          input snar    output snar    output tar
# level 0: <none>        l0.snar        l0.tar
# level 1: l0.snar       l1.snar        l1.tar
# level 2: l1.snar       l2.snar        l2.tar
#
# When we prepare the rootfs for level 2, we have to do the following:
#
# - tar -xf l0.tar -g l0.snar
#
# - tar -xf l1.tar -g l1.snar
# - cp l0.snar l1.snar
# - tar -cf /dev/zero -g l1.snar .
#
# - tar -xf l2.tar -g l2.snar
# - cp l1.snar l2.snar
# - tar -cf /dev/zero -g l2.snar .
#
#################################################################################
templateTrgCmd = "#!/bin/bash\n[ -f ../Wharfile ] && exit 2;\n%s\nexit \$$?\n"
templateHostCmd = "#!/bin/bash\n[ ! -f ../Wharfile ] && exit 2;\n%s\nexit \$$?\n"

makeTargets = list();
environment = list();
finalTarget = list();

#
# Write a Makefile
#
def write_makefile(filename, dry_run, installpath='.', incremental=False, proc=True):
    backup_levels=list()
    f = open(filename, 'w');
    # write header
    f.write("ifneq (${VERBOSE},y)\n")
    f.write("Q=@\n")
    f.write("endif\n")
    f.write("\n")
    f.write("OUTPUT_FILE=%s\n" % (finalTarget[0]))
    f.write("\n")
    f.write("all: %s\n\n" % (finalTarget[0]));
    
    # write all environment variables
    for env in environment:
        l = env.lstrip().split(" ")
        f.write("export %s=%s\n" % (l[0], " ".join(l[1:])));

    # write all targets
    for target in makeTargets:
        postpath = ''
        
        if 'comment' in target:
            f.write("# %s\n" % target['comment']);
        if 'trgcmd' in target or 'hostcmd' in target or 'temporary' in target:
            # independent of if the command was successful or not, we cleanup the directory
            # - killall processes running inside
            # - umount proc, if necessary
            # - delete the directory
            cleanup_commands="${SUDO} fuser -k ./rootfs.piling;" \
                "[ -d ./rootfs.piling/proc ] && ${SUDO} umount ./rootfs.piling/proc;" \
                "${SUDO} rm -Rf ./rootfs.piling";
            f.write("%s: %s\n" % (target['name'], " ".join(target['dep'])));
            f.write("\t${Q}-mkdir rootfs.piling\n");
            f.write("\t${Q}${MAKE} %s_sub || (%s; false)\n" % (target['name'], cleanup_commands));
            f.write("\t${Q}-%s\n" % cleanup_commands);
            # don't add any dependencies to a subtarget. Resolving them in the sub-process would lead
            # to really unexpected effects. The sub-target is just called, that we can tidy things up
            # well in the main build target.
            f.write("\n");
            f.write(".PHONY: %s_sub\n" % (target['name']));
            f.write("%s_sub:\n" % (target['name']));
            # target and host commands have to extract and repack always.
            # simple commands don't do that.
            f.write("\t${Q}${SUDO} rm -f .trg.sh .hst.sh\n");
            if 'trgcmd' in target:
                cmd = templateTrgCmd % (target['trgcmd'].replace('$', '\\$$').replace('"', '\\"'));
                f.write("\t${Q}echo '******************************'\n");
                f.write("\t${Q}echo '%s: %s'\n" % (target['name'], target['comment']));
                f.write("\t${Q}echo '******************************'\n");
                f.write("\t${Q}(echo -e \"%s\") | ${SUDO} tee .trg.sh\n" % cmd.replace('\\n', '\\\\\\n').replace('\n', '\\n').replace('!', '"\'!\'"'));
                f.write("\t${Q}${SUDO} chmod a+x .trg.sh\n");

            if 'trgcmdpost' in target:
                cmd = templateTrgCmd % (target['trgcmdpost'].replace('$', '\\$$').replace('"', '\\"'));
                f.write("\t${Q}echo '******************************'\n");
                f.write("\t${Q}echo '%s: %s'\n" % (target['name'], target['comment']));
                f.write("\t${Q}echo '******************************'\n");
                f.write("\t${Q}(echo -e \"%s\") | ${SUDO} tee .trg-post.sh\n" % cmd.replace('\\n', '\\\\\\n').replace('\n', '\\n').replace('!', '"\'!\'"'));
                f.write("\t${Q}${SUDO} chmod a+x .trg-post.sh\n");

            if 'trgcmdpostpath' in target:
                postpath = target['trgcmdpostpath']
                
            if 'hostcmd' in target:
                cmd = templateHostCmd % (target['hostcmd'].replace('$', '\\$$').replace('"', '\\"'));
                f.write("\t${Q}echo '******************************'\n");
                f.write("\t${Q}echo '%s: %s'\n" % (target['name'], target['comment']));
                f.write("\t${Q}echo '******************************'\n");
                f.write("\t${Q}(echo -e \"%s\") | ${SUDO} tee .hst.sh\n" % cmd.replace('\\n', '\\\\\\\\n').replace('\n', '\\n').replace('!', '"\'!\'"'));
                f.write("\t${Q}${SUDO} chmod a+x .hst.sh\n");

            if 'hostcmdpost' in target:
                cmd = templateHostCmd % (target['hostcmdpost'].replace('$', '\\$$').replace('"', '\\"'));
                f.write("\t${Q}echo '******************************'\n");
                f.write("\t${Q}echo '%s: %s'\n" % (target['name'], target['comment']));
                f.write("\t${Q}echo '******************************'\n");
                f.write("\t${Q}(echo -e \"%s\") | ${SUDO} tee .hst-post.sh\n" % cmd.replace('\\n', '\\\\\\\\n').replace('\n', '\\n').replace('!', '"\'!\'"'));
                f.write("\t${Q}${SUDO} chmod a+x .hst-post.sh\n");

            # start command here ...
            f.write("\t${Q}${SUDO} bash -c \"");

            # extracting from multiple backup levels
            backup_levels.append(target["dep"][0])
            f.write("cd rootfs.piling;")
            lastincrement = None
            for increment in backup_levels[-1:]:
                if lastincrement == None:
                    # level-0 backup
                    f.write("[ -f ../%s.snar ]" % increment);
                    f.write(" && tar -g ../%s.snar -xf ../%s" % (increment, increment)); 
                    f.write(" || tar -xf ../%s;" % increment);
                else:
                    # level-x backup, update snar after extract
                    f.write("tar -g ../%s.snar -xf ../%s;" % (increment, increment)); 

                lastincrement = increment

            if not dry_run:
                # mount proc here if such a folder exists. it is umounted after this command or
                # in the cleanup actions
                # - target
                if proc:
                    f.write("if [ -d proc ]; then mount -t proc proc proc; fi;");
                f.write("if [ -f ../.trg.sh ]; then mv ../.trg.sh .; chroot . ./.trg.sh || exit 1; fi; rm -f ./.trg.sh;");
                if proc:
                    f.write("if [ -d proc ]; then umount proc || umount -l proc; fi;");

                # - host
                f.write("if [ -f ../.hst.sh ]; then ../.hst.sh || exit 1; fi; rm -f ../.hst.sh;");

                # - target post
                if proc:
                    f.write("if [ -d proc ]; then mount -t proc proc proc; fi;");
                f.write("if [ -f ../.trg-post.sh ]; then mv ../.trg-post.sh ./%s; chroot ./%s ./.trg-post.sh || exit 1; fi; rm -f ./.trg-post.sh;" % (postpath, postpath));
                if proc:
                    f.write("if [ -d proc ]; then umount proc || umount -l proc; fi;");

                # - host post
                f.write("if [ -f ../.hst-post.sh ]; then ../.hst-post.sh || exit 1; fi; rm -f ../.hst-post.sh;");


            # pack result
            if not 'temporary' in target or not target['temporary']:
                # copy over snar file from previous level and add changes to the new one
                f.write("[ ! -f ../%s.snar ] || cp ../%s.snar ../%s.snar;" % (target['dep'][0], target['dep'][0], target['name']));
            if incremental:
                f.write("tar -g ../%s.snar -cf '../%s' .;" % (target['name'], target['name']));
            else:
                if not 'temporary' in target or not target['temporary']:
                    f.write("tar -cf '../%s' .;" % target['name']);
            f.write("\"\n");
            # ... end command
            

            if 'temporary' in target and target['temporary']:
                f.write("\t[ ! -f %s ] || cp %s %s;" % (target['dep'][0], target['dep'][0], target['name']));
                f.write("\t[ ! -f %s.snar ] || cp %s.snar %s.snar;\n" % (target['dep'][0], target['dep'][0], target['name']));

        #
        # end of target/host commands.
        # we keep it open that those command types can still be combined
        # with a simple command.
        #

        if 'simplecmd' in target:
            f.write("%s: %s\n" % (target['name'], " ".join(target['dep'])));
            # note, that a simple command has full control
            # over make. it can contain references to the maketarget
            # and to make variables. Therefore we don't replace anything
            # in those commands (contrary to target-/host commands).
            # Also verbose mode can't be supported, because we want to
            # support multiline commands
            for line in iter(target['simplecmd'].splitlines()):
                f.write("\t%s\n" % line)
            
        f.write("\n");

    # write footer
    f.write("WHARFIE_MK ?= %s/wharfie.mk\n" % installpath);
    f.write("include ${WHARFIE_MK}\n");
    f.write("\n");