Wharfie

#!/usr/bin/python
#
# The file format and the principle are lent from Docker.
# It's just all a bit simpler, as we only create images
# and have no infrastructure to provide prebuilt images.
#
# What is more complex for embedded systems is the legal
# part. When s.o. is using docker to build a web service,
# he doesn't include the used open source software into
# a product, which he is commercially using, but he uses
# it himself to provide a web service to his customers.
#
# With most embedded systems, which Wharfie is focusing
# on, this is different. People are usually building
# products, including the open source software. So they
# need to respect the licenses of the software, which
# they are including.
#
# For this reason Wharfie only supports debian based
# systems, as the debian projects takes much care about
# respecting open source licenses, and makes it easy
# to get a list of all licenses used in a system.
#
# It makes it also easy to get the source packets, fitting
# to the installed binary version. Therefore, one can
# easily provide all sources of the open source software,
# included in his image to his customers.
#
# With this background, the supported commands of Wharfie
# are differing slightly from its big brother Docker.
#
# Commands:
# - FROM: Supports only a fix number of build rules.
#         e.g.: debian_armhf_sid, debian_etch, ...
#
# - RUN: Execute a command inside the image
#
# - RUN HOST: Execute a command on the host system,
#             But inside of the image root.
#
# - ADD: Add an archive or file to the root filesystem.
#
# - TO: name of the final archive
#
# - SOURCE: name of an archive, where the currently installed
#           source packages are written to.
#
# - LICENSE: Extract a list of all licenses, of all currently
#            installed packages.
#
# - TOOLCHAIN: Build a cross-toolchain for the currently
#              installed root filesystem.
#
# Note: Technically everything can be done, using RUN
#       and RUN HOST. Other commands, like ADD or TO
#       are only supported for more convenience.
#       So for example if you want to combine the
#       resulting image with a special bootloader or
#       something, it can be flexibly done with RUN HOST.


import os
import sys
import re
import binascii
import argparse

regexCommand = '^[ \t]*(RUN HOST|RUN|FROM|TOOLCHAIN|TO|ADD|SOURCE|LICENSE|ENV)([^\n]*)';
g_depend = '';
g_makeTargets = list();
g_environment = list();
g_archiveName = 'rootfs.tar';
g_finalTarget = list();
# We use templates, which are checking their execution context,
# to be sure, that target commands are really only executed
# in the change root of the image.
g_templateTrgCmd = "#!/bin/bash\n[ -f ../Wharfile ] && exit 2;\nmount -t proc proc /proc\n%s\numount /proc"
g_templateHostCmd = "#!/bin/bash\n[ ! -f ../Wharfile ] && exit 2;\n%s"

#
# Read a Wharfile
#
def read_wharfile(filename):
    global g_archiveName
    content = open(filename, 'r').read()
    content = content.replace('\\\n', '')
    bld = ''
    g_allDepends = '';
    g_depend = '';
    for cmd in re.findall(regexCommand, content, flags=re.MULTILINE):
        # calculate current build target and dependency names
        dep = list();
        if bld != '':
            dep.append(bld);
        if g_depend != '':
            dep.append(g_depend);

        if cmd[0] not in ('FROM', 'TO', 'ENV'):
            g_allDepends += str(cmd);
            if g_allDepends != '':
                bld = format(0xFFFFFFFF & binascii.crc32(g_allDepends), '02X') + ".piling.tar";

        # FROM
        if cmd[0] == 'FROM':
            g_depend = cmd[1] + ".tar";

        # ENV
        elif cmd[0] == 'ENV':
            g_environment.append(cmd[1]);

        # RUN
        elif cmd[0] == 'RUN':
            makeTarget = {
                'comment' : "%s %s" % (cmd[0], cmd[1]),
                'name': bld,
                'dep': dep,
                'trgcmd': cmd[1]
                };
            g_makeTargets.append(makeTarget);
            g_depend = '';

        # RUN HOST
        elif cmd[0] == 'RUN HOST':
            g_allDepends += str(cmd);
        
            makeTarget = {
                'comment' : "%s %s" % (cmd[0], cmd[1]),
                'name': bld,
                'dep': dep,
                'hostcmd': cmd[1]
                };
            g_makeTargets.append(makeTarget);
            g_depend = '';

        # ADD (single file)
        elif cmd[0] == 'ADD':
            g_allDepends += str(cmd);
            src,dst = cmd[1].lstrip().split(" ")
            dep.append(src)
            myHostCmd = 'dest="$(pwd)"; cd ../; mkdir -p $(dirname "${dest}/%s"); cp -R --preserve=mode %s "${dest}/%s"' % (dst, src, dst)
        
            makeTarget = {
                'comment' : "%s %s" % (cmd[0], cmd[1]),
                'name': bld,
                'dep': dep,
                'hostcmd': myHostCmd
                };
            g_makeTargets.append(makeTarget);
            g_depend = '';

        # TO
        elif cmd[0] == 'TO':
            g_archiveName = cmd[1].lstrip();

        # SOURCE
        elif cmd[0] == 'SOURCE':
            myTrgCmd = 'mkdir /sources; cd /sources; [ ! -f .pkg.list ] && dpkg-query -f \'${binary:Package}\\n\' -W > .pkg.list; apt-get install -y dpkg-dev; cat .pkg.list | xargs -n 1 apt-get source'
            myHostCmd = 'tar -cf ../' + cmd[1].lstrip() + ' sources;'
            makeTarget = {
                'comment' : "%s %s" % (cmd[0], cmd[1]),
                'name': bld,
                'dep': dep,
                'trgcmd': myTrgCmd,
                'hostcmd': myHostCmd,
                'temporary': True
                };
            g_makeTargets.append(makeTarget);
            g_depend = '';

        # LICENSE
        elif cmd[0] == 'LICENSE':
            myTrgCmd = 'mkdir /licenses; for i in /usr/share/doc/*/; do [ -f $i/copyright ] && cp $i/copyright licenses/$(basename $i).txt || echo $(basename $i) >> licenses/___MISSING___.txt; done'
            myHostCmd = 'tar -cf ../' + cmd[1].lstrip() + ' licenses;'
            makeTarget = {
                'comment' : "%s %s" % (cmd[0], cmd[1]),
                'name': bld,
                'dep': dep,
                'trgcmd': myTrgCmd,
                'hostcmd': myHostCmd,
                'temporary': True
                };
            g_makeTargets.append(makeTarget);
            g_depend = '';

        # TOOLCHAIN
        elif cmd[0] == 'TOOLCHAIN':
            myTrgCmd = 'apt-get install -y libc6-dev; cd ./usr/lib/arm-linux-gnueabi*/; ln -s crt1.o crt0.o'
            myHostCmd = 'rm .tmp.sh; mkdir target; mv ./* target; mkdir host; (cd host; tar -xf ../../debian_toolchain.tar); cp ../debian_toolchain_env.sh env.sh; tar -cf ../' + cmd[1].lstrip() + ' .;'
            dep.append('debian_toolchain.tar')
            dep.append('debian_toolchain_env.sh')
            makeTarget = {
                'comment' : "%s %s" % (cmd[0], cmd[1]),
                'name': bld,
                'dep': dep,
                'trgcmd': myTrgCmd,
                'hostcmd': myHostCmd,
                'temporary': True
                };
            g_makeTargets.append(makeTarget);
            g_depend = '';

        else:
            print ('unknown command: ' + cmd[0] + ' ' + cmd[1]);

    g_finalTarget.append(bld);


#
# Write a Makefile
#
def write_makefile(filename, dry_run, installpath='.'):
    f = open(filename, 'w');
    # write header
    f.write("ifneq (VERBOSE,y)\n")
    f.write("Q=@\n")
    f.write("endif\n")
    f.write("%s: %s\n" % (g_archiveName, "".join(g_finalTarget)));
    f.write("\tcp $< $@\n");
    f.write("\n");

    
    # write all environment variables
    for env in g_environment:
        key,val = env.lstrip().split(" ")
        f.write("export %s=%s\n" % (key, val));

    # write all targets
    for target in g_makeTargets:
        if 'comment' in target:
            f.write("# %s\n" % target['comment']);
        f.write("%s: %s\n" % (target['name'], " ".join(target['dep'])));
        f.write("\t${Q}-mkdir $$(basename $@ .tar)\n");

        if 'trgcmd' in target:
            cmd = g_templateTrgCmd % (target['trgcmd'].replace('$', '\\$$').replace('"', '\\"'));
            f.write("\t${Q}(cd $$(basename $@ .tar); ${SUDO} tar -xf ../$<;)\n");
            f.write("\t${Q}echo '******************************'\n");
            f.write("\t${Q}echo '%s'\n" % target['comment']);
            f.write("\t${Q}echo '******************************'\n");
            f.write("\t${Q}(echo -e \"%s\") | ${SUDO} tee ./$$(basename $@ .tar)/.tmp.sh\n" % cmd.replace('\\n', '\\\\\\n').replace('\n', '\\n').replace('!', '"\'!\'"'));
            f.write("\t${Q}${SUDO} chmod a+x ./$$(basename $@ .tar)/.tmp.sh\n");
            if not dry_run:
                f.write("\t${Q}(cd $$(basename $@ .tar); ${SUDO} chroot . ./.tmp.sh)\n");
            else:
                f.write("\t${Q}echo 'Skipped the following script:'; cat $$(basename $@ .tar)/.tmp.sh");
            f.write("\t${Q}-${SUDO} rm ./$$(basename $@ .tar)/.tmp.sh\n");

        if 'hostcmd' in target:
            cmd = g_templateHostCmd % (target['hostcmd'].replace('$', '\\$$').replace('"', '\\"'));
            f.write("\t${Q}(cd $$(basename $@ .tar); ${SUDO} tar -xf ../$<;)\n");
            f.write("\t${Q}(echo -e \"%s\") | ${SUDO} tee ./$$(basename $@ .tar)/.tmp.sh\n" % cmd.replace('\\n', '\\\\\\\\n').replace('\n', '\\n').replace('!', '"\'!\'"'));
            f.write("\t${Q}${SUDO} chmod a+x ./$$(basename $@ .tar)/.tmp.sh\n");
            if not dry_run:
                f.write("\t${Q}(cd $$(basename $@ .tar); ${SUDO} ./.tmp.sh)\n");
            else:
                f.write("\t${Q}echo 'Skipped the following script:'; cat $$(basename $@ .tar)/.tmp.sh");
            f.write("\t${Q}-${SUDO} rm ./$$(basename $@ .tar)/.tmp.sh\n");


        if 'temporary' in target and target['temporary']:
            f.write("\t${Q}cp $< $@\n");
        else:
            f.write("\t${Q}cd $$(basename $@ .tar); ${SUDO} tar -cf '../$@' .\n");
        f.write("\t${Q}-${SUDO} rm -R ./$$(basename $@ .tar)\n");
        f.write("\n");

    # write footer
    f.write("include %s/wharfie.mk\n" % installpath);
    if installpath != '.':
        f.write("-include wharfie.mk\n");
    f.write("\n");

        
#
# Main
#
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--gen-only', action='store_true', help="Generate makefile only, but don't build it" )
    parser.add_argument('--dry-run', action='store_true', help="Generate makefile with disabled run actions and don't build it" )
    parser.add_argument('wharfile', default='Wharfile', nargs='?', help="Filename of a 'Wharfile'. By default ./Wharfile is used." )
    args = parser.parse_args()

    read_wharfile('Wharfile');
    if os.path.isfile(os.path.abspath(os.path.dirname(sys.argv[0])) + "/wharfie.mk"):
        write_makefile('Makefile', args.dry_run);
    else:
        write_makefile('Makefile', args.dry_run, os.path.abspath(os.path.dirname(sys.argv[0])) + "/../share/wharfie");

    if not args.gen_only and not args.dry_run:
        os.system("make");

        
if __name__ == "__main__":
    main()