Wharfie

--- a
+++ b/trunk/wharfie/wharfie.py
@@ -0,0 +1,295 @@
+#!/usr/bin/python
+#
+# The file format and the principle are lent from Docker.
+# It's just all a bit simpler, as we only create images
+# and have no infrastructure to provide prebuilt images.
+#
+# What is more complex for embedded systems is the legal
+# part. When s.o. is using docker to build a web service,
+# he doesn't include the used open source software into
+# a product, which he is commercially using, but he uses
+# it himself to provide a web service to his customers.
+#
+# With most embedded systems, which Wharfie is focusing
+# on, this is different. People are usually building
+# products, including the open source software. So they
+# need to respect the licenses of the software, which
+# they are including.
+#
+# For this reason Wharfie only supports debian based
+# systems, as the debian projects takes much care about
+# respecting open source licenses, and makes it easy
+# to get a list of all licenses used in a system.
+#
+# It makes it also easy to get the source packets, fitting
+# to the installed binary version. Therefore, one can
+# easily provide all sources of the open source software,
+# included in his image to his customers.
+#
+# With this background, the supported commands of Wharfie
+# are differing slightly from its big brother Docker.
+#
+# Commands:
+# - FROM: Supports only a fix number of build rules.
+#         e.g.: debian_armhf_sid, debian_etch, ...
+#
+# - RUN: Execute a command inside the image
+#
+# - RUN HOST: Execute a command on the host system,
+#             But inside of the image root.
+#
+# - ADD: Add an archive or file to the root filesystem.
+#
+# - TO: name of the final archive
+#
+# - SOURCE: name of an archive, where the currently installed
+#           source packages are written to.
+#
+# - LICENSE: Extract a list of all licenses, of all currently
+#            installed packages.
+#
+# - TOOLCHAIN: Build a cross-toolchain for the currently
+#              installed root filesystem.
+#
+# Note: Technically everything can be done, using RUN
+#       and RUN HOST. Other commands, like ADD or TO
+#       are only supported for more convenience.
+#       So for example if you want to combine the
+#       resulting image with a special bootloader or
+#       something, it can be flexibly done with RUN HOST.
+
+
+import os
+import sys
+import re
+import binascii
+import argparse
+
+regexCommand = '^[ \t]*(RUN HOST|RUN|FROM|TOOLCHAIN|TO|ADD|SOURCE|LICENSE|ENV)([^\n]*)';
+g_depend = '';
+g_makeTargets = list();
+g_environment = list();
+g_archiveName = 'rootfs.tar';
+g_finalTarget = list();
+# We use templates, which are checking their execution context,
+# to be sure, that target commands are really only executed
+# in the change root of the image.
+g_templateTrgCmd = "#!/bin/bash\n[ -f ../Wharfile ] && exit 2;\nmount -t proc proc /proc\n%s\numount /proc"
+g_templateHostCmd = "#!/bin/bash\n[ ! -f ../Wharfile ] && exit 2;\n%s"
+
+#
+# Read a Wharfile
+#
+def read_wharfile(filename):
+    global g_archiveName
+    content = open(filename, 'r').read()
+    content = content.replace('\\\n', '')
+    bld = ''
+    g_allDepends = '';
+    g_depend = '';
+    for cmd in re.findall(regexCommand, content, flags=re.MULTILINE):
+        # calculate current build target and dependency names
+        dep = list();
+        if bld != '':
+            dep.append(bld);
+        if g_depend != '':
+            dep.append(g_depend);
+
+        if cmd[0] not in ('FROM', 'TO', 'ENV'):
+            g_allDepends += str(cmd);
+            if g_allDepends != '':
+                bld = format(0xFFFFFFFF & binascii.crc32(g_allDepends), '02X') + ".piling.tar";
+
+        # FROM
+        if cmd[0] == 'FROM':
+            g_depend = cmd[1] + ".tar";
+
+        # ENV
+        elif cmd[0] == 'ENV':
+            g_environment.append(cmd[1]);
+
+        # RUN
+        elif cmd[0] == 'RUN':
+            makeTarget = {
+                'comment' : "%s %s" % (cmd[0], cmd[1]),
+                'name': bld,
+                'dep': dep,
+                'trgcmd': cmd[1]
+                };
+            g_makeTargets.append(makeTarget);
+            g_depend = '';
+
+        # RUN HOST
+        elif cmd[0] == 'RUN HOST':
+            g_allDepends += str(cmd);
+        
+            makeTarget = {
+                'comment' : "%s %s" % (cmd[0], cmd[1]),
+                'name': bld,
+                'dep': dep,
+                'hostcmd': cmd[1]
+                };
+            g_makeTargets.append(makeTarget);
+            g_depend = '';
+
+        # ADD (single file)
+        elif cmd[0] == 'ADD':
+            g_allDepends += str(cmd);
+            src,dst = cmd[1].lstrip().split(" ")
+            dep.append(src)
+            myHostCmd = 'dest="$(pwd)"; cd ../; mkdir -p $(dirname "${dest}/%s"); cp -R --preserve=mode %s "${dest}/%s"' % (dst, src, dst)
+        
+            makeTarget = {
+                'comment' : "%s %s" % (cmd[0], cmd[1]),
+                'name': bld,
+                'dep': dep,
+                'hostcmd': myHostCmd
+                };
+            g_makeTargets.append(makeTarget);
+            g_depend = '';
+
+        # TO
+        elif cmd[0] == 'TO':
+            g_archiveName = cmd[1].lstrip();
+
+        # SOURCE
+        elif cmd[0] == 'SOURCE':
+            myTrgCmd = 'mkdir /sources; cd /sources; [ ! -f .pkg.list ] && dpkg-query -f \'${binary:Package}\\n\' -W > .pkg.list; apt-get install -y dpkg-dev; cat .pkg.list | xargs -n 1 apt-get source'
+            myHostCmd = 'tar -cf ../' + cmd[1].lstrip() + ' sources;'
+            makeTarget = {
+                'comment' : "%s %s" % (cmd[0], cmd[1]),
+                'name': bld,
+                'dep': dep,
+                'trgcmd': myTrgCmd,
+                'hostcmd': myHostCmd,
+                'temporary': True
+                };
+            g_makeTargets.append(makeTarget);
+            g_depend = '';
+
+        # LICENSE
+        elif cmd[0] == 'LICENSE':
+            myTrgCmd = 'mkdir /licenses; for i in /usr/share/doc/*/; do [ -f $i/copyright ] && cp $i/copyright licenses/$(basename $i).txt || echo $(basename $i) >> licenses/___MISSING___.txt; done'
+            myHostCmd = 'tar -cf ../' + cmd[1].lstrip() + ' licenses;'
+            makeTarget = {
+                'comment' : "%s %s" % (cmd[0], cmd[1]),
+                'name': bld,
+                'dep': dep,
+                'trgcmd': myTrgCmd,
+                'hostcmd': myHostCmd,
+                'temporary': True
+                };
+            g_makeTargets.append(makeTarget);
+            g_depend = '';
+
+        # TOOLCHAIN
+        elif cmd[0] == 'TOOLCHAIN':
+            myTrgCmd = 'apt-get install -y libc6-dev; cd ./usr/lib/arm-linux-gnueabi*/; ln -s crt1.o crt0.o'
+            myHostCmd = 'rm .tmp.sh; mkdir target; mv ./* target; mkdir host; (cd host; tar -xf ../../debian_toolchain.tar); cp ../debian_toolchain_env.sh env.sh; tar -cf ../' + cmd[1].lstrip() + ' .;'
+            dep.append('debian_toolchain.tar')
+            dep.append('debian_toolchain_env.sh')
+            makeTarget = {
+                'comment' : "%s %s" % (cmd[0], cmd[1]),
+                'name': bld,
+                'dep': dep,
+                'trgcmd': myTrgCmd,
+                'hostcmd': myHostCmd,
+                'temporary': True
+                };
+            g_makeTargets.append(makeTarget);
+            g_depend = '';
+
+        else:
+            print ('unknown command: ' + cmd[0] + ' ' + cmd[1]);
+
+    g_finalTarget.append(bld);
+
+
+#
+# Write a Makefile
+#
+def write_makefile(filename, dry_run, installpath='.'):
+    f = open(filename, 'w');
+    # write header
+    f.write("ifneq (VERBOSE,y)\n")
+    f.write("Q=@\n")
+    f.write("endif\n")
+    f.write("%s: %s\n" % (g_archiveName, "".join(g_finalTarget)));
+    f.write("\tcp $< $@\n");
+    f.write("\n");
+
+    
+    # write all environment variables
+    for env in g_environment:
+        key,val = env.lstrip().split(" ")
+        f.write("export %s=%s\n" % (key, val));
+
+    # write all targets
+    for target in g_makeTargets:
+        if 'comment' in target:
+            f.write("# %s\n" % target['comment']);
+        f.write("%s: %s\n" % (target['name'], " ".join(target['dep'])));
+        f.write("\t${Q}-mkdir $$(basename $@ .tar)\n");
+
+        if 'trgcmd' in target:
+            cmd = g_templateTrgCmd % (target['trgcmd'].replace('$', '\\$$').replace('"', '\\"'));
+            f.write("\t${Q}(cd $$(basename $@ .tar); ${SUDO} tar -xf ../$<;)\n");
+            f.write("\t${Q}echo '******************************'\n");
+            f.write("\t${Q}echo '%s'\n" % target['comment']);
+            f.write("\t${Q}echo '******************************'\n");
+            f.write("\t${Q}(echo -e \"%s\") | ${SUDO} tee ./$$(basename $@ .tar)/.tmp.sh\n" % cmd.replace('\\n', '\\\\\\n').replace('\n', '\\n').replace('!', '"\'!\'"'));
+            f.write("\t${Q}${SUDO} chmod a+x ./$$(basename $@ .tar)/.tmp.sh\n");
+            if not dry_run:
+                f.write("\t${Q}(cd $$(basename $@ .tar); ${SUDO} chroot . ./.tmp.sh)\n");
+            else:
+                f.write("\t${Q}echo 'Skipped the following script:'; cat $$(basename $@ .tar)/.tmp.sh");
+            f.write("\t${Q}-${SUDO} rm ./$$(basename $@ .tar)/.tmp.sh\n");
+
+        if 'hostcmd' in target:
+            cmd = g_templateHostCmd % (target['hostcmd'].replace('$', '\\$$').replace('"', '\\"'));
+            f.write("\t${Q}(cd $$(basename $@ .tar); ${SUDO} tar -xf ../$<;)\n");
+            f.write("\t${Q}(echo -e \"%s\") | ${SUDO} tee ./$$(basename $@ .tar)/.tmp.sh\n" % cmd.replace('\\n', '\\\\\\\\n').replace('\n', '\\n').replace('!', '"\'!\'"'));
+            f.write("\t${Q}${SUDO} chmod a+x ./$$(basename $@ .tar)/.tmp.sh\n");
+            if not dry_run:
+                f.write("\t${Q}(cd $$(basename $@ .tar); ${SUDO} ./.tmp.sh)\n");
+            else:
+                f.write("\t${Q}echo 'Skipped the following script:'; cat $$(basename $@ .tar)/.tmp.sh");
+            f.write("\t${Q}-${SUDO} rm ./$$(basename $@ .tar)/.tmp.sh\n");
+
+
+        if 'temporary' in target and target['temporary']:
+            f.write("\t${Q}cp $< $@\n");
+        else:
+            f.write("\t${Q}cd $$(basename $@ .tar); ${SUDO} tar -cf '../$@' .\n");
+        f.write("\t${Q}-${SUDO} rm -R ./$$(basename $@ .tar)\n");
+        f.write("\n");
+
+    # write footer
+    f.write("include %s/wharfie.mk\n" % installpath);
+    if installpath != '.':
+        f.write("-include wharfie.mk\n");
+    f.write("\n");
+
+        
+#
+# Main
+#
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--gen-only', action='store_true', help="Generate makefile only, but don't build it" )
+    parser.add_argument('--dry-run', action='store_true', help="Generate makefile with disabled run actions and don't build it" )
+    parser.add_argument('wharfile', default='Wharfile', nargs='?', help="Filename of a 'Wharfile'. By default ./Wharfile is used." )
+    args = parser.parse_args()
+
+    read_wharfile('Wharfile');
+    if os.path.isfile(os.path.abspath(os.path.dirname(sys.argv[0])) + "/wharfie.mk"):
+        write_makefile('Makefile', args.dry_run);
+    else:
+        write_makefile('Makefile', args.dry_run, os.path.abspath(os.path.dirname(sys.argv[0])) + "/../share/wharfie");
+
+    if not args.gen_only and not args.dry_run:
+        os.system("make");
+
+        
+if __name__ == "__main__":
+    main()