a | b/trunk/wharfie/wharfie.py | ||
---|---|---|---|
1 | #!/usr/bin/python |
||
2 | # |
||
3 | # The file format and the principle are lent from Docker. |
||
4 | # It's just all a bit simpler, as we only create images |
||
5 | # and have no infrastructure to provide prebuilt images. |
||
6 | # |
||
7 | # What is more complex for embedded systems is the legal |
||
8 | # part. When s.o. is using docker to build a web service, |
||
9 | # he doesn't include the used open source software into |
||
10 | # a product, which he is commercially using, but he uses |
||
11 | # it himself to provide a web service to his customers. |
||
12 | # |
||
13 | # With most embedded systems, which Wharfie is focusing |
||
14 | # on, this is different. People are usually building |
||
15 | # products, including the open source software. So they |
||
16 | # need to respect the licenses of the software, which |
||
17 | # they are including. |
||
18 | # |
||
19 | # For this reason Wharfie only supports debian based |
||
20 | # systems, as the debian projects takes much care about |
||
21 | # respecting open source licenses, and makes it easy |
||
22 | # to get a list of all licenses used in a system. |
||
23 | # |
||
24 | # It makes it also easy to get the source packets, fitting |
||
25 | # to the installed binary version. Therefore, one can |
||
26 | # easily provide all sources of the open source software, |
||
27 | # included in his image to his customers. |
||
28 | # |
||
29 | # With this background, the supported commands of Wharfie |
||
30 | # are differing slightly from its big brother Docker. |
||
31 | # |
||
32 | # Commands: |
||
33 | # - FROM: Supports only a fix number of build rules. |
||
34 | # e.g.: debian_armhf_sid, debian_etch, ... |
||
35 | # |
||
36 | # - RUN: Execute a command inside the image |
||
37 | # |
||
38 | # - RUN HOST: Execute a command on the host system, |
||
39 | # But inside of the image root. |
||
40 | # |
||
41 | # - ADD: Add an archive or file to the root filesystem. |
||
42 | # |
||
43 | # - TO: name of the final archive |
||
44 | # |
||
45 | # - SOURCE: name of an archive, where the currently installed |
||
46 | # source packages are written to. |
||
47 | # |
||
48 | # - LICENSE: Extract a list of all licenses, of all currently |
||
49 | # installed packages. |
||
50 | # |
||
51 | # - TOOLCHAIN: Build a cross-toolchain for the currently |
||
52 | # installed root filesystem. |
||
53 | # |
||
54 | # Note: Technically everything can be done, using RUN |
||
55 | # and RUN HOST. Other commands, like ADD or TO |
||
56 | # are only supported for more convenience. |
||
57 | # So for example if you want to combine the |
||
58 | # resulting image with a special bootloader or |
||
59 | # something, it can be flexibly done with RUN HOST. |
||
60 | |||
61 | |||
62 | import os |
||
63 | import sys |
||
64 | import re |
||
65 | import binascii |
||
66 | import argparse |
||
67 | |||
68 | regexCommand = '^[ \t]*(RUN HOST|RUN|FROM|TOOLCHAIN|TO|ADD|SOURCE|LICENSE|ENV)([^\n]*)'; |
||
69 | g_depend = ''; |
||
70 | g_makeTargets = list(); |
||
71 | g_environment = list(); |
||
72 | g_archiveName = 'rootfs.tar'; |
||
73 | g_finalTarget = list(); |
||
74 | # We use templates, which are checking their execution context, |
||
75 | # to be sure, that target commands are really only executed |
||
76 | # in the change root of the image. |
||
77 | g_templateTrgCmd = "#!/bin/bash\n[ -f ../Wharfile ] && exit 2;\nmount -t proc proc /proc\n%s\numount /proc" |
||
78 | g_templateHostCmd = "#!/bin/bash\n[ ! -f ../Wharfile ] && exit 2;\n%s" |
||
79 | |||
80 | # |
||
81 | # Read a Wharfile |
||
82 | # |
||
83 | def read_wharfile(filename): |
||
84 | global g_archiveName |
||
85 | content = open(filename, 'r').read() |
||
86 | content = content.replace('\\\n', '') |
||
87 | bld = '' |
||
88 | g_allDepends = ''; |
||
89 | g_depend = ''; |
||
90 | for cmd in re.findall(regexCommand, content, flags=re.MULTILINE): |
||
91 | # calculate current build target and dependency names |
||
92 | dep = list(); |
||
93 | if bld != '': |
||
94 | dep.append(bld); |
||
95 | if g_depend != '': |
||
96 | dep.append(g_depend); |
||
97 | |||
98 | if cmd[0] not in ('FROM', 'TO', 'ENV'): |
||
99 | g_allDepends += str(cmd); |
||
100 | if g_allDepends != '': |
||
101 | bld = format(0xFFFFFFFF & binascii.crc32(g_allDepends), '02X') + ".piling.tar"; |
||
102 | |||
103 | # FROM |
||
104 | if cmd[0] == 'FROM': |
||
105 | g_depend = cmd[1] + ".tar"; |
||
106 | |||
107 | # ENV |
||
108 | elif cmd[0] == 'ENV': |
||
109 | g_environment.append(cmd[1]); |
||
110 | |||
111 | # RUN |
||
112 | elif cmd[0] == 'RUN': |
||
113 | makeTarget = { |
||
114 | 'comment' : "%s %s" % (cmd[0], cmd[1]), |
||
115 | 'name': bld, |
||
116 | 'dep': dep, |
||
117 | 'trgcmd': cmd[1] |
||
118 | }; |
||
119 | g_makeTargets.append(makeTarget); |
||
120 | g_depend = ''; |
||
121 | |||
122 | # RUN HOST |
||
123 | elif cmd[0] == 'RUN HOST': |
||
124 | g_allDepends += str(cmd); |
||
125 | |||
126 | makeTarget = { |
||
127 | 'comment' : "%s %s" % (cmd[0], cmd[1]), |
||
128 | 'name': bld, |
||
129 | 'dep': dep, |
||
130 | 'hostcmd': cmd[1] |
||
131 | }; |
||
132 | g_makeTargets.append(makeTarget); |
||
133 | g_depend = ''; |
||
134 | |||
135 | # ADD (single file) |
||
136 | elif cmd[0] == 'ADD': |
||
137 | g_allDepends += str(cmd); |
||
138 | src,dst = cmd[1].lstrip().split(" ") |
||
139 | dep.append(src) |
||
140 | myHostCmd = 'dest="$(pwd)"; cd ../; mkdir -p $(dirname "${dest}/%s"); cp -R --preserve=mode %s "${dest}/%s"' % (dst, src, dst) |
||
141 | |||
142 | makeTarget = { |
||
143 | 'comment' : "%s %s" % (cmd[0], cmd[1]), |
||
144 | 'name': bld, |
||
145 | 'dep': dep, |
||
146 | 'hostcmd': myHostCmd |
||
147 | }; |
||
148 | g_makeTargets.append(makeTarget); |
||
149 | g_depend = ''; |
||
150 | |||
151 | # TO |
||
152 | elif cmd[0] == 'TO': |
||
153 | g_archiveName = cmd[1].lstrip(); |
||
154 | |||
155 | # SOURCE |
||
156 | elif cmd[0] == 'SOURCE': |
||
157 | myTrgCmd = 'mkdir /sources; cd /sources; [ ! -f .pkg.list ] && dpkg-query -f \'${binary:Package}\\n\' -W > .pkg.list; apt-get install -y dpkg-dev; cat .pkg.list | xargs -n 1 apt-get source' |
||
158 | myHostCmd = 'tar -cf ../' + cmd[1].lstrip() + ' sources;' |
||
159 | makeTarget = { |
||
160 | 'comment' : "%s %s" % (cmd[0], cmd[1]), |
||
161 | 'name': bld, |
||
162 | 'dep': dep, |
||
163 | 'trgcmd': myTrgCmd, |
||
164 | 'hostcmd': myHostCmd, |
||
165 | 'temporary': True |
||
166 | }; |
||
167 | g_makeTargets.append(makeTarget); |
||
168 | g_depend = ''; |
||
169 | |||
170 | # LICENSE |
||
171 | elif cmd[0] == 'LICENSE': |
||
172 | myTrgCmd = 'mkdir /licenses; for i in /usr/share/doc/*/; do [ -f $i/copyright ] && cp $i/copyright licenses/$(basename $i).txt || echo $(basename $i) >> licenses/___MISSING___.txt; done' |
||
173 | myHostCmd = 'tar -cf ../' + cmd[1].lstrip() + ' licenses;' |
||
174 | makeTarget = { |
||
175 | 'comment' : "%s %s" % (cmd[0], cmd[1]), |
||
176 | 'name': bld, |
||
177 | 'dep': dep, |
||
178 | 'trgcmd': myTrgCmd, |
||
179 | 'hostcmd': myHostCmd, |
||
180 | 'temporary': True |
||
181 | }; |
||
182 | g_makeTargets.append(makeTarget); |
||
183 | g_depend = ''; |
||
184 | |||
185 | # TOOLCHAIN |
||
186 | elif cmd[0] == 'TOOLCHAIN': |
||
187 | myTrgCmd = 'apt-get install -y libc6-dev; cd ./usr/lib/arm-linux-gnueabi*/; ln -s crt1.o crt0.o' |
||
188 | myHostCmd = 'rm .tmp.sh; mkdir target; mv ./* target; mkdir host; (cd host; tar -xf ../../debian_toolchain.tar); cp ../debian_toolchain_env.sh env.sh; tar -cf ../' + cmd[1].lstrip() + ' .;' |
||
189 | dep.append('debian_toolchain.tar') |
||
190 | dep.append('debian_toolchain_env.sh') |
||
191 | makeTarget = { |
||
192 | 'comment' : "%s %s" % (cmd[0], cmd[1]), |
||
193 | 'name': bld, |
||
194 | 'dep': dep, |
||
195 | 'trgcmd': myTrgCmd, |
||
196 | 'hostcmd': myHostCmd, |
||
197 | 'temporary': True |
||
198 | }; |
||
199 | g_makeTargets.append(makeTarget); |
||
200 | g_depend = ''; |
||
201 | |||
202 | else: |
||
203 | print ('unknown command: ' + cmd[0] + ' ' + cmd[1]); |
||
204 | |||
205 | g_finalTarget.append(bld); |
||
206 | |||
207 | |||
208 | # |
||
209 | # Write a Makefile |
||
210 | # |
||
211 | def write_makefile(filename, dry_run, installpath='.'): |
||
212 | f = open(filename, 'w'); |
||
213 | # write header |
||
214 | f.write("ifneq (VERBOSE,y)\n") |
||
215 | f.write("Q=@\n") |
||
216 | f.write("endif\n") |
||
217 | f.write("%s: %s\n" % (g_archiveName, "".join(g_finalTarget))); |
||
218 | f.write("\tcp $< $@\n"); |
||
219 | f.write("\n"); |
||
220 | |||
221 | |||
222 | # write all environment variables |
||
223 | for env in g_environment: |
||
224 | key,val = env.lstrip().split(" ") |
||
225 | f.write("export %s=%s\n" % (key, val)); |
||
226 | |||
227 | # write all targets |
||
228 | for target in g_makeTargets: |
||
229 | if 'comment' in target: |
||
230 | f.write("# %s\n" % target['comment']); |
||
231 | f.write("%s: %s\n" % (target['name'], " ".join(target['dep']))); |
||
232 | f.write("\t${Q}-mkdir $$(basename $@ .tar)\n"); |
||
233 | |||
234 | if 'trgcmd' in target: |
||
235 | cmd = g_templateTrgCmd % (target['trgcmd'].replace('$', '\\$$').replace('"', '\\"')); |
||
236 | f.write("\t${Q}(cd $$(basename $@ .tar); ${SUDO} tar -xf ../$<;)\n"); |
||
237 | f.write("\t${Q}echo '******************************'\n"); |
||
238 | f.write("\t${Q}echo '%s'\n" % target['comment']); |
||
239 | f.write("\t${Q}echo '******************************'\n"); |
||
240 | f.write("\t${Q}(echo -e \"%s\") | ${SUDO} tee ./$$(basename $@ .tar)/.tmp.sh\n" % cmd.replace('\\n', '\\\\\\n').replace('\n', '\\n').replace('!', '"\'!\'"')); |
||
241 | f.write("\t${Q}${SUDO} chmod a+x ./$$(basename $@ .tar)/.tmp.sh\n"); |
||
242 | if not dry_run: |
||
243 | f.write("\t${Q}(cd $$(basename $@ .tar); ${SUDO} chroot . ./.tmp.sh)\n"); |
||
244 | else: |
||
245 | f.write("\t${Q}echo 'Skipped the following script:'; cat $$(basename $@ .tar)/.tmp.sh"); |
||
246 | f.write("\t${Q}-${SUDO} rm ./$$(basename $@ .tar)/.tmp.sh\n"); |
||
247 | |||
248 | if 'hostcmd' in target: |
||
249 | cmd = g_templateHostCmd % (target['hostcmd'].replace('$', '\\$$').replace('"', '\\"')); |
||
250 | f.write("\t${Q}(cd $$(basename $@ .tar); ${SUDO} tar -xf ../$<;)\n"); |
||
251 | f.write("\t${Q}(echo -e \"%s\") | ${SUDO} tee ./$$(basename $@ .tar)/.tmp.sh\n" % cmd.replace('\\n', '\\\\\\\\n').replace('\n', '\\n').replace('!', '"\'!\'"')); |
||
252 | f.write("\t${Q}${SUDO} chmod a+x ./$$(basename $@ .tar)/.tmp.sh\n"); |
||
253 | if not dry_run: |
||
254 | f.write("\t${Q}(cd $$(basename $@ .tar); ${SUDO} ./.tmp.sh)\n"); |
||
255 | else: |
||
256 | f.write("\t${Q}echo 'Skipped the following script:'; cat $$(basename $@ .tar)/.tmp.sh"); |
||
257 | f.write("\t${Q}-${SUDO} rm ./$$(basename $@ .tar)/.tmp.sh\n"); |
||
258 | |||
259 | |||
260 | if 'temporary' in target and target['temporary']: |
||
261 | f.write("\t${Q}cp $< $@\n"); |
||
262 | else: |
||
263 | f.write("\t${Q}cd $$(basename $@ .tar); ${SUDO} tar -cf '../$@' .\n"); |
||
264 | f.write("\t${Q}-${SUDO} rm -R ./$$(basename $@ .tar)\n"); |
||
265 | f.write("\n"); |
||
266 | |||
267 | # write footer |
||
268 | f.write("include %s/wharfie.mk\n" % installpath); |
||
269 | if installpath != '.': |
||
270 | f.write("-include wharfie.mk\n"); |
||
271 | f.write("\n"); |
||
272 | |||
273 | |||
274 | # |
||
275 | # Main |
||
276 | # |
||
277 | def main(): |
||
278 | parser = argparse.ArgumentParser() |
||
279 | parser.add_argument('--gen-only', action='store_true', help="Generate makefile only, but don't build it" ) |
||
280 | parser.add_argument('--dry-run', action='store_true', help="Generate makefile with disabled run actions and don't build it" ) |
||
281 | parser.add_argument('wharfile', default='Wharfile', nargs='?', help="Filename of a 'Wharfile'. By default ./Wharfile is used." ) |
||
282 | args = parser.parse_args() |
||
283 | |||
284 | read_wharfile('Wharfile'); |
||
285 | if os.path.isfile(os.path.abspath(os.path.dirname(sys.argv[0])) + "/wharfie.mk"): |
||
286 | write_makefile('Makefile', args.dry_run); |
||
287 | else: |
||
288 | write_makefile('Makefile', args.dry_run, os.path.abspath(os.path.dirname(sys.argv[0])) + "/../share/wharfie"); |
||
289 | |||
290 | if not args.gen_only and not args.dry_run: |
||
291 | os.system("make"); |
||
292 | |||
293 | |||
294 | if __name__ == "__main__": |
||
295 | main() |